feat: split storage import and indexing
1 file changed, 29 insertions(+), 3 deletions(-)
changed files
M internal/importer/main_test.go → internal/importer/main_test.go
@@ -1,13 +1,13 @@ package importer import ( - "context" "testing" "alin.ovh/searchix/internal/config" "alin.ovh/searchix/internal/file" "alin.ovh/searchix/internal/index" "alin.ovh/searchix/internal/manpages" + "alin.ovh/searchix/internal/storage" "alin.ovh/x/log" )@@ -21,6 +21,7 @@ } logger := log.Configure(false) _, write, _, err := index.OpenOrCreate(&index.Options{ + Config: &cfg, Force: false, LowMemory: true, BatchSize: cfg.Importer.BatchSize,@@ -31,6 +32,14 @@ if err != nil { b.Fatal(err) } + store, err := storage.New(&storage.Options{ + Root: tmp, + Logger: logger.Named("storage"), + }) + if err != nil { + b.Fatal(err) + } + imp, err := New(&cfg, &Options{ Logger: logger.Named("importer"), LowMemory: true,@@ -47,13 +56,30 @@ if err != nil { b.Fatal(err) } + source := cfg.Importer.Sources["nixpkgs"] + source.Programs.Enable = false + err = imp.Start( - context.Background(), + b.Context(), false, false, - &[]string{"nixpkgs"}, + &[]string{source.Key}, ) if err != nil { b.Fatal(err) + } + + hadErrors, err := ImportSource( + b.Context(), + logger.Named("importer"), + store, + source, + write, + ) + if err != nil { + b.Fatal(err) + } + if hadErrors { + b.Fatal("had errors") } }