all repos — searchix @ 0dc3ac14c8cd367a415bc14f5e0dd682420e5480

Search engine for NixOS, nix-darwin, home-manager and NUR users

refactor: deduplicate indexing code

Alan Pearce
commit

0dc3ac14c8cd367a415bc14f5e0dd682420e5480

parent

d0795bbe26047bd8c270d51dec4a66edc69c2a6a

M cmd/searchix-web/ingest.gocmd/searchix-web/ingest.go
@@ -82,15 +82,9 @@ }
} if !exists || opts.Reindex { - for _, source := range cfg.Importer.Sources { - hadErrors, err := imp.IndexSource(ctx, source) - if err != nil { - return fault.Wrap(err, fmsg.Withf("Failed to import source %s", source.Name)) - } - - if hadErrors { - logger.Warn("Imported source encountered errors", "source", source.Name) - } + err = imp.Index(ctx) + if err != nil { + return fault.Wrap(err, fmsg.With("Failed to index data")) } }
M cmd/searchix-web/serve.gocmd/searchix-web/serve.go
@@ -86,15 +86,9 @@ if err != nil {
return fault.Wrap(err, fmsg.With("Failed to start importer")) } - for _, source := range cfg.Importer.Sources { - hadErrors, err := imp.IndexSource(ctx, source) - if err != nil { - return fault.Wrap(err, fmsg.Withf("Failed to import source %s", source.Name)) - } - - if hadErrors { - logger.Warn("Imported source encountered errors", "source", source.Name) - } + err = imp.Index(ctx) + if err != nil { + return fault.Wrap(err, fmsg.With("Failed to index data")) } }
M internal/importer/importer.gointernal/importer/importer.go
@@ -20,7 +20,7 @@ ImportSource func(context.Context) (<-chan nix.Importable, <-chan error)
ImportDestination func(context.Context, <-chan nix.Importable) <-chan error ) -func Import( +func pipe( ctx context.Context, log *log.Logger, src ImportSource,
@@ -67,11 +67,11 @@
return hadObjectErrors, criticalError } -func (imp *Importer) IndexSource( +func (imp *Importer) indexSource( ctx context.Context, source *config.Source, ) (bool, error) { - return Import( + return pipe( ctx, imp.options.Logger, storage.MakeSourceExporter(imp.options.Storage, source),
M internal/importer/main.gointernal/importer/main.go
@@ -190,7 +190,7 @@ if err != nil {
return fault.Wrap(err, fmsg.Withf("failed to create processor")) } - hadWarnings, err := Import( + hadWarnings, err := pipe( ctx, logger.Named("importer"), processor.Process,
@@ -261,6 +261,21 @@ MarkLastFetch(meta)
err := imp.options.WriteIndex.SaveMeta() if err != nil { return fault.Wrap(err, fmsg.With("failed to save metadata")) + } + + return nil +} + +func (imp *Importer) Index(ctx context.Context) error { + for name, source := range imp.config.Importer.Sources { + hadErrors, err := imp.indexSource(ctx, source) + if err != nil { + return fault.Wrap(err, fmsg.Withf("Failed to import source %s", name)) + } + + if hadErrors { + imp.options.Logger.Warn("Imported source encountered errors", "source", source.Name) + } } return nil
M internal/importer/main_test.gointernal/importer/main_test.go
@@ -72,7 +72,7 @@ if err != nil {
b.Fatal(err) } - hadErrors, err := imp.IndexSource(b.Context(), source) + hadErrors, err := imp.indexSource(b.Context(), source) if err != nil { b.Fatal(err) }