all repos — searchix @ 4acc13e37376b6250623e5c259d4c91e9512dd52

Search engine for NixOS, nix-darwin, home-manager and NUR users

refactor: simplify import/process/index loop

Alan Pearce
commit

4acc13e37376b6250623e5c259d4c91e9512dd52

parent

1720799186c16120f3c8cb20af8077368e1c2f7c

1 file changed, 16 insertions(+), 34 deletions(-)

changed files
M internal/importer/importer.gointernal/importer/importer.go
@@ -4,7 +4,9 @@ import (
"context" "sync" - "alin.ovh/searchix/internal/index" + "github.com/Southclaws/fault" + "github.com/Southclaws/fault/fmsg" + "alin.ovh/searchix/internal/nix" )
@@ -19,47 +21,27 @@ ) (hadObjectErrors bool, criticalError error) {
wg := sync.WaitGroup{} objects := make(chan nix.Importable, 1) - pErrs := make(chan error) + errs := make(chan error) wg.Go(func() { - processor.Process(ctx, objects, pErrs) + processor.Process(ctx, objects, errs) + close(objects) }) - iErrs := make(chan error) wg.Go(func() { - imp.options.WriteIndex.Import(ctx, objects, iErrs) + err := imp.options.WriteIndex.Import(ctx, objects, errs) + if err != nil { + criticalError = fault.Wrap(err, fmsg.With("error writing batch")) + } + close(errs) }) - go func() { - for { - select { - case err, running := <-iErrs: - if !running { - iErrs = nil - imp.options.Logger.Debug("ingest completed") - - continue - } - be, isBatchError := err.(*index.BatchError) - if isBatchError { - criticalError = be - - break - } - hadObjectErrors = true - imp.options.Logger.Warn("error ingesting object", "error", err) - case err, running := <-pErrs: - if !running { - pErrs = nil - - continue - } - hadObjectErrors = true - imp.options.Logger.Warn("error processing object", "error", err) - } - } - }() + for err := range errs { + hadObjectErrors = true + imp.options.Logger.Warn("error processing object", "error", err) + } wg.Wait() + imp.options.Logger.Debug("ingest completed") return hadObjectErrors, criticalError }