refactor: make importer responsible for spawning goroutines
1 file changed, 30 insertions(+), 33 deletions(-)
changed files
M internal/index/indexer.go → internal/index/indexer.go
@@ -318,10 +318,11 @@ func (i *WriteIndex) Import( ctx context.Context, objects <-chan nix.Importable, -) <-chan error { + errs chan<- error, +) { indexMapping := i.index.Mapping() - return i.WithBatchObjects(ctx, objects, func(batch *bleve.Batch, obj nix.Importable) error { + i.WithBatchObjects(ctx, objects, errs, func(batch *bleve.Batch, obj nix.Importable) error { doc := document.NewDocument(nix.GetKey(obj)) if err := indexMapping.MapDocument(doc, obj); err != nil { return fault.Wrap(err, fmsg.Withf("could not map document for object: %s", obj.GetName()))@@ -369,49 +370,45 @@ func (i *WriteIndex) WithBatchObjects( ctx context.Context, objects <-chan nix.Importable, + errs chan<- error, processor func(batch *bleve.Batch, obj nix.Importable) error, -) <-chan error { +) { var err error - errs := make(chan error) - go func() { - defer close(errs) - k := 0 - batch := i.index.NewBatch() + defer close(errs) + k := 0 + batch := i.index.NewBatch() - outer: - for obj := range objects { - select { - case <-ctx.Done(): - i.log.Warn("batch process aborted") +outer: + for obj := range objects { + select { + case <-ctx.Done(): + i.log.Warn("batch process aborted") - break outer - default: - } + break outer + default: + } - if err := processor(batch, obj); err != nil { - errs <- fault.Wrap(err, fmsg.With("could not process object")) + if err := processor(batch, obj); err != nil { + errs <- fault.Wrap(err, fmsg.With("could not process object")) - continue - } + continue + } - if k++; k%i.batchSize == 0 { - err = i.Flush(batch) - if err != nil { - errs <- err + if k++; k%i.batchSize == 0 { + err = i.Flush(batch) + if err != nil { + errs <- err - return - } + return } } - - err := i.Flush(batch) - if err != nil { - errs <- err - } - }() + } - return errs + err = i.Flush(batch) + if err != nil { + errs <- err + } } func (i *WriteIndex) Flush(batch *bleve.Batch) error {