all repos — searchix @ 16f488de7dc8e0bb34ffadad167b443b803c8b73

Search engine for NixOS, nix-darwin, home-manager and NUR users

refactor: extract batch handling into struct

Alan Pearce
commit

16f488de7dc8e0bb34ffadad167b443b803c8b73

parent

e055ed6177bdec5fb286724a8e6b6e8ad715cca1

1 file changed, 33 insertions(+), 10 deletions(-)

changed files
M internal/importer/main.gointernal/importer/main.go
@@ -13,7 +13,6 @@
"alin.ovh/x/log" "github.com/Southclaws/fault" "github.com/Southclaws/fault/fmsg" - "github.com/blevesearch/bleve/v2" "alin.ovh/searchix/internal/config" "alin.ovh/searchix/internal/fetcher"
@@ -226,7 +225,7 @@ if float64(res.Total) > (0.9 * float64(maxCount)) {
return fault.Newf("too many entities to prune: %d/%d (threshold: 90%%)", res.Total, maxCount) } - err = write.WithBatch(func(batch *bleve.Batch) { + err = write.WithBatch(func(batch *index.Batcher) { for _, dm := range res.Hits { batch.Delete(dm.ID) }
@@ -398,18 +397,42 @@ pdb,
) } - objects := processor.Process(ctx) + hadWarnings := false + write := imp.options.WriteIndex + err = write.WithAutoBatch(func(batch *index.Batcher) { + for object := range processor.Process(ctx) { + select { + case <-ctx.Done(): + imp.options.Logger.Warn("context canceled") - err = imp.options.WriteIndex.Import(ctx, objects) + return + default: + if err := processor.Err(); err != nil { + hadWarnings = true + + imp.options.Logger.Warn("error processing object", "error", err) + } + } + + doc, err := write.MapDocument(object) + if err != nil { + imp.options.Logger.Warn("error mapping document", "error", err) + + return + } + + err = batch.IndexAdvanced(doc) + if err != nil { + imp.options.Logger.Warn("error indexing document", "error", err) + + return + } + } + }) if err != nil { - return fault.Wrap(err, fmsg.With("error writing batch")) + return fault.Wrap(err, fmsg.With("error writing index")) } - hadWarnings := false - if err := processor.Err(); err != nil { - hadWarnings = true - imp.options.Logger.Warn("error processing objects", "error", err) - } imp.options.Logger.Debug("ingest completed") sourceMeta.StoredAt = time.Now()