refactor: extract batch handling into struct
1 file changed, 33 insertions(+), 10 deletions(-)
changed files
M internal/importer/main.go → internal/importer/main.go
@@ -13,7 +13,6 @@ "alin.ovh/x/log" "github.com/Southclaws/fault" "github.com/Southclaws/fault/fmsg" - "github.com/blevesearch/bleve/v2" "alin.ovh/searchix/internal/config" "alin.ovh/searchix/internal/fetcher"@@ -226,7 +225,7 @@ if float64(res.Total) > (0.9 * float64(maxCount)) { return fault.Newf("too many entities to prune: %d/%d (threshold: 90%%)", res.Total, maxCount) } - err = write.WithBatch(func(batch *bleve.Batch) { + err = write.WithBatch(func(batch *index.Batcher) { for _, dm := range res.Hits { batch.Delete(dm.ID) }@@ -398,18 +397,42 @@ pdb, ) } - objects := processor.Process(ctx) + hadWarnings := false + write := imp.options.WriteIndex + err = write.WithAutoBatch(func(batch *index.Batcher) { + for object := range processor.Process(ctx) { + select { + case <-ctx.Done(): + imp.options.Logger.Warn("context canceled") - err = imp.options.WriteIndex.Import(ctx, objects) + return + default: + if err := processor.Err(); err != nil { + hadWarnings = true + + imp.options.Logger.Warn("error processing object", "error", err) + } + } + + doc, err := write.MapDocument(object) + if err != nil { + imp.options.Logger.Warn("error mapping document", "error", err) + + return + } + + err = batch.IndexAdvanced(doc) + if err != nil { + imp.options.Logger.Warn("error indexing document", "error", err) + + return + } + } + }) if err != nil { - return fault.Wrap(err, fmsg.With("error writing batch")) + return fault.Wrap(err, fmsg.With("error writing index")) } - hadWarnings := false - if err := processor.Err(); err != nil { - hadWarnings = true - imp.options.Logger.Warn("error processing objects", "error", err) - } imp.options.Logger.Debug("ingest completed") sourceMeta.StoredAt = time.Now()