all repos — searchix @ 8eb869f10a08e93f67a7feb50dc73600bc5304fb

Search engine for NixOS, nix-darwin, home-manager and NUR users

perf: flush index batch in groups of 10000

Alan Pearce
commit

8eb869f10a08e93f67a7feb50dc73600bc5304fb

parent

c5c9f726cae4cf4488312338da9599da53ac8b6c

1 file changed, 26 insertions(+), 5 deletions(-)

changed files
M internal/index/indexer.gointernal/index/indexer.go
@@ -31,6 +31,8 @@ index bleve.Index
meta *Meta } +const batchSize = 10_000 + func createIndexMapping() (mapping.IndexMapping, error) { indexMapping := bleve.NewIndexMapping() indexMapping.StoreDynamic = false
@@ -233,6 +235,7 @@ errs := make(chan error)
go func() { defer close(errs) + k := 0 batch := i.index.NewBatch() indexMapping := i.index.Mapping()
@@ -273,18 +276,36 @@ errs <- errors.WithMessagef(err, "could not index object %s", obj.GetName())
continue } - } - size := batch.Size() - slog.Debug("flushing batch", "size", size) + if k++; k%batchSize == 0 { + err = i.Flush(batch) + if err != nil { + errs <- err + } + } + } - err := i.index.Batch(batch) + err := i.Flush(batch) if err != nil { - errs <- errors.WithMessagef(err, "could not flush batch") + errs <- err } }() return errs +} + +func (i *WriteIndex) Flush(batch *bleve.Batch) error { + size := batch.Size() + slog.Debug("flushing batch", "size", size) + + err := i.index.Batch(batch) + if err != nil { + return errors.WithMessagef(err, "could not flush batch") + } + + batch.Reset() + + return nil } func (i *WriteIndex) Close() error {