perf: optimise import batching
2 files changed, 3 insertions(+), 2 deletions(-)
changed files
M internal/index/indexer.go → internal/index/indexer.go
@@ -226,6 +226,7 @@ err, fmsg.Withf("could not check if index exists at path %s", indexBaseName)) } kvconfig := map[string]any{ + "unsafe_batch": true, "scorchPersisterOptions": map[string]any{ "NumPersisterWorkers": 8, "MaxSizeInMemoryMergePerWorker": 128 * 1024 * 1024,
M internal/storage/store.go → internal/storage/store.go
@@ -17,7 +17,7 @@ "alin.ovh/searchix/internal/file" "alin.ovh/searchix/internal/nix" ) -var BatchSize = 10000 +var BatchSize = 50000 type Options struct { Replace bool@@ -65,7 +65,7 @@ return nil, fault.Wrap(err, fmsg.With("failed to open database")) } if !opts.LowMemory { - bb.Bolt.AllocSize = 100 * 1024 * 1024 + bb.Bolt.AllocSize = 256 * 1024 * 1024 } return &Store{