all repos — searchix @ 63599c719d8e4e09548546bb366a849ef40c602b

Search engine for NixOS, nix-darwin, home-manager and NUR users

feat: add low-memory mode

Alan Pearce
commit

63599c719d8e4e09548546bb366a849ef40c602b

parent

789d0079f84017ddafad763ffba9aadbcd6aa6da

1 file changed, 24 insertions(+), 8 deletions(-)

changed files
M internal/index/indexer.gointernal/index/indexer.go
@@ -40,7 +40,7 @@ func (e *BatchError) Error() string {
return e.error.Error() } -const batchSize = 10_000 +var batchSize = 10_000 func createIndexMapping() (mapping.IndexMapping, error) { indexMapping := bleve.NewIndexMapping()
@@ -126,19 +126,24 @@
return indexMapping, nil } -func createIndex(indexPath string) (bleve.Index, error) { +func createIndex(indexPath string, options *Options) (bleve.Index, error) { indexMapping, err := createIndexMapping() if err != nil { return nil, err } + kvconfig := make(map[string]interface{}) + if options.LowMemory { + kvconfig = map[string]interface{}{ + "PersisterNapTimeMSec": 1000, + "PersisterNapUnderNumFiles": 500, + } + } idx, err := bleve.NewUsing( indexPath, indexMapping, bleve.Config.DefaultIndexType, bleve.Config.DefaultKVStore, - map[string]interface{}{ - "nosync": true, - }, + kvconfig, ) if err != nil { return nil, errors.WithMessagef(err, "unable to create index at path %s", indexPath)
@@ -182,7 +187,15 @@
return nil } -func OpenOrCreate(dataRoot string, force bool) (*ReadIndex, *WriteIndex, bool, error) { +type Options struct { + LowMemory bool +} + +func OpenOrCreate( + dataRoot string, + force bool, + options *Options, +) (*ReadIndex, *WriteIndex, bool, error) { var err error bleve.SetLog(log.Default())
@@ -207,7 +220,7 @@ if err != nil {
return nil, nil, false, err } } - idx, err = createIndex(indexPath) + idx, err = createIndex(indexPath, options) if err != nil { return nil, nil, false, err }
@@ -227,7 +240,10 @@ meta, err = openMeta(metaPath)
if err != nil { return nil, nil, exists, err } + } + if options.LowMemory { + batchSize = 1_000 } return &ReadIndex{
@@ -362,7 +378,7 @@ return errors.WithMessagef(err, "failed to query documents of retired index %s", source)
} batch := i.index.NewBatch() - var k uint + var k int for _, hit := range results.Hits { batch.Delete(hit.ID) if k++; k%batchSize == 0 {