all repos — searchix @ 843de71ca2678f20f71a0fe76082611f5f748326

Search engine for NixOS, nix-darwin, home-manager and NUR users

perf: tune storage backend

Alan Pearce
commit

843de71ca2678f20f71a0fe76082611f5f748326

parent

abef2e5b0db37c771464469c3f83890b4cc0a297

M cmd/searchix-web/ingest.gocmd/searchix-web/ingest.go
@@ -33,8 +33,9 @@ }
defer root.Close() store, err := storage.New(&storage.Options{ - Root: root, - Logger: logger.Named("store"), + LowMemory: cfg.Importer.LowMemory, + Root: root, + Logger: logger.Named("store"), }) if err != nil { return fault.Wrap(err, fmsg.With("Failed to create store"))
M cmd/searchix-web/serve.gocmd/searchix-web/serve.go
@@ -39,8 +39,9 @@ }
defer root.Close() store, err := storage.New(&storage.Options{ - Root: root, - Logger: logger.Named("store"), + LowMemory: cfg.Importer.LowMemory, + Root: root, + Logger: logger.Named("store"), }) if err != nil { return fault.Wrap(err, fmsg.With("Failed to create store"))
M internal/importer/main_test.gointernal/importer/main_test.go
@@ -36,8 +36,9 @@ b.Fatal(err)
} store, err := storage.New(&storage.Options{ - Root: tmp, - Logger: logger.Named("storage"), + LowMemory: true, + Root: tmp, + Logger: logger.Named("storage"), }) if err != nil { b.Fatal(err)
M internal/storage/store.gointernal/storage/store.go
@@ -20,9 +20,10 @@
var BatchSize = 10000 type Options struct { - Replace bool - Root *file.Root - Logger *log.Logger + Replace bool + LowMemory bool + Root *file.Root + Logger *log.Logger } type Store struct {
@@ -53,11 +54,18 @@ path := opts.Root.JoinPath(filename)
bb, err := storm.Open(path, storm.Codec(gob.Codec), storm.BoltOptions(0o600, &bbolt.Options{ - Timeout: 1 * time.Second, + FreelistType: bbolt.FreelistMapType, + NoFreelistSync: true, + NoGrowSync: true, + Timeout: 1 * time.Second, }), ) if err != nil { return nil, fault.Wrap(err, fmsg.With("failed to open database")) + } + + if !opts.LowMemory { + bb.Bolt.AllocSize = 100 * 1024 * 1024 } return &Store{