perf: tune storage backend
4 files changed, 21 insertions(+), 10 deletions(-)
M cmd/searchix-web/ingest.go → cmd/searchix-web/ingest.go
@@ -33,8 +33,9 @@ } defer root.Close() store, err := storage.New(&storage.Options{ - Root: root, - Logger: logger.Named("store"), + LowMemory: cfg.Importer.LowMemory, + Root: root, + Logger: logger.Named("store"), }) if err != nil { return fault.Wrap(err, fmsg.With("Failed to create store"))
M cmd/searchix-web/serve.go → cmd/searchix-web/serve.go
@@ -39,8 +39,9 @@ } defer root.Close() store, err := storage.New(&storage.Options{ - Root: root, - Logger: logger.Named("store"), + LowMemory: cfg.Importer.LowMemory, + Root: root, + Logger: logger.Named("store"), }) if err != nil { return fault.Wrap(err, fmsg.With("Failed to create store"))
M internal/importer/main_test.go → internal/importer/main_test.go
@@ -36,8 +36,9 @@ b.Fatal(err) } store, err := storage.New(&storage.Options{ - Root: tmp, - Logger: logger.Named("storage"), + LowMemory: true, + Root: tmp, + Logger: logger.Named("storage"), }) if err != nil { b.Fatal(err)
M internal/storage/store.go → internal/storage/store.go
@@ -20,9 +20,10 @@ var BatchSize = 10000 type Options struct { - Replace bool - Root *file.Root - Logger *log.Logger + Replace bool + LowMemory bool + Root *file.Root + Logger *log.Logger } type Store struct {@@ -53,11 +54,18 @@ path := opts.Root.JoinPath(filename) bb, err := storm.Open(path, storm.Codec(gob.Codec), storm.BoltOptions(0o600, &bbolt.Options{ - Timeout: 1 * time.Second, + FreelistType: bbolt.FreelistMapType, + NoFreelistSync: true, + NoGrowSync: true, + Timeout: 1 * time.Second, }), ) if err != nil { return nil, fault.Wrap(err, fmsg.With("failed to open database")) + } + + if !opts.LowMemory { + bb.Bolt.AllocSize = 100 * 1024 * 1024 } return &Store{