all repos — searchix @ 12a29a55fe895a964d8d317857ee7fb87d4071f1

Search engine for NixOS, nix-darwin, home-manager and NUR users

fix: always re-index if storage is (re-)created

Alan Pearce
commit

12a29a55fe895a964d8d317857ee7fb87d4071f1

parent

15a5b3210a073c83dbd1ace8969df83c53dac755

M cmd/searchix-web/ingest.gocmd/searchix-web/ingest.go
@@ -41,7 +41,7 @@ return fault.Wrap(err, fmsg.With("Failed to create store"))
} defer store.Close() - _, write, exists, err := index.OpenOrCreate( + _, write, err := index.OpenOrCreate( &index.Options{ Config: cfg, Force: opts.Reindex,
@@ -74,7 +74,8 @@ if err != nil {
return fault.Wrap(err, fmsg.With("Failed to create importer")) } - if !exists || opts.Replace || opts.Fetch { + updateStore := store.IsNew() || opts.Replace || opts.Fetch + if updateStore { importer.MarkImportStarted() err = imp.Fetch(ctx, true, opts.Fetch && !opts.Replace, nil)
@@ -90,7 +91,7 @@ return fault.Wrap(err, fmsg.With("Failed to save index metadata"))
} } - if !exists || opts.Reindex { + if !write.Exists() || updateStore { err = imp.Index(ctx) if err != nil { return fault.Wrap(err, fmsg.With("Failed to index data"))
M cmd/searchix-web/serve.gocmd/searchix-web/serve.go
@@ -47,7 +47,7 @@ return fault.Wrap(err, fmsg.With("Failed to create store"))
} defer store.Close() - read, write, exists, err := index.OpenOrCreate( + read, write, err := index.OpenOrCreate( &index.Options{ Config: cfg, LowMemory: cfg.Importer.LowMemory,
@@ -87,12 +87,14 @@ if err != nil {
return fault.Wrap(err, fmsg.With("Failed to create importer")) } - if !exists { + if store.IsNew() { err = imp.Fetch(ctx, true, false, nil) if err != nil { return fault.Wrap(err, fmsg.With("Failed to start importer")) } + } + if store.IsNew() || !write.Exists() { err = imp.Index(ctx) if err != nil { return fault.Wrap(err, fmsg.With("Failed to index data"))
M internal/importer/main_test.gointernal/importer/main_test.go
@@ -23,7 +23,7 @@ b.Fatal(err)
} logger := log.Configure(false) - _, write, _, err := index.OpenOrCreate(&index.Options{ + _, write, err := index.OpenOrCreate(&index.Options{ Config: &cfg, Force: false, LowMemory: true,
M internal/index/indexer.gointernal/index/indexer.go
@@ -42,6 +42,7 @@ batchSize int
index bleve.Index log *log.Logger store *storage.Store + exists bool Meta *Meta }
@@ -220,14 +221,14 @@
return nil } -func OpenOrCreate(options *Options) (*ReadIndex, *WriteIndex, bool, error) { +func OpenOrCreate(options *Options) (*ReadIndex, *WriteIndex, error) { var err error bleve.SetLog(zap.NewStdLog(options.Logger.Named("bleve").GetLogger())) root := options.Root exists, err := root.Exists(indexBaseName) if err != nil { - return nil, nil, exists, fault.Wrap( + return nil, nil, fault.Wrap( err, fmsg.Withf("could not check if index exists at path %s", indexBaseName)) }
@@ -237,17 +238,17 @@ if !exists || options.Force {
if options.Force { err = deleteIndex(root) if err != nil { - return nil, nil, false, err + return nil, nil, err } } idx, err = createIndex(root, options) if err != nil { - return nil, nil, false, err + return nil, nil, err } meta, err = createMeta(root, options.Logger) if err != nil { - return nil, nil, false, err + return nil, nil, err } } else {
@@ -256,12 +257,12 @@ //nolint:forbidigo // external package
indexPath := root.JoinPath(indexBaseName) idx, baseErr = bleve.Open(indexPath) if baseErr != nil { - return nil, nil, exists, fault.Wrap(baseErr, fmsg.Withf("could not open index at path %s", indexPath)) + return nil, nil, fault.Wrap(baseErr, fmsg.Withf("could not open index at path %s", indexPath)) } meta, err = openMeta(root, options.Logger) if err != nil { - return nil, nil, exists, err + return nil, nil, err } }
@@ -277,18 +278,23 @@ return &ReadIndex{
config: options.Config, log: options.Logger, store: options.Store, + exists: exists, index: idx, meta: meta, }, &WriteIndex{ + exists: exists, index: idx, batchSize: options.BatchSize, log: options.Logger, store: options.Store, Meta: meta, }, - exists, nil +} + +func (i *WriteIndex) Exists() bool { + return i.exists } func (i *WriteIndex) SaveMeta() error {
M internal/index/search.gointernal/index/search.go
@@ -36,7 +36,12 @@ index bleve.Index
config *config.Config store *storage.Store log *log.Logger + exists bool meta *Meta +} + +func (index *ReadIndex) Exists() bool { + return index.exists } func (index *ReadIndex) LastUpdated() time.Time {
M internal/index/search_test.gointernal/index/search_test.go
@@ -38,7 +38,7 @@ t.Fatal(err)
} defer store.Close() - read, _, exists, err := index.OpenOrCreate(&index.Options{ + read, _, err := index.OpenOrCreate(&index.Options{ Logger: log.Named("index"), Root: root, BatchSize: cfg.Importer.BatchSize,
@@ -51,7 +51,7 @@ if err != nil {
t.Fatal(err) } defer read.Close() - if !exists { + if !read.Exists() { t.Fatal("expected index to exist") }
@@ -125,7 +125,7 @@ t.Fatal(err)
} defer store.Close() - read, _, exists, err := index.OpenOrCreate(&index.Options{ + read, _, err := index.OpenOrCreate(&index.Options{ Force: false, LowMemory: false, Logger: log.Named("index"),
@@ -138,7 +138,7 @@ if err != nil {
t.Fatal(err) } defer read.Close() - if !exists { + if !read.Exists() { t.Fatal("expected index to exist") }
M internal/storage/store.gointernal/storage/store.go
@@ -26,12 +26,20 @@ }
type Store struct { *storm.DB + new bool log *log.Logger } +const filename = "searchix.bolt" + func New(opts *Options) (*Store, error) { + exists, err := opts.Root.Exists(filename) + if err != nil { + return nil, fault.Wrap(err, fmsg.With("failed to check if file exists")) + } + //nolint:forbidigo // external package - path := opts.Root.JoinPath("searchix.bolt") + path := opts.Root.JoinPath(filename) bb, err := storm.Open(path, storm.Codec(gob.Codec), storm.BoltOptions(0o600, &bbolt.Options{
@@ -44,8 +52,13 @@ }
return &Store{ DB: bb, + new: !exists, log: opts.Logger, }, nil +} + +func (s *Store) IsNew() bool { + return s.new } func (s *Store) Close() error {