fix: always re-index if storage is (re-)created
7 files changed, 46 insertions(+), 19 deletions(-)
M cmd/searchix-web/ingest.go → cmd/searchix-web/ingest.go
@@ -41,7 +41,7 @@ return fault.Wrap(err, fmsg.With("Failed to create store")) } defer store.Close() - _, write, exists, err := index.OpenOrCreate( + _, write, err := index.OpenOrCreate( &index.Options{ Config: cfg, Force: opts.Reindex,@@ -74,7 +74,8 @@ if err != nil { return fault.Wrap(err, fmsg.With("Failed to create importer")) } - if !exists || opts.Replace || opts.Fetch { + updateStore := store.IsNew() || opts.Replace || opts.Fetch + if updateStore { importer.MarkImportStarted() err = imp.Fetch(ctx, true, opts.Fetch && !opts.Replace, nil)@@ -90,7 +91,7 @@ return fault.Wrap(err, fmsg.With("Failed to save index metadata")) } } - if !exists || opts.Reindex { + if !write.Exists() || updateStore { err = imp.Index(ctx) if err != nil { return fault.Wrap(err, fmsg.With("Failed to index data"))
M cmd/searchix-web/serve.go → cmd/searchix-web/serve.go
@@ -47,7 +47,7 @@ return fault.Wrap(err, fmsg.With("Failed to create store")) } defer store.Close() - read, write, exists, err := index.OpenOrCreate( + read, write, err := index.OpenOrCreate( &index.Options{ Config: cfg, LowMemory: cfg.Importer.LowMemory,@@ -87,12 +87,14 @@ if err != nil { return fault.Wrap(err, fmsg.With("Failed to create importer")) } - if !exists { + if store.IsNew() { err = imp.Fetch(ctx, true, false, nil) if err != nil { return fault.Wrap(err, fmsg.With("Failed to start importer")) } + } + if store.IsNew() || !write.Exists() { err = imp.Index(ctx) if err != nil { return fault.Wrap(err, fmsg.With("Failed to index data"))
M internal/importer/main_test.go → internal/importer/main_test.go
@@ -23,7 +23,7 @@ b.Fatal(err) } logger := log.Configure(false) - _, write, _, err := index.OpenOrCreate(&index.Options{ + _, write, err := index.OpenOrCreate(&index.Options{ Config: &cfg, Force: false, LowMemory: true,
M internal/index/indexer.go → internal/index/indexer.go
@@ -42,6 +42,7 @@ batchSize int index bleve.Index log *log.Logger store *storage.Store + exists bool Meta *Meta }@@ -220,14 +221,14 @@ return nil } -func OpenOrCreate(options *Options) (*ReadIndex, *WriteIndex, bool, error) { +func OpenOrCreate(options *Options) (*ReadIndex, *WriteIndex, error) { var err error bleve.SetLog(zap.NewStdLog(options.Logger.Named("bleve").GetLogger())) root := options.Root exists, err := root.Exists(indexBaseName) if err != nil { - return nil, nil, exists, fault.Wrap( + return nil, nil, fault.Wrap( err, fmsg.Withf("could not check if index exists at path %s", indexBaseName)) }@@ -237,17 +238,17 @@ if !exists || options.Force { if options.Force { err = deleteIndex(root) if err != nil { - return nil, nil, false, err + return nil, nil, err } } idx, err = createIndex(root, options) if err != nil { - return nil, nil, false, err + return nil, nil, err } meta, err = createMeta(root, options.Logger) if err != nil { - return nil, nil, false, err + return nil, nil, err } } else {@@ -256,12 +257,12 @@ //nolint:forbidigo // external package indexPath := root.JoinPath(indexBaseName) idx, baseErr = bleve.Open(indexPath) if baseErr != nil { - return nil, nil, exists, fault.Wrap(baseErr, fmsg.Withf("could not open index at path %s", indexPath)) + return nil, nil, fault.Wrap(baseErr, fmsg.Withf("could not open index at path %s", indexPath)) } meta, err = openMeta(root, options.Logger) if err != nil { - return nil, nil, exists, err + return nil, nil, err } }@@ -277,18 +278,23 @@ return &ReadIndex{ config: options.Config, log: options.Logger, store: options.Store, + exists: exists, index: idx, meta: meta, }, &WriteIndex{ + exists: exists, index: idx, batchSize: options.BatchSize, log: options.Logger, store: options.Store, Meta: meta, }, - exists, nil +} + +func (i *WriteIndex) Exists() bool { + return i.exists } func (i *WriteIndex) SaveMeta() error {
M internal/index/search.go → internal/index/search.go
@@ -36,7 +36,12 @@ index bleve.Index config *config.Config store *storage.Store log *log.Logger + exists bool meta *Meta +} + +func (index *ReadIndex) Exists() bool { + return index.exists } func (index *ReadIndex) LastUpdated() time.Time {
M internal/index/search_test.go → internal/index/search_test.go
@@ -38,7 +38,7 @@ t.Fatal(err) } defer store.Close() - read, _, exists, err := index.OpenOrCreate(&index.Options{ + read, _, err := index.OpenOrCreate(&index.Options{ Logger: log.Named("index"), Root: root, BatchSize: cfg.Importer.BatchSize,@@ -51,7 +51,7 @@ if err != nil { t.Fatal(err) } defer read.Close() - if !exists { + if !read.Exists() { t.Fatal("expected index to exist") }@@ -125,7 +125,7 @@ t.Fatal(err) } defer store.Close() - read, _, exists, err := index.OpenOrCreate(&index.Options{ + read, _, err := index.OpenOrCreate(&index.Options{ Force: false, LowMemory: false, Logger: log.Named("index"),@@ -138,7 +138,7 @@ if err != nil { t.Fatal(err) } defer read.Close() - if !exists { + if !read.Exists() { t.Fatal("expected index to exist") }
M internal/storage/store.go → internal/storage/store.go
@@ -26,12 +26,20 @@ } type Store struct { *storm.DB + new bool log *log.Logger } +const filename = "searchix.bolt" + func New(opts *Options) (*Store, error) { + exists, err := opts.Root.Exists(filename) + if err != nil { + return nil, fault.Wrap(err, fmsg.With("failed to check if file exists")) + } + //nolint:forbidigo // external package - path := opts.Root.JoinPath("searchix.bolt") + path := opts.Root.JoinPath(filename) bb, err := storm.Open(path, storm.Codec(gob.Codec), storm.BoltOptions(0o600, &bbolt.Options{@@ -44,8 +52,13 @@ } return &Store{ DB: bb, + new: !exists, log: opts.Logger, }, nil +} + +func (s *Store) IsNew() bool { + return s.new } func (s *Store) Close() error {