all repos — searchix @ 2fcbebf4ce05c2c02b0c39cfa2236ef0ea0f9870

Search engine for NixOS, nix-darwin, home-manager and NUR users

refactor: extract index metadata code into package

Alan Pearce
commit

2fcbebf4ce05c2c02b0c39cfa2236ef0ea0f9870

parent

bfa2851e2a6d1800569ea5c451242bdf7f96dcac

9 files changed, 82 insertions(+), 72 deletions(-)

changed files
M internal/fetcher/channel.go → internal/fetcher/channel.go
@@ -13,7 +13,7 @@ "github.com/Southclaws/fault"
 	"github.com/Southclaws/fault/fmsg"
 
 	"alin.ovh/searchix/internal/config"
-	"alin.ovh/searchix/internal/index"
+	"alin.ovh/searchix/internal/index/meta"
 )
 
 type ChannelFetcher struct {

@@ -39,7 +39,7 @@ }
 
 func (i *ChannelFetcher) FetchIfNeeded(
 	ctx context.Context,
-	sourceMeta *index.SourceMeta,
+	sourceMeta *meta.SourceMeta,
 ) (*FetchedFiles, error) {
 	target := i.Source.JoinPath("options.json")
 	args := []string{
M internal/fetcher/download.go → internal/fetcher/download.go
@@ -9,7 +9,7 @@ "github.com/Southclaws/fault/fmsg"
 
 	"alin.ovh/searchix/internal/config"
 	"alin.ovh/searchix/internal/fetcher/http"
-	"alin.ovh/searchix/internal/index"
+	"alin.ovh/searchix/internal/index/meta"
 )
 
 type DownloadFetcher struct {

@@ -41,7 +41,7 @@ }
 
 func (i *DownloadFetcher) FetchIfNeeded(
 	ctx context.Context,
-	sourceMeta *index.SourceMeta,
+	sourceMeta *meta.SourceMeta,
 ) (*FetchedFiles, error) {
 	f := &FetchedFiles{}
M internal/fetcher/main.go → internal/fetcher/main.go
@@ -11,7 +11,7 @@ "github.com/Southclaws/fault/fmsg"
 
 	"alin.ovh/searchix/internal/config"
 	"alin.ovh/searchix/internal/file"
-	"alin.ovh/searchix/internal/index"
+	"alin.ovh/searchix/internal/index/meta"
 )
 
 type Options struct {

@@ -26,7 +26,7 @@ Packages io.ReadCloser
 }
 
 type Fetcher interface {
-	FetchIfNeeded(context.Context, *index.SourceMeta) (*FetchedFiles, error)
+	FetchIfNeeded(context.Context, *meta.SourceMeta) (*FetchedFiles, error)
 }
 
 func New(
M internal/fetcher/nixpkgs-channel.go → internal/fetcher/nixpkgs-channel.go
@@ -6,7 +6,7 @@ "net/url"
 
 	"alin.ovh/searchix/internal/config"
 	"alin.ovh/searchix/internal/fetcher/http"
-	"alin.ovh/searchix/internal/index"
+	"alin.ovh/searchix/internal/index/meta"
 
 	"github.com/Southclaws/fault"
 	"github.com/Southclaws/fault/fmsg"

@@ -46,7 +46,7 @@ )
 
 func (i *NixpkgsChannelFetcher) FetchIfNeeded(
 	ctx context.Context,
-	sourceMeta *index.SourceMeta,
+	sourceMeta *meta.SourceMeta,
 ) (f *FetchedFiles, err error) {
 	f = &FetchedFiles{}
M internal/importer/job.go → internal/importer/job.go
@@ -5,7 +5,7 @@ "context"
 	"time"
 
 	"alin.ovh/searchix/internal/config"
-	"alin.ovh/searchix/internal/index"
+	"alin.ovh/searchix/internal/index/meta"
 )
 
 var Job struct {

@@ -27,7 +27,7 @@ Job.LastRun.FinishedAt = time.Now()
 	Job.StartedAt = time.Time{}
 }
 
-func MarkLastRun(meta *index.Meta) {
+func MarkLastRun(meta *meta.Meta) {
 	meta.LastImport = Job.LastRun
 }
M internal/importer/main.go → internal/importer/main.go
@@ -18,6 +18,7 @@ "alin.ovh/searchix/internal/config"
 	"alin.ovh/searchix/internal/fetcher"
 	"alin.ovh/searchix/internal/file"
 	"alin.ovh/searchix/internal/index"
+	"alin.ovh/searchix/internal/index/meta"
 	"alin.ovh/searchix/internal/manpages"
 	"alin.ovh/searchix/internal/programs"
 )

@@ -238,7 +239,7 @@ }
 
 func (imp *Importer) createSourceFetcher(
 	parent context.Context,
-	meta *index.Meta,
+	meta *meta.Meta,
 	forceUpdate bool,
 ) func(config.Source) error {
 	return func(source config.Source) error {

@@ -321,7 +322,7 @@ }
 
 func (imp *Importer) createSourceImporter(
 	parent context.Context,
-	meta *index.Meta,
+	meta *meta.Meta,
 ) func(config.Source) error {
 	return func(source config.Source) error {
 		logger := imp.options.Logger.With("name", source.Key)
M internal/index/index_meta.go → internal/index/meta/meta.go
@@ -1,4 +1,4 @@
-package index
+package meta
 
 import (
 	"encoding/json"

@@ -11,7 +11,10 @@ "github.com/Southclaws/fault"
 	"github.com/Southclaws/fault/fmsg"
 )
 
-const CurrentSchemaVersion = 9
+const (
+	CurrentSchemaVersion = 9
+	BaseName             = "meta.json"
+)
 
 type SourceMeta struct {
 	IndexedAt time.Time

@@ -36,50 +39,6 @@ log  *log.Logger
 	data
 }
 
-func createMeta(root *file.Root, log *log.Logger) (*Meta, error) {
-	exists, err := root.Exists(metaBaseName)
-	if err != nil {
-		return nil, fault.Wrap(err, fmsg.With("could not check for existence of index metadata"))
-	}
-	if exists {
-		return nil, fault.New("index metadata already exists")
-	}
-
-	return &Meta{
-		root: root,
-		log:  log,
-		data: data{
-			SchemaVersion: CurrentSchemaVersion,
-			Sources:       make(map[string]*SourceMeta),
-		},
-	}, nil
-}
-
-func openMeta(root *file.Root, log *log.Logger) (*Meta, error) {
-	exists, err := root.Exists(metaBaseName)
-	if err != nil {
-		return nil, fault.Wrap(err, fmsg.With("could not check for existence of index metadata"))
-	}
-	if !exists {
-		return createMeta(root, log)
-	}
-
-	j, baseErr := root.ReadFile(metaBaseName)
-	if baseErr != nil {
-		return nil, fault.Wrap(baseErr, fmsg.With("could not open index metadata file"))
-	}
-	meta := Meta{
-		root: root,
-		log:  log,
-	}
-
-	if err := json.Unmarshal(j, &meta.data); err != nil {
-		return nil, fault.Wrap(err, fmsg.With("index metadata is corrupt, try replacing the index"))
-	}
-
-	return &meta, nil
-}
-
 func (i *Meta) IsSchemaOutdated() bool {
 	return i.SchemaVersion < CurrentSchemaVersion
 }

@@ -90,8 +49,8 @@ j, err := json.MarshalIndent(i.data, "", "  ")
 	if err != nil {
 		return fault.Wrap(err, fmsg.With("could not prepare index metadata for saving"))
 	}
-	i.log.Debug("saving index metadata", "path", metaBaseName)
-	err = i.root.WriteFile(metaBaseName, j, 0o600)
+	i.log.Debug("saving index metadata", "path", BaseName)
+	err = i.root.WriteFile(BaseName, j, 0o600)
 	if err != nil {
 		return fault.Wrap(err, fmsg.With("could not save index metadata"))
 	}

@@ -133,3 +92,47 @@ }
 
 	return last
 }
+
+func Create(root *file.Root, log *log.Logger) (*Meta, error) {
+	exists, err := root.Exists(BaseName)
+	if err != nil {
+		return nil, fault.Wrap(err, fmsg.With("could not check for existence of index metadata"))
+	}
+	if exists {
+		return nil, fault.New("index metadata already exists")
+	}
+
+	return &Meta{
+		root: root,
+		log:  log,
+		data: data{
+			SchemaVersion: CurrentSchemaVersion,
+			Sources:       make(map[string]*SourceMeta),
+		},
+	}, nil
+}
+
+func Open(root *file.Root, log *log.Logger) (*Meta, error) {
+	exists, err := root.Exists(BaseName)
+	if err != nil {
+		return nil, fault.Wrap(err, fmsg.With("could not check for existence of index metadata"))
+	}
+	if !exists {
+		return Create(root, log)
+	}
+
+	j, baseErr := root.ReadFile(BaseName)
+	if baseErr != nil {
+		return nil, fault.Wrap(baseErr, fmsg.With("could not open index metadata file"))
+	}
+	meta := Meta{
+		root: root,
+		log:  log,
+	}
+
+	if err := json.Unmarshal(j, &meta.data); err != nil {
+		return nil, fault.Wrap(err, fmsg.With("index metadata is corrupt, try replacing the index"))
+	}
+
+	return &meta, nil
+}
M internal/index/indexer.go → internal/index/indexer.go
@@ -8,6 +8,7 @@ "math"
 
 	"alin.ovh/searchix/internal/config"
 	"alin.ovh/searchix/internal/file"
+	"alin.ovh/searchix/internal/index/meta"
 	"alin.ovh/searchix/internal/index/nixattr"
 	"alin.ovh/searchix/internal/nix"
 

@@ -45,7 +46,7 @@ batchSize int
 	index     bleve.Index
 	log       *log.Logger
 	exists    bool
-	Meta      *Meta
+	Meta      *meta.Meta
 }
 
 type BatchError struct {

@@ -200,11 +201,10 @@ }
 
 const (
 	indexBaseName = "index.bleve"
-	metaBaseName  = "meta.json"
 )
 
 var expectedDataFiles = []string{
-	metaBaseName,
+	meta.BaseName,
 	indexBaseName,
 }
 

@@ -245,7 +245,7 @@ }
 	}
 
 	var idx bleve.Index
-	var meta *Meta
+	var m *meta.Meta
 	if !exists || options.Force {
 		if options.Force {
 			err = deleteIndex(root)

@@ -258,9 +258,9 @@ if err != nil {
 			return nil, nil, err
 		}
 
-		meta, err = createMeta(root, options.Logger)
+		m, err = meta.Create(root, options.Logger)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, fault.Wrap(err, fmsg.With("could not create index metadata file"))
 		}
 
 	} else {

@@ -272,9 +272,9 @@ if baseErr != nil {
 			return nil, nil, fault.Wrap(baseErr, fmsg.Withf("could not open index at path %s", indexPath))
 		}
 
-		meta, err = openMeta(root, options.Logger)
+		m, err = meta.Open(root, options.Logger)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, fault.Wrap(err, fmsg.With("could not open index metadata file"))
 		}
 	}
 

@@ -291,14 +291,14 @@ config: options.Config,
 			log:    options.Logger,
 			exists: exists,
 			index:  idx,
-			meta:   meta,
+			meta:   m,
 		},
 		&WriteIndex{
 			exists:    exists,
 			index:     idx,
 			batchSize: options.BatchSize,
 			log:       options.Logger,
-			Meta:      meta,
+			Meta:      m,
 		},
 		nil
 }

@@ -308,7 +308,12 @@ return i.exists
 }
 
 func (i *WriteIndex) SaveMeta() error {
-	return i.Meta.Save()
+	err := i.Meta.Save()
+	if err != nil {
+		return fault.Wrap(err)
+	}
+
+	return nil
 }
 
 func (i *WriteIndex) Import(
M internal/index/search.go → internal/index/search.go
@@ -12,6 +12,7 @@
 	"alin.ovh/x/log"
 
 	"alin.ovh/searchix/internal/config"
+	"alin.ovh/searchix/internal/index/meta"
 	"alin.ovh/searchix/internal/nix"
 
 	"github.com/Southclaws/fault"

@@ -39,7 +40,7 @@ index  bleve.Index
 	config *config.Config
 	log    *log.Logger
 	exists bool
-	meta   *Meta
+	meta   *meta.Meta
 }
 
 type SearchRequest struct {