refactor: move interfaces closer to usage
9 files changed, 65 insertions(+), 89 deletions(-)
M internal/components/combined.go → internal/components/combined.go
@@ -9,7 +9,7 @@ g "alin.ovh/gomponents" . "alin.ovh/gomponents/html" ) -func CombinedData(data nix.Importable) g.Node { +func CombinedData(data index.Indexable) g.Node { switch data := data.(type) { case nix.Option: return firstSentence(data.Description)
M internal/components/detail.go → internal/components/detail.go
@@ -1,12 +1,13 @@ package components import ( + "alin.ovh/searchix/internal/index" "alin.ovh/searchix/internal/nix" g "alin.ovh/gomponents" ) -func Detail(thing nix.Importable) g.Node { +func Detail(thing index.Indexable) g.Node { switch t := thing.(type) { case nix.Option: return OptionDetail(t)@@ -17,6 +18,6 @@ return nil } } -func DetailPage(tdata TemplateData, thing nix.Importable) g.Node { +func DetailPage(tdata TemplateData, thing index.Indexable) g.Node { return Page(tdata, Detail(thing)) }
D internal/importer/importer.go
@@ -1,47 +0,0 @@ -package importer - -import ( - "context" - "sync" - - "github.com/Southclaws/fault" - "github.com/Southclaws/fault/fmsg" - - "alin.ovh/searchix/internal/nix" -) - -type Processor interface { - Process(context.Context, chan<- nix.Importable, chan<- error) -} - -func (imp *Importer) process( - ctx context.Context, - processor Processor, -) (hadObjectErrors bool, criticalError error) { - wg := sync.WaitGroup{} - - objects := make(chan nix.Importable, 1) - errs := make(chan error) - wg.Go(func() { - processor.Process(ctx, objects, errs) - close(objects) - }) - - wg.Go(func() { - err := imp.options.WriteIndex.Import(ctx, objects, errs) - if err != nil { - criticalError = fault.Wrap(err, fmsg.With("error writing batch")) - } - close(errs) - }) - - for err := range errs { - hadObjectErrors = true - imp.options.Logger.Warn("error processing object", "error", err) - } - - wg.Wait() - imp.options.Logger.Debug("ingest completed") - - return hadObjectErrors, criticalError -}
M internal/importer/main.go → internal/importer/main.go
@@ -7,6 +7,7 @@ "maps" "os/exec" "slices" "strings" + "sync" "time" "alin.ovh/x/log"@@ -22,6 +23,10 @@ "alin.ovh/searchix/internal/index/meta" "alin.ovh/searchix/internal/manpages" "alin.ovh/searchix/internal/programs" ) + +type Processor interface { + Process(context.Context, chan<- index.Indexable, chan<- error) +} type Options struct { LowMemory bool@@ -378,25 +383,45 @@ source.Repo.Revision, ) switch source.Importer { case config.Options: - processor, err = NewOptionProcessor( + processor = NewOptionProcessor( files.Options, source, logger.Named("processor"), ) case config.Packages: - processor, err = NewPackageProcessor( + processor = NewPackageProcessor( files.Packages, source, logger.Named("processor"), pdb, ) } - if err != nil { - return fault.Wrap(err, fmsg.Withf("failed to create processor")) - } - hadWarnings, err := imp.process(ctx, processor) - if err != nil { + var ( + hadWarnings bool + criticalError error + ) + wg := sync.WaitGroup{} + objects := make(chan index.Indexable, 1) + errs := make(chan error) + wg.Go(func() { + processor.Process(ctx, objects, errs) + close(objects) + }) + wg.Go(func() { + err := imp.options.WriteIndex.Import(ctx, objects, errs) + if err != nil { + criticalError = fault.Wrap(err, fmsg.With("error writing batch")) + } + close(errs) + }) + for err := range errs { + hadWarnings = true + imp.options.Logger.Warn("error processing object", "error", err) + } + wg.Wait() + imp.options.Logger.Debug("ingest completed") + if criticalError != nil { return fault.Wrap(err, fmsg.Withf("failed to process source")) }
M internal/importer/options.go → internal/importer/options.go
@@ -10,6 +10,7 @@ "alin.ovh/x/log" "alin.ovh/searchix/internal/config" + "alin.ovh/searchix/internal/index" "alin.ovh/searchix/internal/nix" "github.com/Southclaws/fault"@@ -72,8 +73,8 @@ func NewOptionProcessor( infile io.ReadCloser, source config.Source, log *log.Logger, -) (*OptionIngester, error) { - i := OptionIngester{ +) *OptionIngester { + i := &OptionIngester{ dec: jstream.NewDecoder(infile, source.JSONDepth).EmitKV(), log: log, optJSON: nixOptionJSON{},@@ -89,18 +90,16 @@ Squash: true, DecodeHook: mapstructure.TextUnmarshallerHookFunc(), }) if err != nil { - defer infile.Close() - - return nil, fault.Wrap(err, fmsg.With("could not create mapstructure decoder")) + panic("could not create mapstructure decoder: " + err.Error()) } i.ms = ms - return &i, nil + return i } func (i *OptionIngester) Process( ctx context.Context, - results chan<- nix.Importable, + results chan<- index.Indexable, errs chan<- error, ) { defer i.infile.Close()
M internal/importer/package.go → internal/importer/package.go
@@ -12,6 +12,7 @@ "alin.ovh/x/log" "alin.ovh/searchix/internal/config" + "alin.ovh/searchix/internal/index" "alin.ovh/searchix/internal/nix" "alin.ovh/searchix/internal/programs"@@ -73,7 +74,7 @@ infile io.ReadCloser, source config.Source, log *log.Logger, programsDB *programs.DB, -) (*PackageIngester, error) { +) *PackageIngester { i := &PackageIngester{ dec: jstream.NewDecoder(infile, source.JSONDepth).EmitKV(), log: log,@@ -90,13 +91,11 @@ Squash: true, DecodeHook: mapstructure.TextUnmarshallerHookFunc(), }) if err != nil { - defer infile.Close() - - return nil, fault.Wrap(err, fmsg.With("could not create mapstructure decoder")) + panic("could not create mapstructure decoder: " + err.Error()) } i.ms = ms - return i, nil + return i } func convertToLicense(in map[string]any) *nix.License {@@ -122,7 +121,7 @@ } func (i *PackageIngester) Process( ctx context.Context, - results chan<- nix.Importable, + results chan<- index.Indexable, errs chan<- error, ) { if i.programs != nil {
M internal/index/indexer.go → internal/index/indexer.go
@@ -10,7 +10,6 @@ "alin.ovh/searchix/internal/config" "alin.ovh/searchix/internal/file" "alin.ovh/searchix/internal/index/meta" "alin.ovh/searchix/internal/index/nixattr" - "alin.ovh/searchix/internal/nix" "alin.ovh/x/log" "github.com/Southclaws/fault"@@ -50,6 +49,17 @@ } type BatchError struct { error +} + +type Indexable interface { + ImporterType() string + BleveType() string + GetName() string + GetSource() string +} + +func GetKey(i Indexable) string { + return i.ImporterType() + "/" + i.GetSource() + "/" + i.GetName() } var idAnalyzer analysis.Analyzer@@ -317,13 +327,13 @@ } func (i *WriteIndex) Import( ctx context.Context, - objects <-chan nix.Importable, + objects <-chan Indexable, errs chan<- error, ) error { indexMapping := i.index.Mapping() - return i.WithBatchObjects(ctx, objects, errs, func(batch *bleve.Batch, obj nix.Importable) error { - doc := document.NewDocument(nix.GetKey(obj)) + return i.WithBatchObjects(ctx, objects, errs, func(batch *bleve.Batch, obj Indexable) error { + doc := document.NewDocument(GetKey(obj)) if err := indexMapping.MapDocument(doc, obj); err != nil { return fault.Wrap(err, fmsg.Withf("could not map document for object: %s", obj.GetName())) }@@ -369,9 +379,9 @@ } func (i *WriteIndex) WithBatchObjects( ctx context.Context, - objects <-chan nix.Importable, + objects <-chan Indexable, errs chan<- error, - processor func(batch *bleve.Batch, obj nix.Importable) error, + processor func(batch *bleve.Batch, obj Indexable) error, ) error { k := 0 batch := i.index.NewBatch()
M internal/index/search.go → internal/index/search.go
@@ -27,7 +27,7 @@ const DefaultPageSize = 100 type DocumentMatch struct { *search.DocumentMatch - Data nix.Importable + Data Indexable } type Result struct {@@ -329,7 +329,7 @@ func (index *ReadIndex) GetDocument( ctx context.Context, source config.Source, id string, -) (nix.Importable, error) { +) (Indexable, error) { key := nix.MakeKey(source, id) query := bleve.NewDocIDQuery([]string{key}) search := bleve.NewSearchRequest(query)
M internal/nix/importable.go → internal/nix/importable.go
@@ -6,17 +6,6 @@ "alin.ovh/searchix/internal/config" ) -type Importable interface { - ImporterType() string - BleveType() string - GetName() string - GetSource() string -} - -func GetKey(i Importable) string { - return i.ImporterType() + "/" + i.GetSource() + "/" + i.GetName() -} - func MakeKey(source config.Source, id string) string { return source.Importer.String() + "/" + source.Key + "/" + id }