all repos — searchix @ 0e24d9cde2f9a9e94315fa463d6503ddd64255c2

Search engine for NixOS, nix-darwin, home-manager and NUR users

refactor: reduce use of pointers in particular, Source can no longer be nil. Instantiate a Source with an Importer of All instead

Alan Pearce
commit

0e24d9cde2f9a9e94315fa463d6503ddd64255c2

parent

05838942b9468dd1efb6d295a206730fa82264c0

M cmd/searchix-web/generate-error-page.gocmd/searchix-web/generate-error-page.go
@@ -20,8 +20,10 @@ return fault.Wrap(err, fmsg.With("could not create frontend"))
} err = components.ErrorTemplate(components.TemplateData{ - Source: nil, - Sources: []*config.Source{}, + Source: config.Source{ + Importer: config.All, + }, + Sources: []config.Source{}, Query: "", ExtraHeadHTML: "", Code: 0,
M internal/components/data.gointernal/components/data.go
@@ -12,8 +12,8 @@ "alin.ovh/searchix/internal/pagination"
) type TemplateData struct { - Sources []*config.Source - Source *config.Source + Sources []config.Source + Source config.Source Query string ExtraHeadHTML string Code int
M internal/components/page.gointernal/components/page.go
@@ -25,15 +25,15 @@ g.If(len(tdata.Sources) > 0,
Link( Rel("search"), Type("application/opensearchdescription+xml"), - TitleAttr("Searchix "+sourceNameAndType(nil)), + TitleAttr("Searchix "+tdata.Source.String()), Href(joinPath("opensearch.xml")), ), ), - g.Map(tdata.Sources, func(source *config.Source) g.Node { + g.Map(tdata.Sources, func(source config.Source) g.Node { return Link( Rel("search"), Type("application/opensearchdescription+xml"), - TitleAttr("Searchix "+sourceNameAndType(source)), + TitleAttr("Searchix "+source.String()), Href(joinPath("/", source.Importer.String(), source.Key, "opensearch.xml")), ) }),
@@ -45,18 +45,18 @@ H1(A(Href("/"), g.Text("Searchix"))),
g.If(len(tdata.Sources) > 0, A( c.Classes{ - "current": tdata.Source == nil, + "current": tdata.Source.Importer == config.All, }, g.If( - tdata.Source == nil, + tdata.Source.Importer == config.All, Href("/"), Href(joinPathQuery("/", tdata.Query)), ), g.Text("All"), ), ), - g.Map(tdata.Sources, func(source *config.Source) g.Node { - if tdata.Source != nil && tdata.Source.Name == source.Name { + g.Map(tdata.Sources, func(source config.Source) g.Node { + if tdata.Source.Importer != config.All && tdata.Source.Name == source.Name { return A( Class("current"), Href(
@@ -131,21 +131,6 @@ }
func script(s *frontend.Asset) g.Node { return Script(Src(s.ImmutablePath), Defer()) -} - -func sourceNameAndType(source *config.Source) string { - if source == nil { - return "Combined" - } - - switch source.Importer { - case config.Options: - return source.Name + " " + source.Importer.String() - case config.Packages: - return source.Name - } - - return "" } func joinPath(base string, parts ...string) string {
M internal/components/results.gointernal/components/results.go
@@ -22,14 +22,12 @@ return Span(Role("status"), g.Text("Nothing found"))
} var content g.Node - if r.Source != nil { - switch r.Source.Importer { - case config.Options: - content = Options(r.Results) - case config.Packages: - content = Packages(r.Results) - } - } else { + switch r.Source.Importer { + case config.Options: + content = Options(r.Results) + case config.Packages: + content = Packages(r.Results) + case config.All: content = Combined(r.Results) }
M internal/components/search.gointernal/components/search.go
@@ -19,7 +19,7 @@ Search(
FieldSet( Legend( ID("legend"), - H2(g.Textf("%s search", sourceNameAndType(tdata.Source))), + H2(g.Textf("%s search", tdata.Source.String())), A( Class("help"), Target("_blank"),
@@ -50,7 +50,7 @@ return Page(
tdata, P( g.Text("Search Nix packages and options from "), - MapCommaList(tdata.Sources, func(source *config.Source) g.Node { + MapCommaList(tdata.Sources, func(source config.Source) g.Node { return A(Href(source.Repo.String()), g.Text(source.Name)) }), ),
M internal/config/config.gointernal/config/config.go
@@ -25,7 +25,7 @@ type URL struct {
*url.URL } -func (u *URL) MarshalText() ([]byte, error) { +func (u URL) MarshalText() ([]byte, error) { return []byte(u.String()), nil }
@@ -52,7 +52,7 @@ type Duration struct {
time.Duration } -func (d *Duration) MarshalText() ([]byte, error) { +func (d Duration) MarshalText() ([]byte, error) { return []byte(d.String()), nil }
@@ -80,7 +80,7 @@ type LocalTime struct {
toml.LocalTime } -func (t *LocalTime) MarshalText() ([]byte, error) { +func (t LocalTime) MarshalText() ([]byte, error) { b, err := t.LocalTime.MarshalText() if err != nil { return nil, fault.Wrap(err, fmsg.With("could not marshal time value"))
@@ -145,7 +145,7 @@ config.Web.ContentSecurityPolicy.ScriptSrc,
config.Web.BaseURL.String(), ) - maps.DeleteFunc(config.Importer.Sources, func(_ string, v *Source) bool { + maps.DeleteFunc(config.Importer.Sources, func(_ string, v Source) bool { return !v.Enable })
@@ -153,7 +153,7 @@ for k, v := range config.Importer.Sources {
if v.Key == "" { v.Key = k } - if err := defaults.Set(v); err != nil { + if err := defaults.Set(&v); err != nil { return nil, fault.Wrap(err, fmsg.With("setting defaults failed")) } }
M internal/config/default.gointernal/config/default.go
@@ -2,6 +2,7 @@ package config
import ( "strconv" + "strings" "time" "github.com/pelletier/go-toml/v2"
@@ -22,7 +23,7 @@ const maxAge = (1 * 365 * 24 * time.Hour)
var DefaultConfig = Config{ DataPath: "./data", - Web: &Web{ + Web: Web{ ListenAddress: "localhost", Port: 3000, BaseURL: mustURL("http://localhost:3000"),
@@ -50,12 +51,12 @@ },
LogRequests: true, SearchTimeout: Duration{1 * time.Second}, }, - Importer: &Importer{ + Importer: Importer{ LowMemory: false, BatchSize: 10_000, Timeout: Duration{30 * time.Minute}, UpdateAt: mustLocalTime("03:00:00"), - Sources: map[string]*Source{ + Sources: map[string]Source{ "nixos": { Name: "NixOS", Order: 0,
@@ -170,10 +171,14 @@ },
} func GetDefaultConfig() string { - out, err := toml.Marshal(&DefaultConfig) + var out strings.Builder + + enc := toml.NewEncoder(&out) + + err := enc.Encode(DefaultConfig) if err != nil { panic("could not read default configuration") } - return string(out) + return out.String() }
M internal/config/fetcher.gointernal/config/fetcher.go
@@ -49,6 +49,6 @@
return err } -func (f *Fetcher) MarshalText() ([]byte, error) { +func (f Fetcher) MarshalText() ([]byte, error) { return []byte(f.String()), nil }
M internal/config/importer-type.gointernal/config/importer-type.go
@@ -9,10 +9,11 @@
type ImporterType int const ( - All ImporterType = iota - 1 - UnknownType - Packages + Packages ImporterType = 1 << iota Options + + UnknownType = -1 + All = Packages | Options ) func (i ImporterType) String() string {
@@ -26,6 +27,10 @@ return "options"
} return fmt.Sprintf("Type(%d)", i) +} + +func (i ImporterType) Includes(m ImporterType) bool { + return i&m == m } func (i ImporterType) Singular() string {
@@ -59,6 +64,6 @@
return err } -func (i *ImporterType) MarshalText() ([]byte, error) { +func (i ImporterType) MarshalText() ([]byte, error) { return []byte(i.String()), nil }
M internal/config/repository.gointernal/config/repository.go
@@ -107,6 +107,6 @@
return err } -func (f *RepoType) MarshalText() ([]byte, error) { +func (f RepoType) MarshalText() ([]byte, error) { return []byte(f.String()), nil }
M internal/config/structs.gointernal/config/structs.go
@@ -12,9 +12,9 @@ "github.com/creasty/defaults"
) type Config struct { - DataPath string `comment:"Path to store index data."` - Web *Web `comment:"Settings for the web server"` - Importer *Importer `comment:"Settings for the import job"` + DataPath string `comment:"Path to store index data."` + Web Web `comment:"Settings for the web server"` + Importer Importer `comment:"Settings for the import job"` } type Web struct {
@@ -31,7 +31,7 @@ SearchTimeout Duration `comment:"Timeout for search requests"`
} type Importer struct { - Sources map[string]*Source + Sources map[string]Source LowMemory bool `comment:"Use less memory at the expense of import performance"` BatchSize int `comment:"Number of items to process in each batch (affects memory usage)."` Timeout Duration `comment:"Abort fetch and import process for all jobs if it takes longer than this value."`
@@ -67,12 +67,14 @@ Enable bool `comment:"Enable searching for manpages"`
Path string `comment:"Path to the manpage-urls.json file from repository root"` } -func (source *Source) String() string { +func (source Source) String() string { switch source.Importer { case Options: return source.Name + " " + source.Importer.String() case Packages: return source.Name + case All: + return "All" default: return fmt.Sprintf("Source(%s)", source.Name) }
M internal/fetcher/channel.gointernal/fetcher/channel.go
@@ -17,13 +17,13 @@ "alin.ovh/searchix/internal/index"
) type ChannelFetcher struct { - Source *config.Source + Source config.Source SourceFile string *Options } func NewChannelFetcher( - source *config.Source, + source config.Source, options *Options, ) (*ChannelFetcher, error) { switch source.Importer {
M internal/fetcher/download.gointernal/fetcher/download.go
@@ -13,13 +13,13 @@ "alin.ovh/searchix/internal/index"
) type DownloadFetcher struct { - Source *config.Source + Source config.Source SourceFile string *Options } func NewDownloadFetcher( - source *config.Source, + source config.Source, options *Options, ) (*DownloadFetcher, error) { switch source.Importer {
M internal/fetcher/main.gointernal/fetcher/main.go
@@ -30,7 +30,7 @@ FetchIfNeeded(context.Context, *index.SourceMeta) (*FetchedFiles, error)
} func New( - source *config.Source, + source config.Source, opts *Options, ) (fetcher Fetcher, err error) { target := source.JoinPath("")
@@ -61,7 +61,7 @@ return
} func Open( - source *config.Source, + source config.Source, opts *Options, ) (*FetchedFiles, error) { root := opts.Root
M internal/fetcher/nixpkgs-channel.gointernal/fetcher/nixpkgs-channel.go
@@ -13,7 +13,7 @@ "github.com/Southclaws/fault/fmsg"
) type NixpkgsChannelFetcher struct { - Source *config.Source + Source config.Source *Options }
@@ -24,7 +24,7 @@ return url, fault.Wrap(err, fmsg.Withf("error creating URL"))
} func NewNixpkgsChannelFetcher( - source *config.Source, + source config.Source, options *Options, ) (*NixpkgsChannelFetcher, error) { switch source.Importer {
M internal/importer/main.gointernal/importer/main.go
@@ -181,7 +181,7 @@ }
func (imp *Importer) PruneSource( _ context.Context, - source *config.Source, + source config.Source, ) error { read := imp.options.ReadIndex write := imp.options.WriteIndex
@@ -240,8 +240,8 @@ func (imp *Importer) createSourceFetcher(
parent context.Context, meta *index.Meta, forceUpdate bool, -) func(*config.Source) error { - return func(source *config.Source) error { +) func(config.Source) error { + return func(source config.Source) error { logger := imp.options.Logger.With("name", source.Key) pdb, err := programs.New(source, &programs.Options{ Logger: logger,
@@ -322,8 +322,8 @@
func (imp *Importer) createSourceImporter( parent context.Context, meta *index.Meta, -) func(*config.Source) error { - return func(source *config.Source) error { +) func(config.Source) error { + return func(source config.Source) error { logger := imp.options.Logger.With("name", source.Key) pdb, err := programs.New(source, &programs.Options{ Logger: logger,
M internal/importer/options.gointernal/importer/options.go
@@ -65,12 +65,12 @@ ms *mapstructure.Decoder
log *log.Logger optJSON nixOptionJSON infile io.ReadCloser - source *config.Source + source config.Source } func NewOptionProcessor( infile io.ReadCloser, - source *config.Source, + source config.Source, log *log.Logger, ) (*OptionIngester, error) { i := OptionIngester{
M internal/importer/package.gointernal/importer/package.go
@@ -49,7 +49,7 @@ ms *mapstructure.Decoder
log *log.Logger pkg packageJSON infile io.ReadCloser - source *config.Source + source config.Source programs *programs.DB }
@@ -70,7 +70,7 @@ }
func NewPackageProcessor( infile io.ReadCloser, - source *config.Source, + source config.Source, log *log.Logger, programsDB *programs.DB, ) (*PackageIngester, error) {
M internal/importer/utils.gointernal/importer/utils.go
@@ -46,7 +46,7 @@ URL: url,
}, nil } -func setRepoRevision(file io.ReadCloser, source *config.Source) error { +func setRepoRevision(file io.ReadCloser, source config.Source) error { if file != nil { defer file.Close() var str strings.Builder
M internal/index/search.gointernal/index/search.go
@@ -133,7 +133,7 @@ }
func (index *ReadIndex) Search( ctx context.Context, - source *config.Source, + source config.Source, keyword string, from int, pageSize int,
@@ -141,7 +141,7 @@ facets url.Values,
) (*Result, error) { query := bleve.NewBooleanQuery() - if source != nil { + if source.Importer != config.All { query.AddMust( setField(bleve.NewTermQuery(source.Key), "Source"), )
@@ -249,11 +249,11 @@ search.SortBy([]string{"_id"})
} } - if source == nil || source.Importer == config.Packages { + if source.Importer.Includes(config.Packages) { search.AddFacet("Package set", bleve.NewFacetRequest("PackageSet", 10)) search.AddFacet("Platform", bleve.NewFacetRequest("Platforms", 10)) } - if source == nil || source.Importer == config.Options { + if source.Importer.Includes(config.Options) { search.AddFacet("Option set", bleve.NewFacetRequest("Parents", 10)) }
@@ -266,14 +266,14 @@ }
func (index *ReadIndex) ImportedBefore( cutoff time.Time, - source *config.Source, + source config.Source, ) (*bleve.SearchResult, error) { cutoffQuery := bleve.NewDateRangeQuery(time.UnixMilli(0), cutoff) cutoffQuery.SetField("ImportedAt") all := bleve.NewConjunctionQuery(cutoffQuery) - if source != nil { + if source.Importer != config.All { sourceQuery := bleve.NewTermQuery(source.Key) sourceQuery.SetField("Source")
@@ -292,8 +292,8 @@
return res, nil } -func (index *ReadIndex) Count(source *config.Source) (uint64, error) { - if source == nil { +func (index *ReadIndex) Count(source config.Source) (uint64, error) { + if source.Importer == config.All { count, err := index.index.DocCount() if err != nil { return 0, fault.Wrap(err)
@@ -323,7 +323,7 @@ }
func (index *ReadIndex) GetDocument( ctx context.Context, - source *config.Source, + source config.Source, id string, ) (nix.Importable, error) { key := nix.MakeKey(source, id)
M internal/index/search_test.gointernal/index/search_test.go
@@ -47,8 +47,8 @@
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() - source := cfg.Importer.Sources["nixpkgs"] - if source == nil || !source.Enable { + source, exists := cfg.Importer.Sources["nixpkgs"] + if !exists || !source.Enable { t.Fatal("expected source to exist and be enabled") }
M internal/manpages/manpages.gointernal/manpages/manpages.go
@@ -40,7 +40,7 @@ }
func (m *URLMap) Update( ctx context.Context, - source *config.Source, + source config.Source, ) error { if !source.Manpages.Enable { return fault.New("manpages not enabled for this source")
@@ -109,7 +109,7 @@
return url, true } -func makeManpageURL(source *config.Source) (string, error) { +func makeManpageURL(source config.Source) (string, error) { url, err := source.Repo.GetRawFileURL(source.Manpages.Path) if err != nil { return "", fault.Wrap(err, fmsg.With("failed to join manpage URL"))
M internal/nix/importable.gointernal/nix/importable.go
@@ -16,7 +16,7 @@ func GetKey(i Importable) string {
return i.BleveType() + "/" + i.GetSource() + "/" + i.GetName() } -func MakeKey(source *config.Source, id string) string { +func MakeKey(source config.Source, id string) string { return source.Importer.Singular() + "/" + source.Key + "/" + id }
M internal/programs/programs.gointernal/programs/programs.go
@@ -17,7 +17,7 @@ "alin.ovh/searchix/internal/file"
) type DB struct { - source *config.Source + source config.Source logger *log.Logger root *file.Root db *sql.DB
@@ -29,7 +29,7 @@ Logger *log.Logger
Root *file.Root } -func New(source *config.Source, options *Options) (*DB, error) { +func New(source config.Source, options *Options) (*DB, error) { db, err := sql.Open("sqlite", fmt.Sprintf( "file:%s?mode=%s&_pragma=foreign_keys(1)&_pragma=mmap_size(%d)", //nolint:forbidigo // external package
M internal/server/global.gointernal/server/global.go
@@ -30,7 +30,7 @@
errorHandler func(w http.ResponseWriter, r *http.Request, message string, statusCode int) } -func (g *GlobalHandler) NewSourceHandler(source *config.Source) *SourceHandler { +func (g *GlobalHandler) NewSourceHandler(source config.Source) *SourceHandler { h := &SourceHandler{ global: g, source: source,
@@ -44,7 +44,7 @@
return h } -func (g *GlobalHandler) Search(source *config.Source, w http.ResponseWriter, r *http.Request) { +func (g *GlobalHandler) Search(source config.Source, w http.ResponseWriter, r *http.Request) { facets := r.URL.Query() facets.Del("query") facets.Del("page")
@@ -148,7 +148,9 @@ }
} func (g *GlobalHandler) CombinedSearch(w http.ResponseWriter, r *http.Request) { - g.Search(nil, w, r) + g.Search(config.Source{ + Importer: config.All, + }, w, r) } func (g *GlobalHandler) RootOpenSearch(w http.ResponseWriter, _ *http.Request) {
M internal/server/mux.gointernal/server/mux.go
@@ -24,7 +24,7 @@ Message string
Code int } -var sources []*config.Source +var sources []config.Source func applyDevModeOverrides(cfg *config.Config) { if len(cfg.Web.ContentSecurityPolicy.ScriptSrc) == 0 {
@@ -36,8 +36,8 @@ "'unsafe-inline'",
) } -func sortSources(ss map[string]*config.Source) { - sources = slices.SortedFunc(maps.Values(ss), func(a, b *config.Source) int { +func sortSources(ss map[string]config.Source) { + sources = slices.SortedFunc(maps.Values(ss), func(a, b config.Source) int { return cmp.Or( cmp.Compare(a.Order, b.Order), strings.Compare(a.Key, b.Key),
M internal/server/source.gointernal/server/source.go
@@ -14,7 +14,7 @@ )
type SourceHandler struct { global *GlobalHandler - source *config.Source + source config.Source mux *http.ServeMux }