refactor!: split main function into commands BREAKING CHANGE: searchix-web requires `serve` argument
16 files changed, 469 insertions(+), 230 deletions(-)
changed files
- cmd/searchix-web/defaults.go
- cmd/searchix-web/generate-error-page.go
- cmd/searchix-web/ingest.go
- cmd/searchix-web/main.go
- cmd/searchix-web/serve.go
- cmd/searchix-web/version.go
- internal/importer/importer.go
- internal/importer/main.go
- internal/importer/main_test.go
- internal/importer/options.go
- internal/importer/package.go
- justfile
- modd.conf
- nix/modules/default.nix
- nix/package.nix
- nix/pre-commit-checks.nix
A cmd/searchix-web/defaults.go
@@ -0,0 +1,34 @@ +package main + +import ( + "fmt" + + "github.com/Southclaws/fault" + + "alin.ovh/searchix/internal/config" +) + +type PrintDefaultsOptions struct{} + +func (*PrintDefaultsOptions) Execute(_ []string) error { + _, err := fmt.Print(config.GetDefaultConfig()) + if err != nil { + return fault.Wrap(err) + } + + return nil +} + +func init() { + var opts PrintDefaultsOptions + + _, err := parser.AddCommand( + "defaults", + "print defaults", + "print default configuration", + &opts, + ) + if err != nil { + panic(err) + } +}
A cmd/searchix-web/generate-error-page.go
@@ -0,0 +1,50 @@ +package main + +import ( + "os" + + "github.com/Southclaws/fault" + "github.com/Southclaws/fault/fmsg" + + "alin.ovh/searchix/frontend" + "alin.ovh/searchix/internal/components" + "alin.ovh/searchix/internal/config" +) + +type GenerateErrorPageOptions struct{} + +func (*GenerateErrorPageOptions) Execute(_ []string) error { + assets, err := frontend.New() + if err != nil { + return fault.Wrap(err, fmsg.With("could not create frontend")) + } + + err = components.ErrorTemplate(components.TemplateData{ + Source: nil, + Sources: []*config.Source{}, + Query: "", + ExtraHeadHTML: "", + Code: 0, + Message: `{{placeholder "http.error.status_code"}} {{placeholder "http.error.status_text"}}`, + Assets: assets, + }).Render(os.Stdout) + if err != nil { + panic("failed to render error template: " + err.Error()) + } + + return nil +} + +func init() { + var opts GenerateErrorPageOptions + + _, err := parser.AddCommand( + "generate-error-page", + "generate error page template", + "generate error page template for use with a reverse proxy", + &opts, + ) + if err != nil { + panic(err) + } +}
A cmd/searchix-web/ingest.go
@@ -0,0 +1,120 @@ +package main + +import ( + "context" + "os" + "os/signal" + + "github.com/Southclaws/fault" + "github.com/Southclaws/fault/fmsg" + + "alin.ovh/searchix/internal/file" + "alin.ovh/searchix/internal/importer" + "alin.ovh/searchix/internal/index" + "alin.ovh/searchix/internal/manpages" + "alin.ovh/searchix/internal/storage" +) + +type IngestOptions struct { + Fetch bool `long:"fetch" description:"pre-fetch data"` + Offline bool `long:"offline" description:"offline mode"` + Replace bool `long:"replace" description:"replace existing storage"` + Reindex bool `long:"reindex" description:"reindex existing index"` +} + +func (opts *IngestOptions) Execute(_ []string) error { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + root, err := file.CreateAndOpenRoot(cfg.DataPath) + if err != nil { + return fault.Wrap(err, fmsg.With("Failed to open data root")) + } + defer root.Close() + + store, err := storage.New(&storage.Options{ + Root: root, + Logger: logger.Named("store"), + }) + if err != nil { + return fault.Wrap(err, fmsg.With("Failed to create store")) + } + defer store.Close() + + _, write, exists, err := index.OpenOrCreate( + &index.Options{ + Config: cfg, + Force: opts.Reindex, + LowMemory: cfg.Importer.LowMemory, + BatchSize: cfg.Importer.BatchSize, + Logger: logger.Named("index"), + Root: root, + Store: store, + }, + ) + if err != nil { + return fault.Wrap(err, fmsg.With("Failed to open or create index")) + } + + mdb := manpages.New(&manpages.Options{ + Logger: logger.Named("manpages"), + Root: root, + }) + + imp, err := importer.New(cfg, &importer.Options{ + Storage: store, + WriteIndex: write, + LowMemory: cfg.Importer.LowMemory, + Logger: logger.Named("importer"), + Manpages: mdb, + Root: root, + Offline: opts.Offline, + }) + if err != nil { + return fault.Wrap(err, fmsg.With("Failed to create importer")) + } + + if !exists || opts.Replace || opts.Fetch { + err = imp.Start(ctx, true, opts.Fetch && !opts.Replace, nil) + if err != nil { + return fault.Wrap(err, fmsg.With("Failed to start importer")) + } + } + + if !exists || opts.Reindex { + for _, source := range cfg.Importer.Sources { + hadErrors, err := importer.ImportSource( + ctx, + logger.Named("importer"), + store, + source, + write, + ) + if err != nil { + return fault.Wrap(err, fmsg.Withf("Failed to import source %s", source.Name)) + } + + if hadErrors { + logger.Warn("Imported source encountered errors", "source", source.Name) + } + } + } + + return nil +} + +func init() { + var opts IngestOptions + + cmd, err := parser.AddCommand( + "ingest", + "Ingest data", + "Fetch, store and index data", + &opts, + ) + if err != nil { + panic(err) + } + + cmd.Aliases = []string{"import"} +}
M cmd/searchix-web/main.go → cmd/searchix-web/main.go
@@ -1,230 +1,80 @@ package main import ( - "context" "errors" "fmt" "os" - "os/signal" "runtime/pprof" - "sync" - flags "github.com/jessevdk/go-flags" + "alin.ovh/x/log" + "github.com/Southclaws/fault" + "github.com/Southclaws/fault/fmsg" + "github.com/jessevdk/go-flags" - "alin.ovh/searchix/frontend" - "alin.ovh/searchix/internal/components" "alin.ovh/searchix/internal/config" - "alin.ovh/searchix/internal/file" - "alin.ovh/searchix/internal/importer" - "alin.ovh/searchix/internal/index" - "alin.ovh/searchix/internal/manpages" - "alin.ovh/searchix/internal/server" - "alin.ovh/searchix/internal/storage" - "alin.ovh/searchix/web" - "alin.ovh/x/log" ) -var options struct { - Config flags.Filename `long:"config" default:"config.toml" description:"config file to use"` - PrintDefaultConfig bool `long:"print-default-config" description:"print default configuration and exit"` - GenerateErrorPage bool `long:"generate-error-page" description:"generate error page and exit"` - Dev bool `long:"dev" description:"enable live reloading and nicer logging"` - Prefetch bool `long:"prefetch" description:"pre-fetch data and exit"` - Replace bool `long:"replace" description:"replace existing storage and exit"` - Reindex bool `long:"reindex" description:"reindex existing index and exit"` - Rebuild bool `long:"rebuild" description:"rebuild existing index and exit"` - Offline bool `long:"offline" description:"run in offline mode"` - Version bool `long:"version" description:"print version information"` - CPUProfile flags.Filename `long:"cpuprofile" description:"output CPU profile to FILE" value-name:"FILE"` +type Options struct { + Dev bool `long:"dev" description:"enable live reloading and nicer logging"` + Config flags.Filename `long:"config" description:"config file to use"` + CPUProfile flags.Filename `long:"cpuprofile" description:"output CPU profile to FILE" value-name:"FILE"` } -var parser = flags.NewParser(&options, flags.Default) - -func main() { - _, err := parser.Parse() - if err != nil { - os.Exit(1) - } - if flags.WroteHelp(err) { - return - } - - if options.Version { - _, err := fmt.Fprintf(os.Stderr, "searchix %s\n", config.Version) - if err != nil { - panic("can't write to standard error?!") - } - os.Exit(0) - } - if options.PrintDefaultConfig { - _, err := fmt.Print(config.GetDefaultConfig()) - if err != nil { - panic("can't write to standard output?!") - } - os.Exit(0) - } - if options.GenerateErrorPage { - assets, err := frontend.New() - if err != nil { - panic("failed to create assets: " + err.Error()) - } - - err = components.ErrorTemplate(components.TemplateData{ - Source: nil, - Sources: []*config.Source{}, - Query: "", - ExtraHeadHTML: "", - Code: 0, - Message: `{{placeholder "http.error.status_code"}} {{placeholder "http.error.status_text"}}`, - Assets: assets, - }).Render(os.Stdout) - if err != nil { - panic("failed to render error template: " + err.Error()) - } - os.Exit(0) - } - - if options.CPUProfile != "" { - //nolint:forbidigo // admin specifies profile file location - f, err := os.Create(string(options.CPUProfile)) - if err != nil { - panic("can't create CPU profile: " + err.Error()) - } - err = pprof.StartCPUProfile(f) - if err != nil { - panic("can't start CPU profile: " + err.Error()) - } - defer pprof.StopCPUProfile() - } - - logger := log.Configure(!options.Dev) - - cfg, err := config.GetConfig(string(options.Config), logger) - if err != nil { - logger.Fatal("Failed to parse config file", "error", err) - } - - log.SetLevel(cfg.LogLevel) - - ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) - defer cancel() - - root, err := file.CreateAndOpenRoot(cfg.DataPath) - if err != nil { - logger.Fatal("Failed to open data root", "error", err) - } - defer root.Close() - - store, err := storage.New(&storage.Options{ - Root: root, - Logger: logger.Named("store"), - }) - if err != nil { - logger.Fatal("Failed to create store", "error", err) - } - defer store.Close() - - read, write, exists, err := index.OpenOrCreate( - &index.Options{ - Config: cfg, - Force: options.Reindex, - LowMemory: cfg.Importer.LowMemory, - BatchSize: cfg.Importer.BatchSize, - Logger: logger.Named("index"), - Root: root, - Store: store, - }, - ) - if err != nil { - logger.Fatal("Failed to open or create index", "error", err) - } +var ( + globalOptions Options + cfg *config.Config + logger *log.Logger +) - mdb := manpages.New(&manpages.Options{ - Logger: logger.Named("manpages"), - Root: root, - }) +var parser = flags.NewParser(&globalOptions, flags.HelpFlag|flags.PassDoubleDash) - s, err := web.New(cfg, logger, &server.Options{ - ReadIndex: read, - ManpagesURLMap: mdb, - Store: store, - }) - if err != nil { - logger.Fatal("Failed to initialise searchix-web", "error", err) - } +func main() { + parser.CommandHandler = func(cmd flags.Commander, args []string) (err error) { + switch cmd.(type) { + case *PrintDefaultsOptions, *Version: + default: + logger = log.Configure(!globalOptions.Dev) - imp, err := importer.New(cfg, &importer.Options{ - Storage: store, - WriteIndex: write, - LowMemory: cfg.Importer.LowMemory, - Logger: logger.Named("importer"), - Manpages: mdb, - Root: root, - Offline: options.Offline, - }) - if err != nil { - logger.Fatal("Failed to create importer", "error", err) - } + cfg, err = config.GetConfig(string(globalOptions.Config), logger.Named("config")) + if err != nil { + return fault.Wrap(err, fmsg.With("Failed to parse config file")) + } - if !exists || options.Replace || options.Prefetch { - err := imp.Start(ctx, true, options.Prefetch, nil) - if err != nil { - logger.Fatal("Failed to start importer", "error", err) + log.SetLevel(cfg.LogLevel) } - if options.Replace || options.Prefetch { - return - } - } - - if !exists || options.Reindex { - for _, source := range cfg.Importer.Sources { - hadErrors, err := importer.ImportSource( - ctx, - logger.Named("importer"), - store, - source, - write, - ) + if globalOptions.CPUProfile != "" { + //nolint:forbidigo // admin specifies profile file location + f, err := os.Create(string(globalOptions.CPUProfile)) if err != nil { - logger.Fatal("Failed to import source", "source", source.Name, "error", err) + panic("can't create CPU profile: " + err.Error()) } - - if hadErrors { - logger.Warn("Imported source encountered errors", "source", source.Name) + err = pprof.StartCPUProfile(f) + if err != nil { + panic("can't start CPU profile: " + err.Error()) } + defer pprof.StopCPUProfile() } - if options.Reindex { - return - } + return cmd.Execute(args) } - err = imp.EnsureSourcesIndexed(ctx, read) + _, err := parser.Parse() if err != nil { - logger.Fatal("Failed to setup index", "error", err) - } + var flagErr *flags.Error + if errors.As(err, &flagErr) { + switch flagErr.Type { + case flags.ErrHelp: + parser.WriteHelp(os.Stdout) - wg := &sync.WaitGroup{} - wg.Add(1) - go func() { - sCtx, cancel := context.WithCancel(ctx) - defer wg.Done() - defer cancel() - err := s.Start(sCtx, options.Dev) - if err != nil && !errors.Is(err, context.Canceled) { - // Error starting or closing listener: - logger.Fatal("error", "error", err) + return + default: + fmt.Println(flagErr.Error()) + } + } else { + fmt.Println(err) } - }() - wg.Add(1) - go func() { - defer wg.Done() - imp.StartUpdateTimer(ctx) - }() - - <-ctx.Done() - s.Stop() - wg.Wait() + os.Exit(1) + } }
A cmd/searchix-web/serve.go
@@ -0,0 +1,147 @@ +package main + +import ( + "context" + "errors" + "os" + "os/signal" + "sync" + + "github.com/Southclaws/fault" + "github.com/Southclaws/fault/fmsg" + + "alin.ovh/searchix/internal/file" + "alin.ovh/searchix/internal/importer" + "alin.ovh/searchix/internal/index" + "alin.ovh/searchix/internal/manpages" + "alin.ovh/searchix/internal/server" + "alin.ovh/searchix/internal/storage" + "alin.ovh/searchix/web" +) + +type ServeOptions struct{} + +func (opts *ServeOptions) Execute(_ []string) (err error) { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + root, err := file.CreateAndOpenRoot(cfg.DataPath) + if err != nil { + return fault.Wrap(err, fmsg.With("Failed to open data root")) + } + defer root.Close() + + store, err := storage.New(&storage.Options{ + Root: root, + Logger: logger.Named("store"), + }) + if err != nil { + return fault.Wrap(err, fmsg.With("Failed to create store")) + } + defer store.Close() + + read, write, exists, err := index.OpenOrCreate( + &index.Options{ + Config: cfg, + LowMemory: cfg.Importer.LowMemory, + BatchSize: cfg.Importer.BatchSize, + Logger: logger.Named("index"), + Root: root, + Store: store, + }, + ) + if err != nil { + return fault.Wrap(err, fmsg.With("Failed to open or create index")) + } + + mdb := manpages.New(&manpages.Options{ + Logger: logger.Named("manpages"), + Root: root, + }) + + s, err := web.New(cfg, logger, &server.Options{ + ReadIndex: read, + ManpagesURLMap: mdb, + Store: store, + }) + if err != nil { + return fault.Wrap(err, fmsg.With("Failed to initialise searchix-web")) + } + + imp, err := importer.New(cfg, &importer.Options{ + Storage: store, + WriteIndex: write, + LowMemory: cfg.Importer.LowMemory, + Logger: logger.Named("importer"), + Manpages: mdb, + Root: root, + }) + if err != nil { + return fault.Wrap(err, fmsg.With("Failed to create importer")) + } + + if !exists { + err = imp.Start(ctx, true, false, nil) + if err != nil { + return fault.Wrap(err, fmsg.With("Failed to start importer")) + } + + for _, source := range cfg.Importer.Sources { + hadErrors, err := importer.ImportSource( + ctx, + logger.Named("importer"), + store, + source, + write, + ) + if err != nil { + return fault.Wrap(err, fmsg.Withf("Failed to import source %s", source.Name)) + } + + if hadErrors { + logger.Warn("Imported source encountered errors", "source", source.Name) + } + } + } + + err = imp.EnsureSourcesIndexed(ctx, read) + if err != nil { + return fault.Wrap(err, fmsg.With("Failed to ensure sources indexed")) + } + + wg := &sync.WaitGroup{} + wg.Add(1) + go func() { + sCtx, cancel := context.WithCancel(ctx) + defer cancel() + defer wg.Done() + err := s.Start(sCtx, globalOptions.Dev) + if err != nil && !errors.Is(err, context.Canceled) { + // Error starting or closing listener: + logger.Fatal("error", "error", err) + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + imp.StartUpdateTimer(ctx) + }() + + <-ctx.Done() + s.Stop() + wg.Wait() + + return +} + +func init() { + var opts ServeOptions + + cmd, err := parser.AddCommand("serve", "run server", "Serve web interface", &opts) + if err != nil { + panic(err) + } + + cmd.Aliases = []string{"run"} +}
A cmd/searchix-web/version.go
@@ -0,0 +1,29 @@ +package main + +import ( + "fmt" + + "github.com/Southclaws/fault" + + "alin.ovh/searchix/internal/config" +) + +type Version struct{} + +func (*Version) Execute(_ []string) error { + _, err := fmt.Printf("searchix %s\n", config.Version) + if err != nil { + return fault.Wrap(err) + } + + return nil +} + +func init() { + var version Version + + _, err := parser.AddCommand("version", "print version", "print version", &version) + if err != nil { + panic(err) + } +}
M internal/importer/importer.go → internal/importer/importer.go
@@ -4,11 +4,12 @@ import ( "context" "sync" + "alin.ovh/x/log" + "alin.ovh/searchix/internal/config" "alin.ovh/searchix/internal/index" "alin.ovh/searchix/internal/nix" "alin.ovh/searchix/internal/storage" - "alin.ovh/x/log" ) type Processor interface {
M internal/importer/main.go → internal/importer/main.go
@@ -9,6 +9,8 @@ "slices" "strings" "time" + "alin.ovh/x/log" + "alin.ovh/searchix/internal/config" "alin.ovh/searchix/internal/fetcher" "alin.ovh/searchix/internal/file"@@ -16,7 +18,6 @@ "alin.ovh/searchix/internal/index" "alin.ovh/searchix/internal/manpages" "alin.ovh/searchix/internal/programs" "alin.ovh/searchix/internal/storage" - "alin.ovh/x/log" "github.com/Southclaws/fault" "github.com/Southclaws/fault/fmsg"
M internal/importer/main_test.go → internal/importer/main_test.go
@@ -3,12 +3,13 @@ import ( "testing" + "alin.ovh/x/log" + "alin.ovh/searchix/internal/config" "alin.ovh/searchix/internal/file" "alin.ovh/searchix/internal/index" "alin.ovh/searchix/internal/manpages" "alin.ovh/searchix/internal/storage" - "alin.ovh/x/log" ) var cfg = config.DefaultConfig@@ -48,7 +49,7 @@ Manpages: manpages.New(&manpages.Options{ Logger: logger.Named("manpages"), Root: tmp, }), - Offline: false, + Offline: true, Root: tmp, Storage: store, })
M internal/importer/options.go → internal/importer/options.go
@@ -5,9 +5,10 @@ "context" "io" "reflect" + "alin.ovh/x/log" + "alin.ovh/searchix/internal/config" "alin.ovh/searchix/internal/nix" - "alin.ovh/x/log" "github.com/Southclaws/fault" "github.com/Southclaws/fault/fmsg"
M internal/importer/package.go → internal/importer/package.go
@@ -8,10 +8,11 @@ "net/url" "reflect" "strings" + "alin.ovh/x/log" + "alin.ovh/searchix/internal/config" "alin.ovh/searchix/internal/nix" "alin.ovh/searchix/internal/programs" - "alin.ovh/x/log" "github.com/Southclaws/fault" "github.com/Southclaws/fault/fmsg"
M justfile → justfile
@@ -30,7 +30,7 @@ build: nix build .# generate-defaults: - wgo run --exit ./cmd/searchix-web --print-default-config > defaults.toml + wgo run --exit ./cmd/searchix-web defaults > defaults.toml precommit: nix-build -A pre-commit-check@@ -63,11 +63,11 @@ dev: modd -fetch *flags: - wgo run --exit ./cmd/searchix-web --config config.toml --fetch --dev {{ flags }} +ingest *flags: + wgo run --exit ./cmd/searchix-web --config config.toml ingest --dev {{ flags }} -replace *flags: - wgo run --exit ./cmd/searchix-web --config config.toml --replace --dev {{ flags }} +fetch: (ingest "--fetch") -reindex *flags: - wgo run --exit ./cmd/searchix-web --config config.toml --reindex --dev {{ flags }} +replace: (ingest "--replace") + +reindex: (ingest "--reindex")
M modd.conf → modd.conf
@@ -1,7 +1,7 @@ internal/index/indexer.go { - prep +onchange: "just reindex" + prep +onchange: "just ingest --reindex" } **/*.go config.toml { - daemon +sigint: go run -ldflags="-X alin.ovh/searchix/internal/config.Version=$(git describe --tags --abbrev=0)" ./cmd/searchix-web --dev --config config.toml + daemon +sigint: go run -ldflags="-X alin.ovh/searchix/internal/config.Version=$(git describe --tags --abbrev=0)" ./cmd/searchix-web --config config.toml serve --dev }
M nix/modules/default.nix → nix/modules/default.nix
@@ -240,21 +240,25 @@ }; }; config = mkIf cfg.enable { - systemd.services.searchix = { - description = "Searchix Nix option search"; - wantedBy = [ "multi-user.target" ]; - path = with pkgs; [ nix ]; - inherit (cfg) environment; - serviceConfig = - defaultServiceConfig - // { - ExecStart = "${package}/bin/searchix-web --config ${(settingsFormat.generate "searchix-config.toml" cfg.settings)}"; - } - // lib.optionalAttrs (cfg.settings.web.port < 1024) { - AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ]; - CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ]; - }; - }; + systemd.services.searchix = + let + configFile = settingsFormat.generate "searchix-config.toml" cfg.settings; + in + { + description = "Searchix Nix option search"; + wantedBy = [ "multi-user.target" ]; + path = with pkgs; [ nix ]; + inherit (cfg) environment; + serviceConfig = + defaultServiceConfig + // { + ExecStart = "${package}/bin/searchix-web --config ${configFile} serve"; + } + // lib.optionalAttrs (cfg.settings.web.port < 1024) { + AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ]; + CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ]; + }; + }; users.users = optionalAttrs (cfg.user == "searchix") { searchix = {
M nix/package.nix → nix/package.nix
@@ -51,7 +51,7 @@ postInstall = '' install -d -m755 $out/lib/searchix/static cp -r $src/frontend/static $out/lib/searchix cp ${css} $out/lib/searchix/static/base.css - $out/bin/searchix-web --generate-error-page > $out/lib/searchix/error.html + $out/bin/searchix-web generate-error-page > $out/lib/searchix/error.html ''; modules = ../gomod2nix.toml;
M nix/pre-commit-checks.nix → nix/pre-commit-checks.nix
@@ -75,7 +75,7 @@ ]; entry = let script = pkgs.writeShellScript "generate-default-config" '' - ${pkgs.wgo}/bin/wgo run -exit ./cmd/searchix-web --print-default-config > defaults.toml + ${pkgs.wgo}/bin/wgo run -exit ./cmd/searchix-web defaults > defaults.toml ''; in builtins.toString script;