feat: link man pages like how NixOS does it
12 files changed, 276 insertions(+), 18 deletions(-)
changed files
M cmd/searchix-web/main.go → cmd/searchix-web/main.go
@@ -17,6 +17,8 @@ "go.alanpearce.eu/searchix/internal/components" "go.alanpearce.eu/searchix/internal/config" "go.alanpearce.eu/searchix/internal/importer" "go.alanpearce.eu/searchix/internal/index" + "go.alanpearce.eu/searchix/internal/manpages" + "go.alanpearce.eu/searchix/internal/server" "go.alanpearce.eu/searchix/web" "go.alanpearce.eu/x/log" )@@ -104,7 +106,12 @@ if err != nil { logger.Fatal("Failed to open or create index", "error", err) } - s, err := web.New(cfg, logger, read) + mdb := manpages.New(cfg, logger.Named("manpages")) + + s, err := web.New(cfg, logger, &server.Options{ + ReadIndex: read, + ManpagesURLMap: mdb, + }) if err != nil { logger.Fatal("Failed to initialise searchix-web", "error", err) }@@ -113,6 +120,7 @@ imp, err := importer.New(cfg, &importer.Options{ WriteIndex: write, LowMemory: cfg.Importer.LowMemory, Logger: logger.Named("importer"), + Manpages: mdb, }) if err != nil { logger.Fatal("Failed to create importer", "error", err)
M defaults.toml → defaults.toml
@@ -114,6 +114,13 @@ Enable = false # Nix attribute name (i.e. nix-instantiate) that builds a programs.sqlite file Attribute = '' +# Used to enable searching for manpages +[Importer.Sources.darwin.Manpages] +# Enable searching for manpages +Enable = false +# Path to the manpage-urls.json file from repository root +Path = '' + [Importer.Sources.home-manager] # Human-readable name of source for generating links Name = 'Home Manager'@@ -156,6 +163,13 @@ Enable = false # Nix attribute name (i.e. nix-instantiate) that builds a programs.sqlite file Attribute = '' +# Used to enable searching for manpages +[Importer.Sources.home-manager.Manpages] +# Enable searching for manpages +Enable = false +# Path to the manpage-urls.json file from repository root +Path = '' + [Importer.Sources.nixos] # Human-readable name of source for generating links Name = 'NixOS'@@ -198,6 +212,13 @@ Enable = false # Nix attribute name (i.e. nix-instantiate) that builds a programs.sqlite file Attribute = '' +# Used to enable searching for manpages +[Importer.Sources.nixos.Manpages] +# Enable searching for manpages +Enable = false +# Path to the manpage-urls.json file from repository root +Path = '' + [Importer.Sources.nixpkgs] # Human-readable name of source for generating links Name = 'Nix Packages'@@ -240,6 +261,13 @@ Enable = true # Nix attribute name (i.e. nix-instantiate) that builds a programs.sqlite file Attribute = 'programs.sqlite' +# Used to enable searching for manpages +[Importer.Sources.nixpkgs.Manpages] +# Enable searching for manpages +Enable = true +# Path to the manpage-urls.json file from repository root +Path = '/doc/manpage-urls.json' + [Importer.Sources.nur] # Human-readable name of source for generating links Name = 'NUR'@@ -281,3 +309,10 @@ # Enable searching for programs in multi-program packages Enable = false # Nix attribute name (i.e. nix-instantiate) that builds a programs.sqlite file Attribute = '' + +# Used to enable searching for manpages +[Importer.Sources.nur.Manpages] +# Enable searching for manpages +Enable = false +# Path to the manpage-urls.json file from repository root +Path = ''
M internal/config/default.go → internal/config/default.go
@@ -67,7 +67,11 @@ Attribute: "options", OutputPath: "share/doc/nixos", Timeout: Duration{5 * time.Minute}, Repo: nixpkgs, - JSONDepth: 1, + Manpages: Manpages{ + Enable: false, + Path: "", + }, + JSONDepth: 1, }, "darwin": { Name: "Darwin",@@ -87,6 +91,10 @@ Type: GitHub, Owner: "LnL7", Repo: "nix-darwin", }, + Manpages: Manpages{ + Enable: false, + Path: "", + }, JSONDepth: 1, }, "home-manager": {@@ -107,6 +115,10 @@ Type: GitHub, Owner: "nix-community", Repo: "home-manager", }, + Manpages: Manpages{ + Enable: false, + Path: "", + }, JSONDepth: 1, }, "nixpkgs": {@@ -124,6 +136,10 @@ Programs: ProgramsDB{ Enable: true, Attribute: "programs.sqlite", }, + Manpages: Manpages{ + Enable: true, + Path: "/doc/manpage-urls.json", + }, JSONDepth: 2, }, "nur": {@@ -139,6 +155,10 @@ Repo: Repository{ Type: GitHub, Owner: "nix-community", Repo: "nur", + }, + Manpages: Manpages{ + Enable: false, + Path: "", }, JSONDepth: 1, },
M internal/config/repository.go → internal/config/repository.go
@@ -22,6 +22,27 @@ Repo string Revision string `toml:"-"` } +func (r *Repository) GetRawFileURL(path string) (string, errors.E) { + switch r.Type { + case GitHub: + ref := r.Revision + if ref == "" { + ref = "master" + } + u, err := url.JoinPath("https://github.com/", r.Owner, r.Repo, "raw", ref, path) + if err != nil { + return "", errors.Wrap(err, "failed to join path") + } + + return u, nil + default: + return "", errors.Errorf( + "don't know how to generate a repository URL for %s", + r.Type.String(), + ) + } +} + func (r *Repository) GetFileURL(path string, line ...string) (string, errors.E) { switch r.Type { case GitHub:
M internal/config/structs.go → internal/config/structs.go
@@ -51,12 +51,18 @@ Timeout Duration `comment:"Abort import if it takes longer than this."` OutputPath string `comment:"(Fetcher=channel) Path under ./result symlink to folder containing {options,packages}.json."` Repo Repository `comment:"Used to generate declaration/definition links"` Programs ProgramsDB `comment:"Used to enable searching for programs in multi-program packages"` + Manpages Manpages `comment:"Used to enable searching for manpages"` JSONDepth int `comment:"Depth at which packages/object object is to be found"` } type ProgramsDB struct { Enable bool `comment:"Enable searching for programs in multi-program packages"` Attribute string `comment:"Nix attribute name (i.e. nix-instantiate) that builds a programs.sqlite file"` +} + +type Manpages struct { + Enable bool `comment:"Enable searching for manpages"` + Path string `comment:"Path to the manpage-urls.json file from repository root"` } func (source *Source) String() string {
M internal/importer/main.go → internal/importer/main.go
@@ -14,6 +14,7 @@ "github.com/getsentry/sentry-go" "go.alanpearce.eu/searchix/internal/config" "go.alanpearce.eu/searchix/internal/fetcher" "go.alanpearce.eu/searchix/internal/index" + "go.alanpearce.eu/searchix/internal/manpages" "go.alanpearce.eu/searchix/internal/programs" "go.alanpearce.eu/x/log"@@ -24,6 +25,7 @@ type Options struct { LowMemory bool Logger *log.Logger WriteIndex *index.WriteIndex + Manpages *manpages.URLMap } var Job struct {@@ -154,6 +156,13 @@ if err != nil { return errors.WithMessagef(err, "failed to process source") } + if source.Manpages.Enable { + err = imp.manpages.Update(ctx, source) + if err != nil { + logger.Warn("manpages database update failed", "error", err) + } + } + if hadWarnings { logger.Warn("importer succeeded, but with warnings/errors") } else {@@ -169,9 +178,10 @@ } } type Importer struct { - config *config.Config - log *log.Logger - indexer *index.WriteIndex + config *config.Config + log *log.Logger + indexer *index.WriteIndex + manpages *manpages.URLMap } func (imp *Importer) Start(@@ -225,9 +235,10 @@ options *Options, ) (*Importer, errors.E) { return &Importer{ - config: cfg, - log: options.Logger, - indexer: options.WriteIndex, + config: cfg, + log: options.Logger, + indexer: options.WriteIndex, + manpages: options.Manpages, }, nil }
M internal/importer/main_test.go → internal/importer/main_test.go
@@ -6,6 +6,7 @@ "testing" "go.alanpearce.eu/searchix/internal/config" "go.alanpearce.eu/searchix/internal/index" + "go.alanpearce.eu/searchix/internal/manpages" "go.alanpearce.eu/x/log" )@@ -27,6 +28,7 @@ imp, err := New(&cfg, &Options{ Logger: logger.Named("importer"), LowMemory: true, WriteIndex: write, + Manpages: manpages.New(&cfg, logger.Named("manpages")), }) if err != nil { b.Fatal(err)
M internal/index/indexer.go → internal/index/indexer.go
@@ -210,6 +210,7 @@ metaBaseName, indexBaseName, "sources", "nixpkgs-programs.db", + "manpage-urls.json", } func deleteIndex(dataRoot string) errors.E {
A internal/manpages/manpages.go
@@ -0,0 +1,129 @@ +package manpages + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "gitlab.com/tozd/go/errors" + "go.alanpearce.eu/searchix/internal/config" + "go.alanpearce.eu/searchix/internal/fetcher/http" + "go.alanpearce.eu/x/log" +) + +const basename = "manpage-urls.json" + +type URLMap struct { + path string + + mtime time.Time + logger *log.Logger + urlMap map[string]string +} + +func New(cfg *config.Config, logger *log.Logger) *URLMap { + return &URLMap{ + path: filepath.Join(cfg.DataPath, basename), + logger: logger, + } +} + +func (m *URLMap) Update( + ctx context.Context, + source *config.Source, +) errors.E { + if !source.Manpages.Enable { + return errors.New("manpages not enabled for this source") + } + + if source.Manpages.Path == "" { + return errors.New("manpages repo source path not configured") + } + + url, err := makeManpageURL(source) + if err != nil { + return errors.WithMessage(err, "failed to join manpages URL") + } + + m.logger.Debug("fetching manpages URL map", "url", url) + r, mtime, err := http.FetchFileIfNeeded(ctx, m.logger.Named("http"), m.mtime, url) + if err != nil { + return errors.WithMessage(err, "failed to fetch manpages") + } + defer r.Close() + + if err := m.save(r); err != nil { + return errors.WithMessage(err, "failed to save manpages") + } + + m.mtime = mtime + + return nil +} + +// Open loads the manpage URLs from the JSON file +func (m *URLMap) Open() errors.E { + m.logger.Debug("opening manpages file", "path", m.path) + + stat, err := os.Stat(m.path) + if err != nil { + return errors.WithMessagef(err, "failed to stat manpages file: %s", m.path) + } + + data, err := os.ReadFile(m.path) + if err != nil { + return errors.WithMessage(err, "failed to read manpages file") + } + + m.mtime = stat.ModTime() + + m.urlMap = make(map[string]string) + if err := json.Unmarshal(data, &m.urlMap); err != nil { + return errors.WithMessage(err, "failed to parse manpages JSON") + } + + m.logger.Debug("loaded manpages data", "urls", len(m.urlMap)) + + return nil +} + +func (m *URLMap) save(r io.Reader) errors.E { + m.logger.Debug("saving manpages file", "path", m.path) + + f, err := os.Create(m.path) + if err != nil { + return errors.WithMessage(err, "failed to create manpages file") + } + defer f.Close() + + if _, err := io.Copy(f, r); err != nil { + return errors.WithMessage(err, "failed to write manpages file") + } + + return nil +} + +func (m *URLMap) Get(section string, page string) (string, bool) { + key := fmt.Sprintf("%s(%s)", page, section) + + m.logger.Debug("getting manpage URL", "key", key) + url, ok := m.urlMap[key] + if !ok { + return "", false + } + + return url, true +} + +func makeManpageURL(source *config.Source) (string, errors.E) { + url, err := source.Repo.GetRawFileURL(source.Manpages.Path) + if err != nil { + return "", errors.WithMessage(err, "failed to join manpage URL") + } + + return url, nil +}
M internal/server/mux.go → internal/server/mux.go
@@ -60,16 +60,17 @@ } func NewMux( cfg *config.Config, - index *search.ReadIndex, + options *Options, log *log.Logger, liveReload bool, ) (*http.ServeMux, errors.E) { if cfg == nil { return nil, errors.New("cfg is nil") } - if index == nil { - return nil, errors.New("index is nil") + if options.ReadIndex == nil { + return nil, errors.New("read index is nil") } + index := options.ReadIndex sortSources(cfg.Importer.Sources) errorHandler := createErrorHandler(cfg, log)@@ -357,6 +358,25 @@ // optimisation for HTTP/3: first header sent as byte(41), not the string w.Header().Add("Cache-Control", "public, max-age=86400") w.Header().Add("Cache-Control", "stale-while-revalidate") http.ServeFileFS(w, r, frontend.Files, asset.Filename) + }) + + mdb := options.ManpagesURLMap + err := mdb.Open() + if err != nil { + return nil, errors.WithMessage(err, "failed to open manpages URL map") + } + mux.HandleFunc("/man/{section}/{page}", func(w http.ResponseWriter, r *http.Request) { + section := r.PathValue("section") + page := r.PathValue("page") + + url, ok := mdb.Get(section, page) + if !ok { + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + + return + } + + http.Redirect(w, r, url, http.StatusTemporaryRedirect) }) if liveReload {
M internal/server/server.go → internal/server/server.go
@@ -9,6 +9,7 @@ "time" "go.alanpearce.eu/searchix/internal/config" "go.alanpearce.eu/searchix/internal/index" + "go.alanpearce.eu/searchix/internal/manpages" "go.alanpearce.eu/x/log" "gitlab.com/tozd/go/errors"@@ -25,13 +26,18 @@ server *http.Server listener net.Listener } +type Options struct { + ReadIndex *index.ReadIndex + ManpagesURLMap *manpages.URLMap +} + func New( conf *config.Config, - index *index.ReadIndex, + options *Options, log *log.Logger, liveReload bool, ) (*Server, errors.E) { - mux, err := NewMux(conf, index, log, liveReload) + mux, err := NewMux(conf, options, log, liveReload) if err != nil { return nil, err }
M web/searchix.go → web/searchix.go
@@ -4,7 +4,6 @@ import ( "time" "go.alanpearce.eu/searchix/internal/config" - "go.alanpearce.eu/searchix/internal/index" "go.alanpearce.eu/searchix/internal/server" "go.alanpearce.eu/x/log"@@ -17,10 +16,10 @@ sv *server.Server cfg *config.Config log *log.Logger sentryHub *sentry.Hub - readIndex *index.ReadIndex + options *server.Options } -func New(cfg *config.Config, log *log.Logger, read *index.ReadIndex) (*Server, errors.E) { +func New(cfg *config.Config, log *log.Logger, options *server.Options) (*Server, errors.E) { err := sentry.Init(sentry.ClientOptions{ EnableTracing: true, TracesSampleRate: 1.0,@@ -35,13 +34,13 @@ return &Server{ cfg: cfg, log: log, sentryHub: sentry.CurrentHub(), - readIndex: read, + options: options, }, nil } func (s *Server) Start(liveReload bool) errors.E { var err errors.E - s.sv, err = server.New(s.cfg, s.readIndex, s.log.Named("server"), liveReload) + s.sv, err = server.New(s.cfg, s.options, s.log.Named("server"), liveReload) if err != nil { return errors.Wrap(err, "error setting up server") }