feat: limit file operations using os.Root
14 files changed, 262 insertions(+), 99 deletions(-)
changed files
- .golangci.yaml
- cmd/searchix-web/main.go
- frontend/dev.go
- internal/config/config.go
- internal/fetcher/channel.go
- internal/file/root.go
- internal/file/utils.go
- internal/importer/main_test.go
- internal/index/index_meta.go
- internal/index/indexer.go
- internal/index/search_test.go
- internal/manpages/manpages.go
- internal/server/dev.go
- internal/server/mux.go
M .golangci.yaml → .golangci.yaml
@@ -1,9 +1,10 @@ --- -# yamllint disable-line rule:line-length +# yamllint disable rule:line-length # yaml-language-server: $schema=https://golangci-lint.run/jsonschema/golangci.jsonschema.json version: "2" linters: enable: + - forbidigo - gocritic - godox - gosec@@ -21,6 +22,21 @@ - sloglint - unconvert - wrapcheck settings: + forbidigo: + forbid: + - pattern: ^(file)?path.Join$ + msg: "should use os.Root methods instead of path functions" + - pattern: ^filepath\.(Join|Walkdir)$ + msg: "should use os.Root methods instead of filepath functions" + - pattern: ^os\.(Open(File)?|Create|(New|Read|Write)File|Mkdir(All|Temp)?|Chmod|Chown|Chtimes|Link|Lstat|ReadDir|Remove(All)?|Rename|Stat|Symlink|Truncate|CopyFS|DirFS)$ + msg: "should use os.Root methods instead of os.File functions" + - pattern: ^file.Root.JoinPath$ + pkg: internal/file + msg: "should use os.Root methods instead of working with paths" + - pattern: ^os\.Root\.Name$ + msg: "should use or implement methods on file.Root" + exclude-godoc-examples: true + analyze-types: true gosec: excludes: - G115
M cmd/searchix-web/main.go → cmd/searchix-web/main.go
@@ -16,6 +16,7 @@ "alin.ovh/searchix/frontend" "alin.ovh/searchix/internal/components" "alin.ovh/searchix/internal/config" + "alin.ovh/searchix/internal/file" "alin.ovh/searchix/internal/importer" "alin.ovh/searchix/internal/index" "alin.ovh/searchix/internal/manpages"@@ -76,6 +77,7 @@ os.Exit(0) } if *cpuprofile != "" { + //nolint:forbidigo // admin specifies profile file location f, err := os.Create(*cpuprofile) if err != nil { panic("can't create CPU profile: " + err.Error())@@ -98,9 +100,15 @@ log.SetLevel(cfg.LogLevel) ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) defer cancel() + + root, err := file.CreateAndOpenRoot(cfg.DataPath) + if err != nil { + logger.Fatal("Failed to open data root", "error", err) + } + defer root.Close() read, write, exists, err := index.OpenOrCreate( - cfg.DataPath, + root, *replace, &index.Options{ LowMemory: cfg.Importer.LowMemory,@@ -112,7 +120,10 @@ if err != nil { logger.Fatal("Failed to open or create index", "error", err) } - mdb := manpages.New(cfg, logger.Named("manpages")) + mdb := manpages.New(&manpages.Options{ + Logger: logger.Named("manpages"), + Root: root, + }) s, err := web.New(cfg, logger, &server.Options{ ReadIndex: read,
M frontend/dev.go → frontend/dev.go
@@ -3,7 +3,26 @@ package frontend import ( + "io/fs" "os" + "path/filepath" + + "alin.ovh/searchix/internal/file" ) -var Files = os.DirFS("frontend/") +var Files fs.FS + +func init() { + wd, err := os.Getwd() + if err != nil { + panic(err) + } + + //nolint:forbidigo // own source code + root, err := file.OpenRoot(filepath.Join(wd, "frontend")) + if err != nil { + panic(err) + } + + Files = root.FS() +}
M internal/config/config.go → internal/config/config.go
@@ -111,6 +111,7 @@ func GetConfig(filename string, log *log.Logger) (*Config, error) { config := DefaultConfig if filename != "" { log.Debug("reading config", "filename", filename) + //nolint:forbidigo // need to read config file from anywhere f, err := os.Open(filename) if err != nil { return nil, fault.Wrap(err, fmsg.With("reading config failed"))
M internal/fetcher/channel.go → internal/fetcher/channel.go
@@ -63,6 +63,7 @@ if err != nil { return nil, fault.Wrap(err, fmsg.With("failed to run nix-build (--dry-run)")) } + //nolint:forbidigo // nix-build only gives the top-level path outPath := path.Join(strings.TrimSpace(string(out)), i.Source.OutputPath, "options.json") i.Logger.Debug( "checking output path",@@ -75,6 +76,7 @@ sourceMeta.Path = outPath sourceMeta.Updated = time.Now().Truncate(time.Second) } + //nolint:forbidigo // nix builds the file in the nix store file, err := os.Open(outPath) if err != nil { return nil, fault.Wrap(err, fmsg.With("failed to open options.json"))
A internal/file/root.go
@@ -0,0 +1,99 @@ +//nolint:forbidigo // wrappers for os.File functions go here +package file + +import ( + "io/fs" + "os" + "path/filepath" + + "github.com/Southclaws/fault" + "github.com/Southclaws/fault/fmsg" +) + +type Root struct { + *os.Root +} + +func CreateAndOpenRoot(name string) (*Root, error) { + exists, err := exists(name) + if err != nil { + return nil, fault.Wrap(err, fmsg.Withf("failed to check data root existence %s", name)) + } + if !exists { + err := mkdirp(name) + if err != nil { + return nil, fault.Wrap(err, fmsg.Withf("failed to create data root %s", name)) + } + } + + return OpenRoot(name) +} + +func OpenRoot(name string) (*Root, error) { + if !filepath.IsAbs(name) { + wd, err := os.Getwd() + if err != nil { + return nil, fault.Wrap(err, fmsg.With("failed to get current working directory")) + } + + name = filepath.Join(wd, name) + } + + r, err := os.OpenRoot(name) + if err != nil { + return nil, fault.Wrap(err, fmsg.Withf("failed to open data root directory %s", name)) + } + + return &Root{ + Root: r, + }, nil +} + +func (r *Root) JoinPath(path string) string { + return filepath.Join(r.Name(), path) +} + +func (r *Root) StatIfExists(name string) (fs.FileInfo, error) { + stat, err := r.Stat(name) + + return stat, needNotExist(err) +} + +func (r *Root) Exists(file string) (bool, error) { + stat, err := r.StatIfExists(file) + + return stat != nil, err +} + +func (r *Root) ReadFile(name string) ([]byte, error) { + b, err := fs.ReadFile(r.FS(), name) + if err != nil { + return nil, fault.Wrap(err, fmsg.Withf("failed to read file %s", name)) + } + + return b, nil +} + +func (r *Root) WriteFile(name string, data []byte, perm fs.FileMode) error { + f, err := r.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil { + return fault.Wrap(err, fmsg.Withf("failed to write file %s", name)) + } + defer f.Close() + + _, err = f.Write(data) + if err != nil { + return fault.Wrap(err, fmsg.Withf("failed to write file %s", name)) + } + + return nil +} + +func (r *Root) RemoveAll() error { + err := os.RemoveAll(r.Name()) + if err != nil { + return fault.Wrap(err, fmsg.Withf("failed to remove data root %s", r.Name())) + } + + return nil +}
M internal/file/utils.go → internal/file/utils.go
@@ -1,8 +1,8 @@ +//nolint:forbidigo // wrappers for os.File functions go here package file import ( "errors" - "io" "io/fs" "os"@@ -10,7 +10,7 @@ "github.com/Southclaws/fault" "github.com/Southclaws/fault/fmsg" ) -func Mkdirp(dir string) error { +func mkdirp(dir string) error { err := os.MkdirAll(dir, os.ModeDir|os.ModePerm) if err != nil { return fault.Wrap(err, fmsg.Withf("could not create directory %s", dir))@@ -19,7 +19,7 @@ return nil } -func NeedNotExist(err error) error { +func needNotExist(err error) error { if err != nil && !errors.Is(err, fs.ErrNotExist) { return fault.Wrap(err) }@@ -27,29 +27,14 @@ return nil } -func StatIfExists(file string) (fs.FileInfo, error) { +func statIfExists(file string) (fs.FileInfo, error) { stat, err := os.Stat(file) - return stat, NeedNotExist(err) + return stat, needNotExist(err) } -func Exists(file string) (bool, error) { - stat, err := StatIfExists(file) +func exists(file string) (bool, error) { + stat, err := statIfExists(file) return stat != nil, err } - -func WriteToFile(path string, body io.Reader) error { - file, err := os.Create(path) - if err != nil { - return fault.Wrap(err, fmsg.Withf("error creating file at %s", path)) - } - defer file.Close() - - _, err = io.Copy(file, body) - if err != nil { - return fault.Wrap(err, fmsg.Withf("error downloading file %s", path)) - } - - return nil -}
M internal/importer/main_test.go → internal/importer/main_test.go
@@ -5,6 +5,7 @@ "context" "testing" "alin.ovh/searchix/internal/config" + "alin.ovh/searchix/internal/file" "alin.ovh/searchix/internal/index" "alin.ovh/searchix/internal/manpages" "alin.ovh/x/log"@@ -13,7 +14,11 @@ var cfg = config.DefaultConfig func BenchmarkImporterLowMemory(b *testing.B) { - tmp := b.TempDir() + tmp, err := file.OpenRoot(b.TempDir()) + if err != nil { + b.Fatal(err) + } + logger := log.Configure(false) _, write, _, err := index.OpenOrCreate(tmp, false, &index.Options{ LowMemory: true,@@ -28,7 +33,10 @@ imp, err := New(&cfg, &Options{ Logger: logger.Named("importer"), LowMemory: true, WriteIndex: write, - Manpages: manpages.New(&cfg, logger.Named("manpages")), + Manpages: manpages.New(&manpages.Options{ + Logger: logger.Named("manpages"), + Root: tmp, + }), }) if err != nil { b.Fatal(err)
M internal/index/index_meta.go → internal/index/index_meta.go
@@ -2,7 +2,6 @@ package index import ( "encoding/json" - "os" "time" "alin.ovh/searchix/internal/file"@@ -26,13 +25,13 @@ Sources map[string]*SourceMeta } type Meta struct { - path string + root *file.Root log *log.Logger data } -func createMeta(path string, log *log.Logger) (*Meta, error) { - exists, err := file.Exists(path) +func createMeta(root *file.Root, log *log.Logger) (*Meta, error) { + exists, err := root.Exists(metaBaseName) if err != nil { return nil, fault.Wrap(err, fmsg.With("could not check for existence of index metadata")) }@@ -41,7 +40,7 @@ return nil, fault.New("index metadata already exists") } return &Meta{ - path: path, + root: root, log: log, data: data{ SchemaVersion: CurrentSchemaVersion,@@ -49,21 +48,21 @@ }, }, nil } -func openMeta(path string, log *log.Logger) (*Meta, error) { - exists, err := file.Exists(path) +func openMeta(root *file.Root, log *log.Logger) (*Meta, error) { + exists, err := root.Exists(metaBaseName) if err != nil { return nil, fault.Wrap(err, fmsg.With("could not check for existence of index metadata")) } if !exists { - return createMeta(path, log) + return createMeta(root, log) } - j, baseErr := os.ReadFile(path) + j, baseErr := root.ReadFile(metaBaseName) if baseErr != nil { return nil, fault.Wrap(baseErr, fmsg.With("could not open index metadata file")) } meta := Meta{ - path: path, + root: root, log: log, }@@ -84,8 +83,8 @@ j, err := json.Marshal(i.data) if err != nil { return fault.Wrap(err, fmsg.With("could not prepare index metadata for saving")) } - i.log.Debug("saving index metadata", "path", i.path) - err = os.WriteFile(i.path, j, 0o600) + i.log.Debug("saving index metadata", "path", metaBaseName) + err = i.root.WriteFile(metaBaseName, j, 0o600) if err != nil { return fault.Wrap(err, fmsg.With("could not save index metadata")) }
M internal/index/indexer.go → internal/index/indexer.go
@@ -6,9 +6,6 @@ "context" "encoding/gob" "io/fs" "math" - "os" - "path" - "path/filepath" "slices" "alin.ovh/searchix/internal/config"@@ -176,7 +173,7 @@ return indexMapping, nil } -func createIndex(indexPath string, options *Options) (bleve.Index, error) { +func createIndex(root *file.Root, options *Options) (bleve.Index, error) { indexMapping, err := createIndexMapping() if err != nil { return nil, err@@ -188,6 +185,9 @@ "PersisterNapTimeMSec": 1000, "PersisterNapUnderNumFiles": 500, } } + + //nolint:forbidigo // external package + indexPath := root.JoinPath(indexBaseName) idx, baseErr := bleve.NewUsing( indexPath, indexMapping,@@ -215,83 +215,72 @@ "nixpkgs-programs.db", "manpage-urls.json", } -func deleteIndex(dataRoot string) error { - dir, err := os.ReadDir(dataRoot) +func deleteIndex(root *file.Root) error { + dir, err := fs.ReadDir(root.FS(), ".") if err != nil { - return fault.Wrap(err, fmsg.Withf("could not read data directory %s", dataRoot)) + return fault.Wrap(err, fmsg.Withf("could not read data directory")) } remainingFiles := slices.DeleteFunc(dir, func(e fs.DirEntry) bool { return slices.Contains(expectedDataFiles, e.Name()) }) if len(remainingFiles) > 0 { return fault.Newf( - "cowardly refusing to remove data directory %s as it contains unknown files: %v", - dataRoot, + "cowardly refusing to remove data directory as it contains unknown files: %v", remainingFiles, ) } - err = os.RemoveAll(dataRoot) + err = root.RemoveAll() if err != nil { - return fault.Wrap(err, fmsg.Withf("could not remove data directory %s", dataRoot)) + return fault.Wrap(err) } return nil } func OpenOrCreate( - dataRoot string, + root *file.Root, force bool, options *Options, ) (*ReadIndex, *WriteIndex, bool, error) { var err error bleve.SetLog(zap.NewStdLog(options.Logger.Named("bleve").GetLogger())) - if !filepath.IsAbs(dataRoot) { - wd, err := os.Getwd() - if err != nil { - return nil, nil, false, fault.Wrap(err, fmsg.Withf("could not get working directory")) - } - dataRoot = filepath.Join(wd, dataRoot) - } - indexPath := path.Join(dataRoot, indexBaseName) - metaPath := path.Join(dataRoot, metaBaseName) - - exists, err := file.Exists(indexPath) + exists, err := root.Exists(indexBaseName) if err != nil { return nil, nil, exists, fault.Wrap( - err, fmsg.Withf("could not check if index exists at path %s", - indexPath, - )) + err, fmsg.Withf("could not check if index exists at path %s", indexBaseName)) } var idx bleve.Index var meta *Meta if !exists || force { if force { - err = deleteIndex(dataRoot) + err = deleteIndex(root) if err != nil { return nil, nil, false, err } } - idx, err = createIndex(indexPath, options) + idx, err = createIndex(root, options) if err != nil { return nil, nil, false, err } - meta, err = createMeta(metaPath, options.Logger) + meta, err = createMeta(root, options.Logger) if err != nil { return nil, nil, false, err } } else { var baseErr error + //nolint:forbidigo // external package + indexPath := root.JoinPath(indexBaseName) idx, baseErr = bleve.Open(indexPath) if baseErr != nil { return nil, nil, exists, fault.Wrap(baseErr, fmsg.Withf("could not open index at path %s", indexPath)) } - meta, err = openMeta(metaPath, options.Logger) + meta, err = openMeta(root, options.Logger) if err != nil { return nil, nil, exists, err }
M internal/index/search_test.go → internal/index/search_test.go
@@ -9,18 +9,25 @@ "testing" "time" "alin.ovh/searchix/internal/config" + "alin.ovh/searchix/internal/file" "alin.ovh/searchix/internal/index" "alin.ovh/searchix/internal/nix" "alin.ovh/x/log" ) -const dataRoot = "../../data" +const rootPath = "../../data" func TestSearchGitPackagesFirst(t *testing.T) { log := log.Configure(false) cfg := config.DefaultConfig - read, _, exists, err := index.OpenOrCreate(dataRoot, false, &index.Options{ + root, err := file.CreateAndOpenRoot(rootPath) + if err != nil { + t.Fatal(err) + } + defer root.Close() + + read, _, exists, err := index.OpenOrCreate(root, false, &index.Options{ Logger: log.Named("index"), BatchSize: cfg.Importer.BatchSize, LowMemory: false,@@ -84,7 +91,13 @@ func TestSearchJujutsuPackagesFirst(t *testing.T) { log := log.Configure(false) cfg := config.DefaultConfig - read, _, exists, err := index.OpenOrCreate(dataRoot, false, &index.Options{ + root, err := file.CreateAndOpenRoot(rootPath) + if err != nil { + t.Fatal(err) + } + defer root.Close() + + read, _, exists, err := index.OpenOrCreate(root, false, &index.Options{ Logger: log.Named("index"), LowMemory: false, })
M internal/manpages/manpages.go → internal/manpages/manpages.go
@@ -5,12 +5,11 @@ "context" "encoding/json" "fmt" "io" - "os" - "path/filepath" "time" "alin.ovh/searchix/internal/config" "alin.ovh/searchix/internal/fetcher/http" + "alin.ovh/searchix/internal/file" "alin.ovh/x/log" "github.com/Southclaws/fault" "github.com/Southclaws/fault/fmsg"@@ -19,17 +18,22 @@ const basename = "manpage-urls.json" type URLMap struct { - path string + root *file.Root mtime time.Time logger *log.Logger urlMap map[string]string } -func New(cfg *config.Config, logger *log.Logger) *URLMap { +type Options struct { + Logger *log.Logger + Root *file.Root +} + +func New(opts *Options) *URLMap { return &URLMap{ - path: filepath.Join(cfg.DataPath, basename), - logger: logger, + logger: opts.Logger, + root: opts.Root, } }@@ -68,14 +72,14 @@ } // Open loads the manpage URLs from the JSON file func (m *URLMap) Open() error { - m.logger.Debug("opening manpages file", "path", m.path) + m.logger.Debug("opening manpages file", "path", basename) - stat, err := os.Stat(m.path) + stat, err := m.root.Stat(basename) if err != nil { - return fault.Wrap(err, fmsg.Withf("failed to stat manpages file: %s", m.path)) + return fault.Wrap(err, fmsg.Withf("failed to stat manpages file: %s", basename)) } - data, err := os.ReadFile(m.path) + data, err := m.root.ReadFile(basename) if err != nil { return fault.Wrap(err, fmsg.With("failed to read manpages file")) }@@ -93,9 +97,9 @@ return nil } func (m *URLMap) save(r io.Reader) error { - m.logger.Debug("saving manpages file", "path", m.path) + m.logger.Debug("saving manpages file", "path", basename) - f, err := os.Create(m.path) + f, err := m.root.Create(basename) if err != nil { return fault.Wrap(err, fmsg.With("failed to create manpages file")) }
M internal/server/dev.go → internal/server/dev.go
@@ -3,10 +3,9 @@ import ( "fmt" "io/fs" - "os" - "path/filepath" "time" + "alin.ovh/searchix/internal/file" "alin.ovh/x/log" "github.com/Southclaws/fault" "github.com/Southclaws/fault/fmsg"@@ -15,31 +14,49 @@ ) type FileWatcher struct { watcher *fsnotify.Watcher + root *file.Root + rootDir string log *log.Logger } -func NewFileWatcher(log *log.Logger) (*FileWatcher, error) { +func NewFileWatcher(log *log.Logger, rootDir string) (*FileWatcher, error) { watcher, err := fsnotify.NewWatcher() if err != nil { return nil, fault.Wrap(err, fmsg.With("could not create watcher")) } + root, err := file.OpenRoot(rootDir) + if err != nil { + return nil, fault.Wrap(err, fmsg.With("could not open root directory")) + } + return &FileWatcher{ - watcher, - log, + watcher: watcher, + root: root, + rootDir: rootDir, + log: log, }, nil } +func (i FileWatcher) Add(path string) error { + //nolint:forbidigo // external package + fullpath := i.root.JoinPath(path) + i.log.Debug(fmt.Sprintf("adding file %s to watcher", fullpath)) + if err := i.watcher.Add(fullpath); err != nil { + return fault.Wrap(err, fmsg.Withf("could not add file %s to watcher", path)) + } + + return nil +} + func (i FileWatcher) AddRecursive(from string) error { - i.log.Debug(fmt.Sprintf("watching files under %s", from)) - err := filepath.WalkDir(from, func(path string, entry fs.DirEntry, err error) error { + err := fs.WalkDir(i.root.FS(), from, func(path string, entry fs.DirEntry, err error) error { if err != nil { return fault.Wrap(err, fmsg.Withf("could not walk directory %s", path)) } if entry.IsDir() { - i.log.Debug(fmt.Sprintf("adding directory %s to watcher", path)) - if err = i.watcher.Add(path); err != nil { - return fault.Wrap(err, fmsg.Withf("could not add directory %s to watcher", path)) + if err = i.Add(path); err != nil { + return err } }@@ -54,7 +71,7 @@ for { select { case event := <-i.watcher.Events: if event.Has(fsnotify.Create) || event.Has(fsnotify.Rename) { - f, err := os.Stat(event.Name) + f, err := i.root.Stat(event.Name) if err != nil { i.log.Error(fmt.Sprintf("error handling %s event: %v", event.Op.String(), err)) } else if f.IsDir() {
M internal/server/mux.go → internal/server/mux.go
@@ -387,17 +387,17 @@ cfg.Web.ExtraHeadHTML = livereload.JsSnippet liveReload := livereload.New() liveReload.Start() top.Handle("/livereload", liveReload) - fw, err := NewFileWatcher(log.Named("watcher")) + fw, err := NewFileWatcher(log.Named("watcher"), "frontend") if err != nil { return nil, fault.Wrap(err, fmsg.With("could not create file watcher")) } - err = fw.AddRecursive(path.Join("frontend")) + err = fw.AddRecursive(".") if err != nil { return nil, fault.Wrap(err, fmsg.With("could not add directory to file watcher")) } go fw.Start(func(filename string) { log.Debug(fmt.Sprintf("got filename %s", filename)) - if match, _ := path.Match("frontend/static/*", filename); match { + if match, _ := path.Match("static/*", filename); match { err := assets.Rehash() if err != nil { log.Error("failed to re-hash frontend assets", "error", err)