all repos — searchix @ bdb5a54c661407c39668096074a1f4a57898eb77

Search engine for NixOS, nix-darwin, home-manager and NUR users

feat: store fetched files in data directory

Alan Pearce
commit

bdb5a54c661407c39668096074a1f4a57898eb77

parent

3e928369c3af69d0ef24a2f5d20c938689b15aa1

1 file changed, 17 insertions(+), 10 deletions(-)

changed files
M internal/fetcher/nixpkgs-channel.gointernal/fetcher/nixpkgs-channel.go
@@ -3,19 +3,19 @@
import ( "context" "net/url" + "strings" "alin.ovh/searchix/internal/config" "alin.ovh/searchix/internal/fetcher/http" "alin.ovh/searchix/internal/index" - "alin.ovh/x/log" "github.com/Southclaws/fault" "github.com/Southclaws/fault/fmsg" ) type NixpkgsChannelFetcher struct { Source *config.Source - Logger *log.Logger + *Options } func makeChannelURL(channel string, subPath string) (string, error) {
@@ -26,13 +26,13 @@ }
func NewNixpkgsChannelFetcher( source *config.Source, - logger *log.Logger, + options *Options, ) (*NixpkgsChannelFetcher, error) { switch source.Importer { case config.Options, config.Packages: return &NixpkgsChannelFetcher{ - Source: source, - Logger: logger, + Source: source, + Options: options, }, nil default: return nil, fault.Newf("unsupported importer type %s", source.Importer)
@@ -60,23 +60,30 @@ case config.Options:
filesToFetch[1] = optionsFilename } + fetcher := http.NewFetcher(&http.Options{ + Logger: i.Logger.Named("http"), + Root: i.Root, + }) + var fetchURL string for _, filename := range filesToFetch { + target := i.Source.JoinPath(strings.TrimSuffix(filename, ".br")) fetchURL, err = makeChannelURL(i.Source.Channel, filename) if err != nil { return } i.Logger.Debug("attempting to fetch file", "url", fetchURL) - body, mtime, err := http.FetchFileIfNeeded(ctx, i.Logger, sourceMeta.Updated, fetchURL) + body, err := fetcher.FetchFileIfNeeded(ctx, target, fetchURL) if err != nil { return f, fault.Wrap(err, fmsg.Withf("failed to fetch file with url %s", fetchURL)) } - // don't bother to issue requests for the later files - if mtime.Before(sourceMeta.Updated) { - break + + stat, err := i.Root.Stat(target) + if err != nil { + return nil, fault.Wrap(err, fmsg.Withf("could not stat file %s", target)) } - sourceMeta.Updated = mtime + sourceMeta.Updated = stat.ModTime() switch filename { case revisionFilename: