all repos — searchix @ bdb5a54c661407c39668096074a1f4a57898eb77

Search engine for NixOS, nix-darwin, home-manager and NUR users

feat: store fetched files in data directory

Alan Pearce
commit

bdb5a54c661407c39668096074a1f4a57898eb77

parent

3e928369c3af69d0ef24a2f5d20c938689b15aa1

1 file changed, 85 insertions(+), 25 deletions(-)

changed files
M internal/fetcher/http/http.gointernal/fetcher/http/http.go
@@ -5,10 +5,12 @@ "context"
"fmt" "io" "net/http" + "os" "strings" "time" "alin.ovh/searchix/internal/config" + "alin.ovh/searchix/internal/file" "alin.ovh/x/log" "github.com/Southclaws/fault"
@@ -32,22 +34,49 @@ func (r *brotliReadCloser) Close() error {
return fault.Wrap(r.src.Close(), fmsg.With("failed to call close on underlying reader")) } -func FetchFileIfNeeded( +type Options struct { + Logger *log.Logger + Root *file.Root +} + +type Fetcher struct { + logger *log.Logger + root *file.Root +} + +func NewFetcher(options *Options) *Fetcher { + return &Fetcher{ + logger: options.Logger, + root: options.Root, + } +} + +func (h *Fetcher) FetchFileIfNeeded( ctx context.Context, - log *log.Logger, - mtime time.Time, + filename string, url string, -) (io.ReadCloser, time.Time, error) { - var newMtime time.Time +) (io.ReadCloser, error) { + stat, err := h.root.StatIfExists(filename) + if err != nil { + return nil, fault.Wrap( + err, + fmsg.Withf("failed to stat file %s", filename), + ) + } var ifModifiedSince string - if !mtime.IsZero() { - ifModifiedSince = strings.Replace(mtime.UTC().Format(time.RFC1123), "UTC", "GMT", 1) + if stat != nil && stat.Size() > 0 && !stat.ModTime().IsZero() { + ifModifiedSince = strings.Replace( + stat.ModTime().UTC().Format(time.RFC1123), + "UTC", + "GMT", + 1, + ) } - req, baseErr := http.NewRequestWithContext(ctx, "GET", url, http.NoBody) - if baseErr != nil { - return nil, newMtime, fault.Wrap( - baseErr, + req, err := http.NewRequestWithContext(ctx, "GET", url, http.NoBody) + if err != nil { + return nil, fault.Wrap( + err, fmsg.Withf("could not create HTTP request for %s", url), ) }
@@ -57,26 +86,24 @@
if ifModifiedSince != "" { req.Header.Set("If-Modified-Since", ifModifiedSince) } - res, baseErr := http.DefaultClient.Do(req) - if baseErr != nil { - return nil, newMtime, fault.Wrap( - baseErr, + res, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fault.Wrap( + err, fmsg.Withf("could not make HTTP request to %s", url), ) } var body io.ReadCloser - var err error + var newMtime time.Time + encoding := res.Header.Get("Content-Encoding") switch res.StatusCode { case http.StatusNotModified: - newMtime = mtime - - return nil, newMtime, nil case http.StatusOK: var baseErr error newMtime, baseErr = time.Parse(time.RFC1123, res.Header.Get("Last-Modified")) if baseErr != nil { - log.Warn( + h.logger.Warn( "could not parse Last-Modified header from response", "value", res.Header.Get("Last-Modified"),
@@ -84,18 +111,51 @@ )
newMtime = time.Now() } - switch ce := res.Header.Get("Content-Encoding"); ce { + switch encoding { case "br": - log.Debug("using brotli encoding") body = newBrotliReader(res.Body) case "", "identity", "gzip": body = res.Body default: - err = fault.Newf("cannot handle a body with content-encoding %s", ce) + return nil, fault.Newf("cannot handle a body with content-encoding %s", encoding) + } + + writer, err := h.root.OpenFile(filename, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0o644) + if err != nil { + return nil, fault.Wrap(err, fmsg.Withf("failed to open file %s", filename)) + } + + _, err = io.Copy(writer, body) + if err != nil { + return nil, fault.Wrap( + err, + fmsg.Withf("failed to copy response body to file %s", filename), + ) + } + + err = writer.Sync() + if err != nil { + return nil, fault.Wrap(err, fmsg.Withf("failed to sync file %s", filename)) + } + + err = writer.Close() + if err != nil { + return nil, fault.Wrap(err, fmsg.Withf("failed to close file %s", filename)) + } + + err = h.root.Chtimes(filename, time.Time{}, newMtime) + if err != nil { + return nil, fault.Wrap(err, fmsg.Withf("failed to update file times %s", filename)) } + default: - err = fault.Newf("got response code %d, don't know what to do", res.StatusCode) + return nil, fault.Newf("got response code %d, don't know what to do", res.StatusCode) + } + + reader, err := h.root.Open(filename) + if err != nil { + return nil, fault.Wrap(err, fmsg.Withf("failed to open file %s", filename)) } - return NewReadCloser(body), newMtime, err + return reader, nil }