internal/fetcher/http/http.go (view raw)
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 | package http
import (
"context"
"fmt"
"io"
"net/http"
"os"
"strings"
"time"
"alin.ovh/searchix/internal/config"
"alin.ovh/searchix/internal/file"
"alin.ovh/x/log"
"github.com/Southclaws/fault"
"github.com/Southclaws/fault/fmsg"
"github.com/andybalholm/brotli"
)
type brotliReadCloser struct {
src io.ReadCloser
*brotli.Reader
}
func newBrotliReader(src io.ReadCloser) *brotliReadCloser {
return &brotliReadCloser{
src: src,
Reader: brotli.NewReader(src),
}
}
func (r *brotliReadCloser) Close() error {
return fault.Wrap(r.src.Close(), fmsg.With("failed to call close on underlying reader"))
}
type Options struct {
Logger *log.Logger
Root *file.Root
}
type Fetcher struct {
logger *log.Logger
root *file.Root
}
func NewFetcher(options *Options) *Fetcher {
return &Fetcher{
logger: options.Logger,
root: options.Root,
}
}
func (h *Fetcher) FetchFileIfNeeded(
ctx context.Context,
filename string,
url string,
) (io.ReadCloser, error) {
stat, err := h.root.StatIfExists(filename)
if err != nil {
return nil, fault.Wrap(
err,
fmsg.Withf("failed to stat file %s", filename),
)
}
var ifModifiedSince string
if stat != nil && stat.Size() > 0 && !stat.ModTime().IsZero() {
ifModifiedSince = strings.Replace(
stat.ModTime().UTC().Format(time.RFC1123),
"UTC",
"GMT",
1,
)
}
req, err := http.NewRequestWithContext(ctx, "GET", url, http.NoBody)
if err != nil {
return nil, fault.Wrap(
err,
fmsg.Withf("could not create HTTP request for %s", url),
)
}
req.Header.Set("User-Agent", fmt.Sprintf("Searchix %s", config.Version))
if ifModifiedSince != "" {
req.Header.Set("If-Modified-Since", ifModifiedSince)
}
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, fault.Wrap(
err,
fmsg.Withf("could not make HTTP request to %s", url),
)
}
var body io.ReadCloser
var newMtime time.Time
encoding := res.Header.Get("Content-Encoding")
switch res.StatusCode {
case http.StatusNotModified:
case http.StatusOK:
var baseErr error
newMtime, baseErr = time.Parse(time.RFC1123, res.Header.Get("Last-Modified"))
if baseErr != nil {
h.logger.Warn(
"could not parse Last-Modified header from response",
"value",
res.Header.Get("Last-Modified"),
)
newMtime = time.Now()
}
switch encoding {
case "br":
body = newBrotliReader(res.Body)
case "", "identity", "gzip":
body = res.Body
default:
return nil, fault.Newf("cannot handle a body with content-encoding %s", encoding)
}
writer, err := h.root.OpenFile(filename, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0o644)
if err != nil {
return nil, fault.Wrap(err, fmsg.Withf("failed to open file %s", filename))
}
_, err = io.Copy(writer, body)
if err != nil {
return nil, fault.Wrap(
err,
fmsg.Withf("failed to copy response body to file %s", filename),
)
}
err = writer.Sync()
if err != nil {
return nil, fault.Wrap(err, fmsg.Withf("failed to sync file %s", filename))
}
err = writer.Close()
if err != nil {
return nil, fault.Wrap(err, fmsg.Withf("failed to close file %s", filename))
}
err = h.root.Chtimes(filename, time.Time{}, newMtime)
if err != nil {
return nil, fault.Wrap(err, fmsg.Withf("failed to update file times %s", filename))
}
default:
return nil, fault.Newf("got response code %d, don't know what to do", res.StatusCode)
}
reader, err := h.root.Open(filename)
if err != nil {
return nil, fault.Wrap(err, fmsg.Withf("failed to open file %s", filename))
}
return reader, nil
}
|