fetcher: support fetching archives
1 file changed, 55 insertions(+), 12 deletions(-)
changed files
M domain/content/fetcher/fetcher.go → domain/content/fetcher/fetcher.go
@@ -28,6 +28,7 @@ options *Options log *log.Logger updater events.Listener current uint64 + root *os.Root } type Options struct {@@ -38,21 +39,28 @@ Listener events.Listener } var ( - files = []string{"config.toml", "site.db"} - numericFilename = regexp.MustCompile("[0-9]{3,}") + files = []string{"config.toml"} + archive = "site.tar.bz2" + numericFilename = regexp.MustCompile("[0-9]+") timeout = 10 * time.Second ) -func New(log *log.Logger, options *Options) Fetcher { - return Fetcher{ +func New(log *log.Logger, options *Options) (*Fetcher, error) { + root, err := os.OpenRoot(options.Root) + if err != nil { + return nil, fault.Wrap(err, fmsg.With("could not open root")) + } + + return &Fetcher{ log: log, options: options, updater: options.Listener, - } + root: root, + }, nil } func (f *Fetcher) CleanOldRevisions() error { - contents, err := os.ReadDir(f.options.Root) + contents, err := f.root.FS().(fs.ReadDirFS).ReadDir(".") if err != nil { return fault.Wrap(err, fmsg.With("could not read root directory")) }@@ -136,7 +144,7 @@ func (f *Fetcher) getArtefacts(run uint64) error { runID := strconv.FormatUint(run, 10) f.log.Debug("getting artefacts", "run_id", runID) - err := os.MkdirAll(filepath.Join(f.options.Root, runID), 0o750) + err := f.root.MkdirAll(runID, 0o750) if err != nil { return fault.Wrap(err, fmsg.With("could not create directory")) }@@ -146,10 +154,15 @@ err := f.getFile(runID, file) if err != nil { return fault.Wrap(err, fmsg.With("could not fetch file")) } + } + + err = f.getArchive(runID, archive) + if err != nil { + return fault.Wrap(err, fmsg.With("could not fetch archive")) } f.current = run - err = renameio.Symlink(runID, filepath.Join(f.options.Root, "current")) + err = renameio.Symlink(runID, filepath.Join(f.root.Name(), "current")) if err != nil { return fault.Wrap(err, fmsg.With("could not create/update symlink")) }@@ -158,7 +171,7 @@ return nil } func (f *Fetcher) checkFolder() error { - contents, err := os.ReadDir(f.options.Root) + contents, err := f.root.FS().(fs.ReadDirFS).ReadDir(".") if err != nil { return fault.Wrap(err, fmsg.With("could not read root directory")) }@@ -187,13 +200,17 @@ } }() } +func (f *Fetcher) makeURL(runID, basename string) string { + return f.options.FetchURL.JoinPath(runID, basename).String() +} + func (f *Fetcher) getFile(runID, basename string) error { - filename := filepath.Join(f.options.Root, runID, basename) - url := f.options.FetchURL.JoinPath(runID, basename).String() + filename := filepath.Join(runID, basename) + url := f.makeURL(runID, basename) f.log.Debug("getting file", "filename", filename, "url", url) - file, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0o600) + file, err := f.root.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0o600) if err != nil { return fault.Wrap(err, fmsg.With("could not open file")) }@@ -223,6 +240,32 @@ return fault.Wrap(err, fmsg.With("could not sync file")) } return nil +} + +func (f *Fetcher) getArchive(runID, basename string) error { + url := f.makeURL(runID, basename) + + f.log.Debug("getting file", "url", url) + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return fault.Wrap(err, fmsg.With("could not create request")) + } + res, err := http.DefaultClient.Do(req) + if err != nil { + return fault.Wrap(err, fmsg.With("could not issue request")) + } + defer res.Body.Close() + + subRoot, err := f.root.OpenRoot(runID) + if err != nil { + return fault.Wrap(err, fmsg.With("could not open root")) + } + + return extract(bunzip(res.Body), subRoot, f.log) } func (f *Fetcher) getCurrentVersion() (uint64, error) {