all repos — homestead @ f0db45107699eb1294f6d64cbf2ddd48783f7cc0

Code for my website

fetcher: support fetching archives

Alan Pearce
commit

f0db45107699eb1294f6d64cbf2ddd48783f7cc0

parent

c4502da96d8899b770a621ce07b0a636a84f6f53

3 files changed, 128 insertions(+), 13 deletions(-)

changed files
M domain/content/fetcher/fetcher.godomain/content/fetcher/fetcher.go
@@ -28,6 +28,7 @@ options *Options
log *log.Logger updater events.Listener current uint64 + root *os.Root } type Options struct {
@@ -38,21 +39,28 @@ Listener events.Listener
} var ( - files = []string{"config.toml", "site.db"} - numericFilename = regexp.MustCompile("[0-9]{3,}") + files = []string{"config.toml"} + archive = "site.tar.bz2" + numericFilename = regexp.MustCompile("[0-9]+") timeout = 10 * time.Second ) -func New(log *log.Logger, options *Options) Fetcher { - return Fetcher{ +func New(log *log.Logger, options *Options) (*Fetcher, error) { + root, err := os.OpenRoot(options.Root) + if err != nil { + return nil, fault.Wrap(err, fmsg.With("could not open root")) + } + + return &Fetcher{ log: log, options: options, updater: options.Listener, - } + root: root, + }, nil } func (f *Fetcher) CleanOldRevisions() error { - contents, err := os.ReadDir(f.options.Root) + contents, err := f.root.FS().(fs.ReadDirFS).ReadDir(".") if err != nil { return fault.Wrap(err, fmsg.With("could not read root directory")) }
@@ -136,7 +144,7 @@ func (f *Fetcher) getArtefacts(run uint64) error {
runID := strconv.FormatUint(run, 10) f.log.Debug("getting artefacts", "run_id", runID) - err := os.MkdirAll(filepath.Join(f.options.Root, runID), 0o750) + err := f.root.MkdirAll(runID, 0o750) if err != nil { return fault.Wrap(err, fmsg.With("could not create directory")) }
@@ -146,10 +154,15 @@ err := f.getFile(runID, file)
if err != nil { return fault.Wrap(err, fmsg.With("could not fetch file")) } + } + + err = f.getArchive(runID, archive) + if err != nil { + return fault.Wrap(err, fmsg.With("could not fetch archive")) } f.current = run - err = renameio.Symlink(runID, filepath.Join(f.options.Root, "current")) + err = renameio.Symlink(runID, filepath.Join(f.root.Name(), "current")) if err != nil { return fault.Wrap(err, fmsg.With("could not create/update symlink")) }
@@ -158,7 +171,7 @@ return nil
} func (f *Fetcher) checkFolder() error { - contents, err := os.ReadDir(f.options.Root) + contents, err := f.root.FS().(fs.ReadDirFS).ReadDir(".") if err != nil { return fault.Wrap(err, fmsg.With("could not read root directory")) }
@@ -187,13 +200,17 @@ }
}() } +func (f *Fetcher) makeURL(runID, basename string) string { + return f.options.FetchURL.JoinPath(runID, basename).String() +} + func (f *Fetcher) getFile(runID, basename string) error { - filename := filepath.Join(f.options.Root, runID, basename) - url := f.options.FetchURL.JoinPath(runID, basename).String() + filename := filepath.Join(runID, basename) + url := f.makeURL(runID, basename) f.log.Debug("getting file", "filename", filename, "url", url) - file, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0o600) + file, err := f.root.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0o600) if err != nil { return fault.Wrap(err, fmsg.With("could not open file")) }
@@ -223,6 +240,32 @@ return fault.Wrap(err, fmsg.With("could not sync file"))
} return nil +} + +func (f *Fetcher) getArchive(runID, basename string) error { + url := f.makeURL(runID, basename) + + f.log.Debug("getting file", "url", url) + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return fault.Wrap(err, fmsg.With("could not create request")) + } + res, err := http.DefaultClient.Do(req) + if err != nil { + return fault.Wrap(err, fmsg.With("could not issue request")) + } + defer res.Body.Close() + + subRoot, err := f.root.OpenRoot(runID) + if err != nil { + return fault.Wrap(err, fmsg.With("could not open root")) + } + + return extract(bunzip(res.Body), subRoot, f.log) } func (f *Fetcher) getCurrentVersion() (uint64, error) {
A domain/content/fetcher/tar.go
@@ -0,0 +1,68 @@
+package fetcher + +import ( + "archive/tar" + "compress/bzip2" + "io" + "os" + + "alin.ovh/x/log" + "github.com/Southclaws/fault" + "github.com/Southclaws/fault/fmsg" +) + +func bunzip(r io.Reader) io.Reader { + return bzip2.NewReader(r) +} + +func extract(body io.Reader, root *os.Root, log *log.Logger) error { + tr := tar.NewReader(body) + for { + head, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return fault.Wrap(err, fmsg.With("could not read tar header")) + } + + log.Debug("extracting file", "name", head.Name) + + switch head.Typeflag { + case tar.TypeReg: + err := writeFile(tr, head, root) + if err != nil { + return fault.Wrap(err, fmsg.Withf("could not write file %s", head.Name)) + } + case tar.TypeDir: + err := root.MkdirAll(head.Name, head.FileInfo().Mode().Perm()) + if err != nil { + return fault.Wrap(err, fmsg.Withf("could not create directory %s", head.Name)) + } + default: + return fault.Newf("unsupported file type %d", head.Typeflag) + } + } + + return nil +} + +func writeFile(tr *tar.Reader, head *tar.Header, root *os.Root) error { + file, err := root.OpenFile(head.Name, os.O_CREATE|os.O_WRONLY, head.FileInfo().Mode().Perm()) + if err != nil { + return fault.Wrap(err, fmsg.Withf("could not open file %s", head.Name)) + } + defer file.Close() + + _, err = io.Copy(file, tr) + if err != nil { + return fault.Wrap(err, fmsg.Withf("could not write file %s", head.Name)) + } + + err = file.Sync() + if err != nil { + return fault.Wrap(err, fmsg.Withf("could not sync file %s", head.Name)) + } + + return nil +}
M domain/web/website.godomain/web/website.go
@@ -89,12 +89,16 @@ return nil, fault.Wrap(err, fmsg.With("could not create update listener"))
} var cfg *config.Config - fetcher := fetcher.New(log.Named("fetcher"), &fetcher.Options{ + fetcher, err := fetcher.New(log.Named("fetcher"), &fetcher.Options{ FetchURL: opts.FetchURL, RedisEnabled: opts.Redis.Address != "", Root: opts.Root, Listener: listener, }) + if err != nil { + return nil, fault.Wrap(err, fmsg.With("could not create fetcher")) + } + roots, err := fetcher.Subscribe() if err != nil { return nil, fault.Wrap(err, fmsg.With("could not set up fetcher"))