use redis notifications to update website
1 file changed, 82 insertions(+), 111 deletions(-)
changed files
M internal/website/website.go → internal/website/website.go
@@ -2,18 +2,21 @@ package website import ( "net/http" + "os" + "path/filepath" "slices" "strings" + "sync" "gitlab.com/tozd/go/errors" - "go.alanpearce.eu/homestead/internal/builder" "go.alanpearce.eu/homestead/internal/config" "go.alanpearce.eu/homestead/internal/events" + "go.alanpearce.eu/homestead/internal/fetcher" + "go.alanpearce.eu/homestead/internal/file" ihttp "go.alanpearce.eu/homestead/internal/http" "go.alanpearce.eu/homestead/internal/server" "go.alanpearce.eu/homestead/internal/storage" "go.alanpearce.eu/homestead/internal/storage/sqlite" - "go.alanpearce.eu/homestead/internal/vcs" "go.alanpearce.eu/homestead/templates" "go.alanpearce.eu/x/log"@@ -22,16 +25,14 @@ "github.com/osdevisnot/sorvor/pkg/livereload" ) type Options struct { - Source string `conf:"default:../website"` - DBPath string `conf:"default:site.db"` + DataRoot string `conf:"noprint"` + Root string `conf:"default:./website"` Redirect bool `conf:"default:true"` Development bool `conf:"default:false,flag:dev"` - BaseURL config.URL `conf:"default:localhost"` - VCS struct { - Branch string `conf:"default:main"` - RemoteURL config.URL `conf:"default:https://git.alanpearce.eu/website"` - } + FetchURL config.URL `conf:"default:https://ci.alanpearce.eu/archive/website/"` + BaseURL config.URL + Redis *events.RedisOptions LiveReload *livereload.LiveReload `conf:"-"` }@@ -54,34 +55,78 @@ App: &server.App{ Shutdown: func() {}, }, } - builderOptions := &builder.Options{ - Source: opts.Source, - Development: opts.Development, - } - repo, exists, err := vcs.CloneOrOpen(&vcs.Options{ - LocalPath: opts.Source, - RemoteURL: opts.VCS.RemoteURL, - Branch: opts.VCS.Branch, - }, log.Named("vcs")) + err := prepareRootDirectory(opts.DataRoot, opts.Root) if err != nil { - return nil, errors.WithMessage(err, "could not open repository") + return nil, errors.WithMessage(err, "could not prepare root directory") } - builderOptions.Repo = repo - if exists && !opts.Development { - _, err := repo.Update("") - if err != nil { - return nil, errors.WithMessage(err, "could not update repository") - } + var listener events.Listener + if opts.Redis.Enabled { + log.Debug("using redis listener") + listener, err = events.NewRedisListener(opts.Redis, log.Named("redis")) + } else { + log.Debug("using file watcher") + listener, err = events.NewFileWatcher(log.Named("events"), "website") + } + if err != nil { + return nil, errors.WithMessage(err, "could not create update listener") } - log.Debug("getting config from source", "source", opts.Source) - cfg, err := config.GetConfig(opts.Source, log) + var cfg *config.Config + fetcher := fetcher.New(log.Named("fetcher"), &fetcher.Options{ + FetchURL: opts.FetchURL, + RedisEnabled: false, + Root: opts.Root, + Listener: listener, + }) + roots, err := fetcher.Subscribe() if err != nil { - return nil, errors.WithMessage(err, "could not load configuration") + return nil, errors.WithMessage(err, "could not set up fetcher") } + firstUpdate := make(chan bool) + go func() { + updated := sync.OnceFunc(func() { + firstUpdate <- true + close(firstUpdate) + }) + for root := range roots { + log.Debug("getting config from source", "source", root) + cfg, err = config.GetConfig(root, log) + if err != nil { + log.Panic("could not load configuration", "error", err) + } + website.config = cfg + + if opts.Development { + cfg.CSP.ScriptSrc = slices.Insert(cfg.CSP.ScriptSrc, 0, "'unsafe-inline'") + cfg.CSP.ConnectSrc = slices.Insert(cfg.CSP.ConnectSrc, 0, "'self'") + } + + if opts.BaseURL.Hostname() != "" { + cfg.BaseURL = opts.BaseURL + } + + website.Domain = cfg.BaseURL.Hostname() + + siteDB := filepath.Join(root, "site.db") + db, err := sqlite.OpenDB(siteDB) + if err != nil { + log.Panic("could not open database", "error", err) + } + + website.reader, err = sqlite.NewReader(db, log.Named("reader")) + if err != nil { + log.Panic("could not create database reader", "error", err) + } + + updated() + } + }() + + <-firstUpdate + mux := ihttp.NewServeMux() mux.HandleError(func(err *ihttp.Error, w http.ResponseWriter, r *http.Request) { if strings.Contains(r.Header.Get("Accept"), "text/html") {@@ -95,84 +140,6 @@ http.Error(w, err.Message, err.Code) } }) - if opts.Development { - opts.DBPath = ":memory:" - - cfg.CSP.ScriptSrc = slices.Insert(cfg.CSP.ScriptSrc, 0, "'unsafe-inline'") - cfg.CSP.ConnectSrc = slices.Insert(cfg.CSP.ConnectSrc, 0, "'self'") - - cfg.BaseURL = opts.BaseURL - } - - db, err := sqlite.OpenDB(opts.DBPath) - if err != nil { - return nil, errors.WithMessage(err, "could not open database") - } - - builderOptions.Storage, err = sqlite.NewWriter(db, log.Named("storage"), &sqlite.Options{ - Compress: true, - }) - if err != nil { - return nil, errors.WithMessage(err, "could not create storage writer") - } - - website.Domain = cfg.BaseURL.Hostname() - - err = rebuild(builderOptions, cfg, log) - if err != nil { - return nil, errors.WithMessage(err, "could not build site") - } - - if opts.Development { - fw, err := events.NewFileWatcher( - log.Named("watcher"), - opts.Source, - "templates", - ) - if err != nil { - return nil, errors.WithMessage(err, "could not create file listener") - } - - go func(events chan events.Event, errs chan error) { - err := fw.Wait(events, errs) - if err != nil { - log.Panic("could not start update listener", "error", err) - } - for event := range events { - filename := event.FileEvents[0].Name - log.Info("rebuilding site", "changed_file", filename) - db.Close() - db, err = sqlite.OpenDB(opts.DBPath) - if err != nil { - log.Error("error opening database", "error", err) - } - website.reader, err = sqlite.NewReader(db, log.Named("reader")) - if err != nil { - log.Error("error creating sqlite reader", "error", err) - } - builderOptions.Storage, err = sqlite.NewWriter( - db, - log.Named("storage"), - &sqlite.Options{}, - ) - if err != nil { - log.Error("error creating sqlite writer", "error", err) - } - - err := rebuild(builderOptions, cfg, log) - if err != nil { - log.Error("error rebuilding site", "error", err) - } - opts.LiveReload.Reload() - } - }(make(chan events.Event, 1), make(chan error, 1)) - } - - website.reader, err = sqlite.NewReader(db, log.Named("reader")) - if err != nil { - return nil, errors.WithMessage(err, "error creating sqlite reader") - } - website.acctResource = "acct:" + cfg.Email website.me = digit.NewResource(website.acctResource). Link("http://openid.net/specs/connect/1.0/issuer", "", cfg.OIDCHost.String())@@ -182,16 +149,20 @@ mux.HandleFunc("/.well-known/webfinger", website.webfinger) const oidcPath = "/.well-known/openid-configuration" mux.ServeMux.Handle(oidcPath, ihttp.RedirectHandler(cfg.OIDCHost.JoinPath(oidcPath), 302)) - website.config = cfg website.App.Handler = mux return website, nil } -func rebuild(builderConfig *builder.Options, config *config.Config, log *log.Logger) error { - err := builder.BuildSite(builderConfig, config, log.Named("builder")) - if err != nil { - return errors.WithMessage(err, "could not build site") +func prepareRootDirectory(dataRoot, root string) error { + if !file.Exists(root) { + err := os.MkdirAll(root, 0755) + if err != nil { + return errors.WithMessage(err, "could not create root directory") + } + } else if dataRoot != "" && file.Exists(filepath.Join(dataRoot, "public")) { + // must be pre-sqlite if this exists + return file.CleanDir(dataRoot) } return nil