@@ -2,18 +2,21 @@ package website
 
 import (
 	"net/http"
+	"os"
+	"path/filepath"
 	"slices"
 	"strings"
+	"sync"
 
 	"gitlab.com/tozd/go/errors"
-	"go.alanpearce.eu/homestead/internal/builder"
 	"go.alanpearce.eu/homestead/internal/config"
 	"go.alanpearce.eu/homestead/internal/events"
+	"go.alanpearce.eu/homestead/internal/fetcher"
+	"go.alanpearce.eu/homestead/internal/file"
 	ihttp "go.alanpearce.eu/homestead/internal/http"
 	"go.alanpearce.eu/homestead/internal/server"
 	"go.alanpearce.eu/homestead/internal/storage"
 	"go.alanpearce.eu/homestead/internal/storage/sqlite"
-	"go.alanpearce.eu/homestead/internal/vcs"
 	"go.alanpearce.eu/homestead/templates"
 	"go.alanpearce.eu/x/log"
 
@@ -22,16 +25,14 @@ "github.com/osdevisnot/sorvor/pkg/livereload"
 )
 
 type Options struct {
-	Source      string     `conf:"default:../website"`
-	DBPath      string     `conf:"default:site.db"`
+	DataRoot    string     `conf:"noprint"`
+	Root        string     `conf:"default:./website"`
 	Redirect    bool       `conf:"default:true"`
 	Development bool       `conf:"default:false,flag:dev"`
-	BaseURL     config.URL `conf:"default:localhost"`
-	VCS         struct {
-		Branch    string     `conf:"default:main"`
-		RemoteURL config.URL `conf:"default:https://git.alanpearce.eu/website"`
-	}
+	FetchURL    config.URL `conf:"default:https://ci.alanpearce.eu/archive/website/"`
+	BaseURL     config.URL
 
+	Redis      *events.RedisOptions
 	LiveReload *livereload.LiveReload `conf:"-"`
 }
 
@@ -54,34 +55,78 @@ App: &server.App{
 			Shutdown: func() {},
 		},
 	}
-	builderOptions := &builder.Options{
-		Source:      opts.Source,
-		Development: opts.Development,
-	}
 
-	repo, exists, err := vcs.CloneOrOpen(&vcs.Options{
-		LocalPath: opts.Source,
-		RemoteURL: opts.VCS.RemoteURL,
-		Branch:    opts.VCS.Branch,
-	}, log.Named("vcs"))
+	err := prepareRootDirectory(opts.DataRoot, opts.Root)
 	if err != nil {
-		return nil, errors.WithMessage(err, "could not open repository")
+		return nil, errors.WithMessage(err, "could not prepare root directory")
 	}
-	builderOptions.Repo = repo
 
-	if exists && !opts.Development {
-		_, err := repo.Update("")
-		if err != nil {
-			return nil, errors.WithMessage(err, "could not update repository")
-		}
+	var listener events.Listener
+	if opts.Redis.Enabled {
+		log.Debug("using redis listener")
+		listener, err = events.NewRedisListener(opts.Redis, log.Named("redis"))
+	} else {
+		log.Debug("using file watcher")
+		listener, err = events.NewFileWatcher(log.Named("events"), "website")
+	}
+	if err != nil {
+		return nil, errors.WithMessage(err, "could not create update listener")
 	}
 
-	log.Debug("getting config from source", "source", opts.Source)
-	cfg, err := config.GetConfig(opts.Source, log)
+	var cfg *config.Config
+	fetcher := fetcher.New(log.Named("fetcher"), &fetcher.Options{
+		FetchURL:     opts.FetchURL,
+		RedisEnabled: false,
+		Root:         opts.Root,
+		Listener:     listener,
+	})
+	roots, err := fetcher.Subscribe()
 	if err != nil {
-		return nil, errors.WithMessage(err, "could not load configuration")
+		return nil, errors.WithMessage(err, "could not set up fetcher")
 	}
 
+	firstUpdate := make(chan bool)
+	go func() {
+		updated := sync.OnceFunc(func() {
+			firstUpdate <- true
+			close(firstUpdate)
+		})
+		for root := range roots {
+			log.Debug("getting config from source", "source", root)
+			cfg, err = config.GetConfig(root, log)
+			if err != nil {
+				log.Panic("could not load configuration", "error", err)
+			}
+			website.config = cfg
+
+			if opts.Development {
+				cfg.CSP.ScriptSrc = slices.Insert(cfg.CSP.ScriptSrc, 0, "'unsafe-inline'")
+				cfg.CSP.ConnectSrc = slices.Insert(cfg.CSP.ConnectSrc, 0, "'self'")
+			}
+
+			if opts.BaseURL.Hostname() != "" {
+				cfg.BaseURL = opts.BaseURL
+			}
+
+			website.Domain = cfg.BaseURL.Hostname()
+
+			siteDB := filepath.Join(root, "site.db")
+			db, err := sqlite.OpenDB(siteDB)
+			if err != nil {
+				log.Panic("could not open database", "error", err)
+			}
+
+			website.reader, err = sqlite.NewReader(db, log.Named("reader"))
+			if err != nil {
+				log.Panic("could not create database reader", "error", err)
+			}
+
+			updated()
+		}
+	}()
+
+	<-firstUpdate
+
 	mux := ihttp.NewServeMux()
 	mux.HandleError(func(err *ihttp.Error, w http.ResponseWriter, r *http.Request) {
 		if strings.Contains(r.Header.Get("Accept"), "text/html") {
@@ -95,84 +140,6 @@ http.Error(w, err.Message, err.Code)
 		}
 	})
 
-	if opts.Development {
-		opts.DBPath = ":memory:"
-
-		cfg.CSP.ScriptSrc = slices.Insert(cfg.CSP.ScriptSrc, 0, "'unsafe-inline'")
-		cfg.CSP.ConnectSrc = slices.Insert(cfg.CSP.ConnectSrc, 0, "'self'")
-
-		cfg.BaseURL = opts.BaseURL
-	}
-
-	db, err := sqlite.OpenDB(opts.DBPath)
-	if err != nil {
-		return nil, errors.WithMessage(err, "could not open database")
-	}
-
-	builderOptions.Storage, err = sqlite.NewWriter(db, log.Named("storage"), &sqlite.Options{
-		Compress: true,
-	})
-	if err != nil {
-		return nil, errors.WithMessage(err, "could not create storage writer")
-	}
-
-	website.Domain = cfg.BaseURL.Hostname()
-
-	err = rebuild(builderOptions, cfg, log)
-	if err != nil {
-		return nil, errors.WithMessage(err, "could not build site")
-	}
-
-	if opts.Development {
-		fw, err := events.NewFileWatcher(
-			log.Named("watcher"),
-			opts.Source,
-			"templates",
-		)
-		if err != nil {
-			return nil, errors.WithMessage(err, "could not create file listener")
-		}
-
-		go func(events chan events.Event, errs chan error) {
-			err := fw.Wait(events, errs)
-			if err != nil {
-				log.Panic("could not start update listener", "error", err)
-			}
-			for event := range events {
-				filename := event.FileEvents[0].Name
-				log.Info("rebuilding site", "changed_file", filename)
-				db.Close()
-				db, err = sqlite.OpenDB(opts.DBPath)
-				if err != nil {
-					log.Error("error opening database", "error", err)
-				}
-				website.reader, err = sqlite.NewReader(db, log.Named("reader"))
-				if err != nil {
-					log.Error("error creating sqlite reader", "error", err)
-				}
-				builderOptions.Storage, err = sqlite.NewWriter(
-					db,
-					log.Named("storage"),
-					&sqlite.Options{},
-				)
-				if err != nil {
-					log.Error("error creating sqlite writer", "error", err)
-				}
-
-				err := rebuild(builderOptions, cfg, log)
-				if err != nil {
-					log.Error("error rebuilding site", "error", err)
-				}
-				opts.LiveReload.Reload()
-			}
-		}(make(chan events.Event, 1), make(chan error, 1))
-	}
-
-	website.reader, err = sqlite.NewReader(db, log.Named("reader"))
-	if err != nil {
-		return nil, errors.WithMessage(err, "error creating sqlite reader")
-	}
-
 	website.acctResource = "acct:" + cfg.Email
 	website.me = digit.NewResource(website.acctResource).
 		Link("http://openid.net/specs/connect/1.0/issuer", "", cfg.OIDCHost.String())
@@ -182,16 +149,20 @@ mux.HandleFunc("/.well-known/webfinger", website.webfinger)
 	const oidcPath = "/.well-known/openid-configuration"
 	mux.ServeMux.Handle(oidcPath, ihttp.RedirectHandler(cfg.OIDCHost.JoinPath(oidcPath), 302))
 
-	website.config = cfg
 	website.App.Handler = mux
 
 	return website, nil
 }
 
-func rebuild(builderConfig *builder.Options, config *config.Config, log *log.Logger) error {
-	err := builder.BuildSite(builderConfig, config, log.Named("builder"))
-	if err != nil {
-		return errors.WithMessage(err, "could not build site")
+func prepareRootDirectory(dataRoot, root string) error {
+	if !file.Exists(root) {
+		err := os.MkdirAll(root, 0755)
+		if err != nil {
+			return errors.WithMessage(err, "could not create root directory")
+		}
+	} else if dataRoot != "" && file.Exists(filepath.Join(dataRoot, "public")) {
+		// must be pre-sqlite if this exists
+		return file.CleanDir(dataRoot)
 	}
 
 	return nil