@@ -34,7 +34,8 @@ )
 	generateErrorPage = flag.Bool("generate-error-page", false, "generate error page and exit")
 	dev               = flag.Bool("dev", false, "enable live reloading and nicer logging")
 	replace           = flag.Bool("replace", false, "replace existing index and exit")
-	fetch             = flag.Bool("fetch", false, "fetch data and exit")
+	prefetch          = flag.Bool("fetch", false, "pre-fetch data and exit")
+	offline           = flag.Bool("offline", false, "run in offline mode")
 	version           = flag.Bool("version", false, "print version information")
 	cpuprofile        = flag.String("cpuprofile", "", "enable CPU profiling and save to `file`")
 )
@@ -139,18 +140,19 @@ LowMemory:  cfg.Importer.LowMemory,
 		Logger:     logger.Named("importer"),
 		Manpages:   mdb,
 		Root:       root,
+		Offline:    *offline,
 	})
 	if err != nil {
 		logger.Fatal("Failed to create importer", "error", err)
 	}
 
-	if !exists || *replace || *fetch {
-		err := imp.Start(ctx, true, *fetch, nil)
+	if !exists || *replace || *prefetch {
+		err := imp.Start(ctx, true, *prefetch, nil)
 		if err != nil {
 			logger.Fatal("Failed to start importer", "error", err)
 		}
 
-		if *replace || *fetch {
+		if *replace || *prefetch {
 			return
 		}
 	}

@@ -3,6 +3,7 @@
 import (
 	"context"
 	"io"
+	"os"
 
 	"alin.ovh/searchix/internal/config"
 	"alin.ovh/searchix/internal/file"
@@ -57,3 +58,49 @@ }
 
 	return
 }
+
+func Open(
+	source *config.Source,
+	opts *Options,
+) (*FetchedFiles, error) {
+	root := opts.Root
+
+	f := &FetchedFiles{}
+
+	rev, err := openIfExists(root, source.JoinPath("revision"))
+	if err != nil {
+		opts.Logger.Warn("failed to open revision file", "error", err)
+	}
+	if rev != nil {
+		f.Revision = rev
+	}
+
+	switch source.Importer {
+	case config.Options:
+		f.Options, err = root.Open(source.JoinPath("options.json"))
+	case config.Packages:
+		f.Packages, err = root.Open(source.JoinPath("packages.json"))
+	default:
+		err = fault.Newf("unsupported importer type %s", source.Importer.String())
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return f, nil
+}
+
+func openIfExists(root *file.Root, filename string) (*os.File, error) {
+	if exists, err := root.Exists(filename); err != nil {
+		return nil, fault.Wrap(err, fmsg.With("failed to check if file exists"))
+	} else if !exists {
+		return nil, nil
+	}
+
+	f, err := root.Open(filename)
+	if err != nil {
+		return nil, fault.Wrap(err, fmsg.Withf("failed to open file %s", filename))
+	}
+
+	return f, nil
+}

@@ -60,69 +60,100 @@ forceUpdate bool,
 	fetchOnly bool,
 ) func(*config.Source) error {
 	return func(source *config.Source) error {
-		logger := imp.options.Logger.With("name", source.Key)
-		logger.Debug("starting fetcher")
+		var files *fetcher.FetchedFiles
 
-		fetcher, err := fetcher.New(source, &fetcher.Options{
+		logger := imp.options.Logger.With("name", source.Key)
+		pdb, err := programs.New(source, &programs.Options{
 			Logger: logger,
 			Root:   imp.options.Root,
 		})
 		if err != nil {
-			return fault.Wrap(err, fmsg.With("error creating fetcher"))
+			return fault.Wrap(err, fmsg.With("error creating program database"))
 		}
 
 		sourceMeta := meta.GetSourceMeta(source.Key)
 		previousUpdate := sourceMeta.UpdatedAt
+
+		fopts := &fetcher.Options{
+			Logger: logger,
+			Root:   imp.options.Root,
+		}
+
 		ctx, cancel := context.WithTimeout(parent, source.Timeout.Duration)
 		defer cancel()
-		files, err := fetcher.FetchIfNeeded(ctx, sourceMeta)
-		if err != nil {
-			var exerr *exec.ExitError
-			if errors.As(err, &exerr) {
-				lines := strings.SplitSeq(strings.TrimSpace(string(exerr.Stderr)), "\n")
-				for line := range lines {
-					logger.Error(
-						"importer fetch failed",
-						"fetcher",
-						source.Fetcher.String(),
-						"stderr",
-						line,
-						"status",
-						exerr.ExitCode(),
-					)
-				}
+
+		if imp.options.Offline {
+			logger.Debug("skipping fetch; in offline mode")
+
+			files, err = fetcher.Open(source, fopts)
+			if err != nil {
+				return fault.Wrap(err, fmsg.With("error opening fetched files"))
 			}
+		} else {
+			logger.Debug("starting fetcher")
 
-			return fault.Wrap(err, fmsg.With("importer fetch failed"))
-		}
-		logger.Info(
-			"importer fetch succeeded",
-			"previous",
-			previousUpdate.Format(time.DateTime),
-			"current",
-			sourceMeta.UpdatedAt.Format(time.DateTime),
-			"is_updated",
-			sourceMeta.UpdatedAt.After(previousUpdate),
-			"update_force",
-			forceUpdate,
-			"fetch_only",
-			fetchOnly,
-		)
+			fetcher, err := fetcher.New(source, fopts)
+			if err != nil {
+				return fault.Wrap(err, fmsg.With("error creating fetcher"))
+			}
 
-		if !fetchOnly &&
-			(!sourceMeta.UpdatedAt.After(sourceMeta.ImportedAt) || sourceMeta.ImportedAt.IsZero() || forceUpdate) {
-			var pdb *programs.DB
+			files, err = fetcher.FetchIfNeeded(ctx, sourceMeta)
+			if err != nil {
+				var exerr *exec.ExitError
+				if errors.As(err, &exerr) {
+					lines := strings.SplitSeq(strings.TrimSpace(string(exerr.Stderr)), "\n")
+					for line := range lines {
+						logger.Error(
+							"importer fetch failed",
+							"fetcher",
+							source.Fetcher.String(),
+							"stderr",
+							line,
+							"status",
+							exerr.ExitCode(),
+						)
+					}
+				}
+
+				return fault.Wrap(err, fmsg.With("importer fetch failed"))
+			}
+			logger.Info(
+				"importer fetch succeeded",
+				"previous",
+				previousUpdate.Format(time.DateTime),
+				"current",
+				sourceMeta.UpdatedAt.Format(time.DateTime),
+				"is_updated",
+				sourceMeta.UpdatedAt.After(previousUpdate),
+				"update_force",
+				forceUpdate,
+				"fetch_only",
+				fetchOnly,
+			)
 
 			if source.Programs.Enable {
-				pdb, err = programs.Instantiate(ctx, source, imp.options.Logger.Named("programs"))
+				err = pdb.Instantiate(ctx)
 				if err != nil {
 					logger.Warn("programs database instantiation failed", "error", err)
 				}
 			}
 
-			err = setRepoRevision(files.Revision, source)
-			if err != nil {
-				logger.Warn("could not set source repo revision", "error", err)
+			if source.Manpages.Enable {
+				err = imp.options.Manpages.Update(ctx, source)
+				if err != nil {
+					logger.Warn("manpages database update failed", "error", err)
+				}
+			}
+		}
+
+		if !fetchOnly &&
+			(!sourceMeta.UpdatedAt.After(sourceMeta.ImportedAt) || sourceMeta.ImportedAt.IsZero() || forceUpdate) {
+
+			if files.Revision != nil {
+				err = setRepoRevision(files.Revision, source)
+				if err != nil {
+					logger.Warn("could not set source repo revision", "error", err)
+				}
 			}
 
 			var processor Processor
@@ -155,13 +186,6 @@
 			hadWarnings, err := process(ctx, imp.options.WriteIndex, processor, logger)
 			if err != nil {
 				return fault.Wrap(err, fmsg.Withf("failed to process source"))
-			}
-
-			if source.Manpages.Enable {
-				err = imp.options.Manpages.Update(ctx, source)
-				if err != nil {
-					logger.Warn("manpages database update failed", "error", err)
-				}
 			}
 
 			sourceMeta.ImportedAt = time.Now()

@@ -39,6 +39,9 @@ Manpages: manpages.New(&manpages.Options{
 			Logger: logger.Named("manpages"),
 			Root:   tmp,
 		}),
+		Offline: false,
+		Root:    tmp,
+		Storage: store,
 	})
 	if err != nil {
 		b.Fatal(err)

@@ -123,7 +123,7 @@ results := make(chan nix.Importable)
 	errs := make(chan error)
 
 	if i.programs != nil {
-		err := i.programs.Open()
+		err := i.programs.Open(ctx)
 		if err != nil {
 			errs <- fault.Wrap(err, fmsg.With("could not open programs database"))
 			i.programs = nil
@@ -284,7 +284,7 @@
 				continue
 			}
 
-			if i.programs != nil {
+			if i.source.Programs.Enable {
 				programs, err = i.programs.GetPackagePrograms(ctx, kv.Key)
 				if err != nil {
 					errs <- fault.Wrap(err, fmsg.Withf("failed to get programs for package %s", i.pkg.Name))

@@ -210,7 +210,6 @@
 var expectedDataFiles = []string{
 	metaBaseName,
 	indexBaseName,
-	"manpage-urls.json",
 }
 
 func deleteIndex(root *file.Root) error {
@@ -224,9 +223,7 @@
 	return nil
 }
 
-func OpenOrCreate(
-	options *Options,
-) (*ReadIndex, *WriteIndex, bool, error) {
+func OpenOrCreate(options *Options) (*ReadIndex, *WriteIndex, bool, error) {
 	var err error
 	bleve.SetLog(zap.NewStdLog(options.Logger.Named("bleve").GetLogger()))
 

@@ -8,6 +8,7 @@ "os/exec"
 	"strings"
 
 	"alin.ovh/searchix/internal/config"
+	"alin.ovh/searchix/internal/file"
 	"alin.ovh/x/log"
 	"github.com/Southclaws/fault"
 	"github.com/Southclaws/fault/fmsg"
@@ -15,62 +16,75 @@ _ "modernc.org/sqlite" //nolint:blank-imports // sqlite driver needed for database/sql
 )
 
 type DB struct {
-	Path   string
-	Source *config.Source
-
+	source *config.Source
 	logger *log.Logger
+	root   *file.Root
 	db     *sql.DB
 	stmt   *sql.Stmt
 }
 
-func Instantiate(
-	ctx context.Context,
-	source *config.Source,
-	logger *log.Logger,
-) (*DB, error) {
+type Options struct {
+	Logger *log.Logger
+	Root   *file.Root
+}
+
+func New(source *config.Source, options *Options) (*DB, error) {
+	db, err := sql.Open("sqlite", fmt.Sprintf(
+		"file:%s?mode=%s&_pragma=foreign_keys(1)&_pragma=mmap_size(%d)",
+		//nolint:forbidigo // external package
+		options.Root.JoinPath(
+			source.JoinPath("programs.db"),
+		),
+		"rwc",
+		16*1024*1024,
+	))
+	if err != nil {
+		return nil, fault.Wrap(err, fmsg.With("failed to open sqlite database"))
+	}
+	options.Logger.Debug("opened sqlite database")
+
+	return &DB{
+		source: source,
+
+		logger: options.Logger,
+		root:   options.Root,
+		db:     db,
+	}, nil
+}
+
+func (p *DB) Instantiate(ctx context.Context) error {
 	// nix-instantiate --eval --json -I nixpkgs=channel:nixos-unstable --expr 'toString <nixpkgs/programs.sqlite>'
 	args := []string{
 		"--eval",
 		"--json",
-		"-I", fmt.Sprintf("%s=channel:%s", source.Key, source.Channel),
-		"--expr", fmt.Sprintf("toString <%s/%s>", source.Key, source.Programs.Attribute),
+		"-I", fmt.Sprintf("%s=channel:%s", p.source.Key, p.source.Channel),
+		"--expr", fmt.Sprintf("toString <%s/%s>", p.source.Key, p.source.Programs.Attribute),
 	}
 
-	logger.Debug("nix-instantiate command", "args", args)
+	p.logger.Debug("nix-instantiate command", "args", args)
 	cmd := exec.CommandContext(ctx, "nix-instantiate", args...)
 	out, err := cmd.Output()
 	if err != nil {
-		return nil, fault.Wrap(err, fmsg.With("failed to run nix-instantiate"))
+		return fault.Wrap(err, fmsg.With("failed to run nix-instantiate"))
 	}
 
 	outPath := strings.Trim(strings.TrimSpace(string(out)), "\"")
-	logger.Debug("got output path", "outputPath", outPath)
+	p.logger.Debug("got output path", "outputPath", outPath)
 
-	return &DB{
-		Source: source,
-		Path:   outPath,
-
-		logger: logger,
-	}, nil
-}
-
-func (p *DB) Open() error {
-	var err error
-	p.db, err = sql.Open("sqlite", p.Path)
+	_, err = p.db.ExecContext(ctx, "ATTACH DATABASE ? AS input", outPath)
 	if err != nil {
-		return fault.Wrap(err, fmsg.With("failed to open sqlite database"))
+		return fault.Wrap(err, fmsg.With("failed to attach nix-store programs database"))
 	}
-	p.logger.Debug("opened sqlite database")
 
-	_, err = p.db.Exec("ATTACH DATABASE ':memory:' AS mem")
+	_, err = p.db.ExecContext(ctx, "DROP TABLE IF EXISTS programs")
 	if err != nil {
-		return fault.Wrap(err, fmsg.With("failed to attach in-memory database"))
+		return fault.Wrap(err, fmsg.With("failed to drop programs table"))
 	}
 
-	_, err = p.db.Exec(`
-CREATE TABLE mem.programs AS
+	_, err = p.db.ExecContext(ctx, `
+CREATE TABLE programs AS
 SELECT name, package
-FROM main.Programs
+FROM input.Programs
 GROUP BY name, package
 `)
 	if err != nil {
@@ -78,15 +92,28 @@ return fault.Wrap(err, fmsg.With("failed to create programs table"))
 	}
 	p.logger.Debug("created programs table")
 
-	_, err = p.db.Exec(`CREATE INDEX mem.idx_package ON programs(package)`)
+	_, err = p.db.ExecContext(ctx, `CREATE INDEX idx_package ON programs(package)`)
 	if err != nil {
 		return fault.Wrap(err, fmsg.With("failed to create idx_package index"))
 	}
 	p.logger.Debug("created idx_package index")
 
-	p.stmt, err = p.db.Prepare(`
+	_, err = p.db.ExecContext(ctx, "DETACH DATABASE input")
+	if err != nil {
+		return fault.Wrap(err, fmsg.With("failed to detach nix-store programs database"))
+	}
+
+	return nil
+}
+
+func (p *DB) Open(ctx context.Context) (err error) {
+	if p.db == nil {
+		return fault.New("database not open")
+	}
+
+	p.stmt, err = p.db.PrepareContext(ctx, `
 SELECT name
-FROM mem.programs
+FROM programs
 WHERE package = ?
 `)
 	if err != nil {
@@ -107,9 +134,6 @@ }
 
 func (p *DB) GetPackagePrograms(ctx context.Context, pkg string) ([]string, error) {
 	var programs []string
-	if p.db == nil {
-		return nil, fault.New("database not open")
-	}
 	rows, err := p.stmt.QueryContext(ctx, pkg)
 	if err != nil {
 		return nil, fault.Wrap(err, fmsg.With("failed to execute query"))