feat: improve error logging of programs database
3 files changed, 66 insertions(+), 48 deletions(-)
M internal/importer/main.go → internal/importer/main.go
@@ -254,13 +254,6 @@ forceUpdate bool, ) func(config.Source) error { return func(source config.Source) error { logger := imp.options.Logger.With("name", source.Key) - pdb, err := programs.New(source, &programs.Options{ - Logger: logger, - Root: imp.options.Root, - }) - if err != nil { - return fault.Wrap(err, fmsg.With("error creating program database")) - } sourceMeta := meta.GetSourceMeta(source.Key) previousUpdate := sourceMeta.UpdatedAt@@ -313,6 +306,13 @@ forceUpdate, ) if source.Programs.Enable { + pdb, err := programs.New(source, &programs.Options{ + Logger: logger, + Root: imp.options.Root, + }) + if err != nil { + return fault.Wrap(err, fmsg.With("error creating program database")) + } err = pdb.Instantiate(ctx) if err != nil { logger.Warn("programs database instantiation failed", "error", err)@@ -335,15 +335,8 @@ parent context.Context, meta *meta.Meta, ) func(config.Source) error { return func(source config.Source) error { + var pdb *programs.DB logger := imp.options.Logger.With("name", source.Key) - pdb, err := programs.New(source, &programs.Options{ - Logger: logger, - Root: imp.options.Root, - }) - if err != nil { - return fault.Wrap(err, fmsg.With("error creating program database")) - } - sourceMeta := meta.GetSourceMeta(source.Key) fopts := &fetcher.Options{@@ -360,10 +353,20 @@ return fault.Wrap(err, fmsg.With("error opening fetched files")) } if source.Programs.Enable { - err = pdb.Instantiate(ctx) + pdb, err = programs.New(source, &programs.Options{ + Logger: logger, + Root: imp.options.Root, + }) if err != nil { - logger.Warn("programs database instantiation failed", "error", err) + return fault.Wrap(err, fmsg.With("error creating program database")) } + + err = pdb.Open(ctx) + if err != nil { + return fault.Wrap(err, fmsg.With("error opening program database")) + } + + defer pdb.Close() } if files.Revision != nil {
M internal/importer/package.go → internal/importer/package.go
@@ -89,18 +89,6 @@ ) iter.Seq[index.Indexable] { return func(yield func(index.Indexable) bool) { defer i.infile.Close() - if i.programs != nil { - err := i.programs.Open(ctx) - if err != nil { - i.errs = append(i.errs, fault.Wrap(err, fmsg.With("could not open programs database"))) - i.programs = nil - } - } - - if i.programs != nil { - defer i.programs.Close() - } - for mv := range i.dec.Stream() { if ctx.Err() != nil { break
M internal/programs/programs.go → internal/programs/programs.go
@@ -3,14 +3,16 @@ import ( "context" "database/sql" + "errors" "fmt" + "net/url" "os/exec" "strings" "alin.ovh/x/log" "github.com/Southclaws/fault" "github.com/Southclaws/fault/fmsg" - _ "modernc.org/sqlite" //nolint:revive // sqlite driver needed for database/sql + "modernc.org/sqlite" "alin.ovh/searchix/internal/config" "alin.ovh/searchix/internal/file"@@ -30,26 +32,11 @@ Root *file.Root } func New(source config.Source, options *Options) (*DB, error) { - db, err := sql.Open("sqlite", fmt.Sprintf( - "file:%s?mode=%s&_pragma=foreign_keys(1)&_pragma=mmap_size(%d)", - //nolint:forbidigo // external package - options.Root.JoinPath( - source.JoinPath("programs.db"), - ), - "rwc", - 16*1024*1024, - )) - if err != nil { - return nil, fault.Wrap(err, fmsg.With("failed to open sqlite database")) - } - options.Logger.Debug("opened sqlite database") - return &DB{ source: source, logger: options.Logger, root: options.Root, - db: db, }, nil }@@ -72,12 +59,27 @@ outPath := strings.Trim(strings.TrimSpace(string(out)), "\"") p.logger.Debug("got output path", "outputPath", outPath) + p.db, err = p.openDB("rwc") + if err != nil { + return fault.Wrap(err, fmsg.With("failed to open sqlite database")) + } + _, err = p.db.ExecContext(ctx, "DROP TABLE IF EXISTS programs") if err != nil { return fault.Wrap(err, fmsg.With("failed to drop programs table")) } - _, err = p.db.ExecContext(ctx, "ATTACH DATABASE ? AS input", outPath) + _, err = p.db.ExecContext( + ctx, + "ATTACH DATABASE ? AS input", + (&url.URL{ + Scheme: "file", + Path: outPath, + RawQuery: url.Values{ + "immutable": {"1"}, + }.Encode(), + }).String(), + ) if err != nil { return fault.Wrap(err, fmsg.With("failed to attach nix-store programs database")) }@@ -107,9 +109,34 @@ return nil } +func (p *DB) openDB(mode string) (*sql.DB, error) { + u := fmt.Sprintf( + "file:%s?mode=%s&_pragma=foreign_keys(1)&_pragma=mmap_size(%d)", + //nolint: forbidigo // external package + p.root.JoinPath(p.source.JoinPath("programs.db")), + mode, + 16*1024*1024, + ) + + db, err := sql.Open("sqlite", u) + if err != nil { + var sqliteErr *sqlite.Error + if errors.As(err, &sqliteErr) { + return nil, fault.New( + sqlite.ErrorCodeString[sqliteErr.Code()], + ) + } + + return nil, fault.Wrap(err, fmsg.With("failed to open sqlite database")) + } + + return db, nil +} + func (p *DB) Open(ctx context.Context) (err error) { - if p.db == nil { - return fault.New("database not open") + p.db, err = p.openDB("rw") + if err != nil { + return fault.Wrap(err) } p.stmt, err = p.db.PrepareContext(ctx, `