feat: improve tracking and display of import runs
7 files changed, 83 insertions(+), 55 deletions(-)
M go.mod → go.mod
@@ -13,6 +13,7 @@ github.com/blevesearch/bleve/v2 v2.5.2 github.com/blevesearch/bleve_index_api v1.2.8 github.com/creasty/defaults v1.8.0 github.com/crewjam/csp v0.0.2 + github.com/dustin/go-humanize v1.0.1 github.com/fsnotify/fsnotify v1.9.0 github.com/getsentry/sentry-go v0.33.0 github.com/mitchellh/mapstructure v1.5.0@@ -46,7 +47,6 @@ github.com/blevesearch/zapx/v13 v13.4.2 // indirect github.com/blevesearch/zapx/v14 v14.4.2 // indirect github.com/blevesearch/zapx/v15 v15.4.2 // indirect github.com/blevesearch/zapx/v16 v16.2.4 // indirect - github.com/dustin/go-humanize v1.0.1 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/google/uuid v1.6.0 // indirect
M internal/components/search.go → internal/components/search.go
@@ -1,12 +1,14 @@ package components import ( + "fmt" "time" g "alin.ovh/gomponents" . "alin.ovh/gomponents/html" "alin.ovh/searchix/internal/config" "alin.ovh/searchix/internal/importer" + "github.com/dustin/go-humanize" ) func SearchForm(tdata TemplateData, r ResultData) g.Node {@@ -48,7 +50,7 @@ MapCommaList(tdata.Sources, func(source *config.Source) g.Node { return A(Href(source.Repo.String()), g.Text(source.Name)) }), ), - g.If(importer.Job.InProgress, + g.If(!importer.Job.StartedAt.IsZero(), P(Class("notice"), g.Text("Indexing in progress, started "), Time(@@ -57,23 +59,32 @@ Title(importer.Job.StartedAt.Format(time.DateTime)), g.Text(time.Since(importer.Job.StartedAt).Round(time.Second).String()), ), g.Text(" ago. "), - g.If(!importer.Job.FinishedAt.IsZero(), - g.Group([]g.Node{ + g.Iff(!importer.Job.LastRun.FinishedAt.IsZero(), func() g.Node { + duration := importer.Job.LastRun.FinishedAt.Sub(importer.Job.LastRun.StartedAt) + + return g.Group([]g.Node{ g.Text("Last run took "), Time( - DateTime(importer.Job.FinishedAt.Format(time.RFC3339)), - Title(importer.Job.FinishedAt.Format(time.DateTime)), - g.Text(time.Since(importer.Job.FinishedAt).Round(time.Minute).String()), + DateTime(DurationDateTimeString(duration)), + Title(duration.Round(time.Second).String()), + g.Text(duration.Round(time.Minute).String()), ), - }), - ), + }) + }), ), P( g.Text("Indexing last ran "), Time( - DateTime(importer.Job.FinishedAt.Format(time.RFC3339)), - Title(importer.Job.FinishedAt.Format(time.DateTime)), - g.Textf("%.0f hours ago", time.Since(importer.Job.FinishedAt).Hours()), + DateTime(importer.Job.LastRun.FinishedAt.Format(time.RFC3339)), + Title(importer.Job.LastRun.FinishedAt.Format(time.DateTime)), + g.Text( + humanize.RelTime( + importer.Job.LastRun.FinishedAt, + time.Now(), + "ago", + "in the future", + ), + ), ), g.Text(", will run again in "), Time(@@ -128,3 +139,8 @@ } return g.Group(out) } + +func DurationDateTimeString(d time.Duration) string { + // PThHmMsS + return fmt.Sprintf("PT%dH%dM%dS", d/time.Hour, d/time.Minute%60, d/time.Second%60) +}
M internal/fetcher/channel.go → internal/fetcher/channel.go
@@ -74,7 +74,7 @@ outPath != sourceMeta.Path, ) if outPath != sourceMeta.Path { sourceMeta.Path = outPath - sourceMeta.Updated = time.Now().Truncate(time.Second) + sourceMeta.UpdatedAt = time.Now().Truncate(time.Second) if exists, err := i.Root.Exists(target); err != nil { return nil, fault.Wrap(err, fmsg.With("failed to check if target path exists"))
M internal/fetcher/download.go → internal/fetcher/download.go
@@ -85,7 +85,7 @@ stat, err := i.Root.Stat(target) if err != nil { return nil, fault.Wrap(err, fmsg.Withf("could not stat file %s", target)) } - sourceMeta.Updated = stat.ModTime() + sourceMeta.UpdatedAt = stat.ModTime() switch basename { case files["revision"]:
M internal/fetcher/nixpkgs-channel.go → internal/fetcher/nixpkgs-channel.go
@@ -82,7 +82,7 @@ stat, err := i.Root.Stat(target) if err != nil { return nil, fault.Wrap(err, fmsg.Withf("could not stat file %s", target)) } - sourceMeta.Updated = stat.ModTime() + sourceMeta.UpdatedAt = stat.ModTime() switch urlname { case revisionFilename:
M internal/importer/main.go → internal/importer/main.go
@@ -33,28 +33,25 @@ Root *file.Root } var Job struct { - InProgress bool - StartedAt time.Time - FinishedAt time.Time - NextRun time.Time + StartedAt time.Time + LastRun struct { + StartedAt time.Time + FinishedAt time.Time + } + NextRun time.Time } -func SetNextRun(nextRun time.Time) { - Job.NextRun = nextRun +func MarkIndexingStarted() { + Job.StartedAt = time.Now() } -func SetLastUpdated(last time.Time) { - Job.FinishedAt = last -} - -func MarkIndexingStarted() { - Job.StartedAt = time.Now() - Job.InProgress = true +func MarkIndexingFinished() { + Job.LastRun.StartedAt = Job.StartedAt + Job.LastRun.FinishedAt = time.Now() + Job.StartedAt = time.Time{} } -func MarkIndexingFinished(nextRun time.Time) { - Job.FinishedAt = time.Now() - Job.InProgress = false +func SetNextRun(nextRun time.Time) { Job.NextRun = nextRun }@@ -77,10 +74,7 @@ return fault.Wrap(err, fmsg.With("error creating fetcher")) } sourceMeta := meta.GetSourceMeta(source.Key) - if forceUpdate { - sourceMeta.Updated = time.Unix(0, 0) - } - previousUpdate := sourceMeta.Updated + previousUpdate := sourceMeta.UpdatedAt ctx, cancel := context.WithTimeout(parent, source.Timeout.Duration) defer cancel() files, err := fetcher.FetchIfNeeded(ctx, sourceMeta)@@ -108,16 +102,17 @@ "importer fetch succeeded", "previous", previousUpdate.Format(time.DateTime), "current", - sourceMeta.Updated.Format(time.DateTime), + sourceMeta.UpdatedAt.Format(time.DateTime), "is_updated", - sourceMeta.Updated.After(previousUpdate), + sourceMeta.UpdatedAt.After(previousUpdate), "update_force", forceUpdate, "fetch_only", fetchOnly, ) - if !fetchOnly && (!sourceMeta.Updated.After(previousUpdate) || forceUpdate) { + if !fetchOnly && + (!sourceMeta.UpdatedAt.After(sourceMeta.ImportedAt) || sourceMeta.ImportedAt.IsZero() || forceUpdate) { var pdb *programs.DB if source.Programs.Enable {@@ -172,6 +167,8 @@ logger.Warn("manpages database update failed", "error", err) } } + sourceMeta.ImportedAt = time.Now() + if hadWarnings { logger.Warn("importer succeeded, but with warnings/errors") } else {@@ -213,6 +210,7 @@ forceUpdate = forceUpdate || (onlyUpdateSources != nil && len(*onlyUpdateSources) > 0) meta := imp.options.WriteIndex.Meta + MarkIndexingStarted() importSource := imp.createSourceImporter(importCtx, meta, forceUpdate, fetchOnly) for name, source := range imp.config.Importer.Sources {@@ -227,12 +225,12 @@ imp.options.Logger.Error("import failed", "source", name, "error", err) } } + MarkIndexingFinished() + err := imp.options.WriteIndex.SaveMeta() if err != nil { return fault.Wrap(err, fmsg.With("failed to save metadata")) } - - SetLastUpdated(time.Now()) return nil }@@ -241,6 +239,8 @@ func New( cfg *config.Config, options *Options, ) (*Importer, error) { + Job.LastRun = options.WriteIndex.Meta.LastImport + return &Importer{ config: cfg, options: options,@@ -307,11 +307,9 @@ CheckInMargin: 5, Timezone: time.Local.String(), } - Job.FinishedAt = imp.options.WriteIndex.Meta.LastUpdated() - var nextRun time.Time switch { - case Job.FinishedAt.Before(time.Now().Add(-24 * time.Hour)): + case Job.LastRun.FinishedAt.Before(time.Now().Add(-24 * time.Hour)): imp.options.Logger.Info( "indexing last ran more than 24 hours ago, scheduling immediate update", )@@ -339,7 +337,6 @@ eventID := localHub.CaptureCheckIn(&sentry.CheckIn{ MonitorSlug: monitorSlug, Status: sentry.CheckInStatusInProgress, }, monitorConfig) - MarkIndexingStarted() ctx, cancel := context.WithTimeout(parentCtx, imp.config.Importer.Timeout.Duration) err = imp.Start(ctx, false, false, nil)@@ -364,7 +361,8 @@ Status: sentry.CheckInStatusOK, }, monitorConfig) } nextRun = nextUTCOccurrenceOfTime(imp.config.Importer.UpdateAt) - MarkIndexingFinished(nextRun) + SetNextRun(nextRun) + imp.options.Logger.Info( "scheduling next run", "next-run",
M internal/index/index_meta.go → internal/index/index_meta.go
@@ -11,17 +11,22 @@ "github.com/Southclaws/fault" "github.com/Southclaws/fault/fmsg" ) -const CurrentSchemaVersion = 5 +const CurrentSchemaVersion = 6 type SourceMeta struct { - Updated time.Time - Path string - Rev string + ImportedAt time.Time + UpdatedAt time.Time + Path string + Rev string } type data struct { SchemaVersion int - Sources map[string]*SourceMeta + LastImport struct { + StartedAt time.Time + FinishedAt time.Time + } + Sources map[string]*SourceMeta } type Meta struct {@@ -44,6 +49,7 @@ root: root, log: log, data: data{ SchemaVersion: CurrentSchemaVersion, + Sources: make(map[string]*SourceMeta), }, }, nil }@@ -102,17 +108,25 @@ return sourceMeta } func (i *Meta) SetSourceMeta(source string, meta *SourceMeta) { - if i.Sources == nil { - i.Sources = make(map[string]*SourceMeta) - } i.Sources[source] = meta } +func (i *Meta) LastImported() time.Time { + var last time.Time + for _, sourceMeta := range i.Sources { + if sourceMeta.ImportedAt.After(last) { + last = sourceMeta.ImportedAt + } + } + + return last +} + func (i *Meta) LastUpdated() time.Time { var last time.Time for _, sourceMeta := range i.Sources { - if sourceMeta.Updated.After(last) { - last = sourceMeta.Updated + if sourceMeta.UpdatedAt.After(last) { + last = sourceMeta.UpdatedAt } }