feat: improve tracking and display of import runs
1 file changed, 28 insertions(+), 30 deletions(-)
changed files
M internal/importer/main.go → internal/importer/main.go
@@ -33,28 +33,25 @@ Root *file.Root } var Job struct { - InProgress bool - StartedAt time.Time - FinishedAt time.Time - NextRun time.Time + StartedAt time.Time + LastRun struct { + StartedAt time.Time + FinishedAt time.Time + } + NextRun time.Time } -func SetNextRun(nextRun time.Time) { - Job.NextRun = nextRun +func MarkIndexingStarted() { + Job.StartedAt = time.Now() } -func SetLastUpdated(last time.Time) { - Job.FinishedAt = last -} - -func MarkIndexingStarted() { - Job.StartedAt = time.Now() - Job.InProgress = true +func MarkIndexingFinished() { + Job.LastRun.StartedAt = Job.StartedAt + Job.LastRun.FinishedAt = time.Now() + Job.StartedAt = time.Time{} } -func MarkIndexingFinished(nextRun time.Time) { - Job.FinishedAt = time.Now() - Job.InProgress = false +func SetNextRun(nextRun time.Time) { Job.NextRun = nextRun }@@ -77,10 +74,7 @@ return fault.Wrap(err, fmsg.With("error creating fetcher")) } sourceMeta := meta.GetSourceMeta(source.Key) - if forceUpdate { - sourceMeta.Updated = time.Unix(0, 0) - } - previousUpdate := sourceMeta.Updated + previousUpdate := sourceMeta.UpdatedAt ctx, cancel := context.WithTimeout(parent, source.Timeout.Duration) defer cancel() files, err := fetcher.FetchIfNeeded(ctx, sourceMeta)@@ -108,16 +102,17 @@ "importer fetch succeeded", "previous", previousUpdate.Format(time.DateTime), "current", - sourceMeta.Updated.Format(time.DateTime), + sourceMeta.UpdatedAt.Format(time.DateTime), "is_updated", - sourceMeta.Updated.After(previousUpdate), + sourceMeta.UpdatedAt.After(previousUpdate), "update_force", forceUpdate, "fetch_only", fetchOnly, ) - if !fetchOnly && (!sourceMeta.Updated.After(previousUpdate) || forceUpdate) { + if !fetchOnly && + (!sourceMeta.UpdatedAt.After(sourceMeta.ImportedAt) || sourceMeta.ImportedAt.IsZero() || forceUpdate) { var pdb *programs.DB if source.Programs.Enable {@@ -172,6 +167,8 @@ logger.Warn("manpages database update failed", "error", err) } } + sourceMeta.ImportedAt = time.Now() + if hadWarnings { logger.Warn("importer succeeded, but with warnings/errors") } else {@@ -213,6 +210,7 @@ forceUpdate = forceUpdate || (onlyUpdateSources != nil && len(*onlyUpdateSources) > 0) meta := imp.options.WriteIndex.Meta + MarkIndexingStarted() importSource := imp.createSourceImporter(importCtx, meta, forceUpdate, fetchOnly) for name, source := range imp.config.Importer.Sources {@@ -227,12 +225,12 @@ imp.options.Logger.Error("import failed", "source", name, "error", err) } } + MarkIndexingFinished() + err := imp.options.WriteIndex.SaveMeta() if err != nil { return fault.Wrap(err, fmsg.With("failed to save metadata")) } - - SetLastUpdated(time.Now()) return nil }@@ -241,6 +239,8 @@ func New( cfg *config.Config, options *Options, ) (*Importer, error) { + Job.LastRun = options.WriteIndex.Meta.LastImport + return &Importer{ config: cfg, options: options,@@ -307,11 +307,9 @@ CheckInMargin: 5, Timezone: time.Local.String(), } - Job.FinishedAt = imp.options.WriteIndex.Meta.LastUpdated() - var nextRun time.Time switch { - case Job.FinishedAt.Before(time.Now().Add(-24 * time.Hour)): + case Job.LastRun.FinishedAt.Before(time.Now().Add(-24 * time.Hour)): imp.options.Logger.Info( "indexing last ran more than 24 hours ago, scheduling immediate update", )@@ -339,7 +337,6 @@ eventID := localHub.CaptureCheckIn(&sentry.CheckIn{ MonitorSlug: monitorSlug, Status: sentry.CheckInStatusInProgress, }, monitorConfig) - MarkIndexingStarted() ctx, cancel := context.WithTimeout(parentCtx, imp.config.Importer.Timeout.Duration) err = imp.Start(ctx, false, false, nil)@@ -364,7 +361,8 @@ Status: sentry.CheckInStatusOK, }, monitorConfig) } nextRun = nextUTCOccurrenceOfTime(imp.config.Importer.UpdateAt) - MarkIndexingFinished(nextRun) + SetNextRun(nextRun) + imp.options.Logger.Info( "scheduling next run", "next-run",