all repos — searchix @ 001c5b4f4e6dcf0110504211d3f1298d5e6abc48

Search engine for NixOS, nix-darwin, home-manager and NUR users

feat: add --fetch action to populate data/sources

Alan Pearce
commit

001c5b4f4e6dcf0110504211d3f1298d5e6abc48

parent

a49d405f81ed9b872ae638832f3a1a5f2339780c

3 files changed, 14 insertions(+), 7 deletions(-)

changed files
M cmd/searchix-web/main.go → cmd/searchix-web/main.go
@@ -35,6 +35,7 @@ )
 	generateErrorPage = flag.Bool("generate-error-page", false, "generate error page and exit")
 	dev               = flag.Bool("dev", false, "enable live reloading and nicer logging")
 	replace           = flag.Bool("replace", false, "replace existing index and exit")
+	fetch             = flag.Bool("fetch", false, "fetch data and exit")
 	version           = flag.Bool("version", false, "print version information")
 	cpuprofile        = flag.String("cpuprofile", "", "enable CPU profiling and save to `file`")
 )

@@ -144,13 +145,13 @@ if err != nil {
 		logger.Fatal("Failed to create importer", "error", err)
 	}
 
-	if !exists || *replace {
-		err := imp.Start(ctx, true, nil)
+	if !exists || *replace || *fetch {
+		err := imp.Start(ctx, true, *fetch, nil)
 		if err != nil {
 			logger.Fatal("Failed to start importer", "error", err)
 		}
 
-		if *replace {
+		if *replace || *fetch {
 			return
 		}
 	}
M internal/importer/main.go → internal/importer/main.go
@@ -62,6 +62,7 @@ func (imp *Importer) createSourceImporter(
 	parent context.Context,
 	meta *index.Meta,
 	forceUpdate bool,
+	fetchOnly bool,
 ) func(*config.Source) error {
 	return func(source *config.Source) error {
 		logger := imp.options.Logger.With("name", source.Key)

@@ -105,16 +106,18 @@ }
 		logger.Info(
 			"importer fetch succeeded",
 			"previous",
-			previousUpdate,
+			previousUpdate.Format(time.DateTime),
 			"current",
 			sourceMeta.Updated.Format(time.DateTime),
 			"is_updated",
 			sourceMeta.Updated.After(previousUpdate),
 			"update_force",
 			forceUpdate,
+			"fetch_only",
+			fetchOnly,
 		)
 
-		if sourceMeta.Updated.After(previousUpdate) || forceUpdate {
+		if !fetchOnly && (!sourceMeta.Updated.After(previousUpdate) || forceUpdate) {
 			var pdb *programs.DB
 
 			if source.Programs.Enable {

@@ -191,6 +194,7 @@
 func (imp *Importer) Start(
 	ctx context.Context,
 	forceUpdate bool,
+	fetchOnly bool,
 	onlyUpdateSources *[]string,
 ) error {
 	if len(imp.config.Importer.Sources) == 0 {

@@ -210,7 +214,7 @@ forceUpdate = forceUpdate || (onlyUpdateSources != nil && len(*onlyUpdateSources) > 0)
 
 	meta := imp.options.WriteIndex.Meta
 
-	importSource := imp.createSourceImporter(importCtx, meta, forceUpdate)
+	importSource := imp.createSourceImporter(importCtx, meta, forceUpdate, fetchOnly)
 	for name, source := range imp.config.Importer.Sources {
 		if onlyUpdateSources != nil && len(*onlyUpdateSources) > 0 {
 			if !slices.Contains(*onlyUpdateSources, name) {

@@ -267,6 +271,7 @@ imp.options.Logger.Info("adding new sources", "sources", newSources)
 			err := imp.Start(
 				ctx,
 				false,
+				false,
 				&newSources,
 			)
 			if err != nil {

@@ -337,7 +342,7 @@ }, monitorConfig)
 			MarkIndexingStarted()
 
 			ctx, cancel := context.WithTimeout(parentCtx, imp.config.Importer.Timeout.Duration)
-			err = imp.Start(ctx, false, nil)
+			err = imp.Start(ctx, false, false, nil)
 			cancel()
 
 			if err != nil {
M internal/importer/main_test.go → internal/importer/main_test.go
@@ -45,6 +45,7 @@
 	err = imp.Start(
 		context.Background(),
 		false,
+		false,
 		&[]string{"nixpkgs"},
 	)
 	if err != nil {