all repos — searchix @ 53c9fb2f72cbc82177404b76a4aad14283f8ced1

Search engine for NixOS, nix-darwin, home-manager and NUR users

import/main.go (view raw)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
package main

import (
	"context"
	"errors"
	"log"
	"log/slog"
	"os"
	"os/exec"
	"path"
	"searchix/internal/config"
	"searchix/internal/importer"
	"searchix/internal/search"
	"strings"
	"time"

	"github.com/ardanlabs/conf/v3"
)

type Config struct {
	ConfigFile string        `conf:"short:c"`
	LogLevel   slog.Level    `conf:"default:INFO"`
	Timeout    time.Duration `conf:"default:30m,help:maximum time to wait for all fetchers and importers combined"`
	Replace    bool          `conf:"default:false,help:whether to remove existing database, if exists"`
}

func main() {
	if _, found := os.LookupEnv("DEBUG"); found {
		slog.SetLogLoggerLevel(slog.LevelDebug)
	}
	var runtimeConfig Config
	help, err := conf.Parse("", &runtimeConfig)
	if err != nil {
		if errors.Is(err, conf.ErrHelpWanted) {
			log.Println(help)
			os.Exit(1)
		}
		log.Panicf("parsing runtime configuration: %v", err)
	}
	slog.SetLogLoggerLevel(runtimeConfig.LogLevel)

	cfg, err := config.GetConfig(runtimeConfig.ConfigFile)
	if err != nil {
		log.Fatal(err)
	}

	if len(cfg.Sources) == 0 {
		slog.Info("No sources enabled")

		return
	}

	indexer, err := search.NewIndexer(cfg.DataPath, runtimeConfig.Replace)
	if err != nil {
		log.Fatalf("Failed to create indexer: %v", err)
	}

	ctx, cancel := context.WithTimeout(context.Background(), runtimeConfig.Timeout)
	defer cancel()

	var imp importer.Importer
	var hadErrors bool
	for name, source := range cfg.Sources {
		logger := slog.With("name", name, "importer", source.Type.String())
		logger.Debug("starting importer")

		importerDataPath := path.Join(cfg.DataPath, "sources", source.Channel)

		switch source.Type {
		case importer.ChannelNixpkgs:
			imp = importer.NewNixpkgsChannelImporter(source, importerDataPath, logger)
		case importer.Channel:
			imp = importer.NewChannelImporter(source, importerDataPath, logger)
		default:
			log.Printf("unsupported importer type %s", source.Type.String())

			continue
		}

		updated, err := imp.FetchIfNeeded(ctx)

		if err != nil {
			var exerr *exec.ExitError
			if errors.As(err, &exerr) {
				lines := strings.Split(strings.TrimSpace(string(exerr.Stderr)), "\n")
				for _, line := range lines {
					logger.Warn("importer fetch failed", "stderr", line, "status", exerr.ExitCode())
				}
			} else {
				logger.Warn("importer fetch failed", "error", err)
			}
			hadErrors = true

			continue
		}
		logger.Info("importer fetch succeeded", "updated", updated)

		if updated || runtimeConfig.Replace {
			hadWarnings, err := imp.Import(ctx, indexer)

			if err != nil {
				msg := err.Error()
				for _, line := range strings.Split(strings.TrimSpace(msg), "\n") {
					logger.Error("importer init failed", "error", line)
				}

				continue
			}
			if hadWarnings {
				logger.Warn("importer succeeded, but with warnings/errors")
			} else {
				logger.Info("importer succeeded")
			}
		}
	}

	err = indexer.Close()
	if err != nil {
		slog.Error("error closing indexer", "error", err)
	}

	if hadErrors {
		defer os.Exit(1)
	}
}