all repos — searchix @ 9b430ab18dc024058ea9c039fb64af0a3d5e3529

Search engine for NixOS, nix-darwin, home-manager and NUR users

internal/importer/importer.go (view raw)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
package importer

import (
	"context"
	"log/slog"
	"searchix/internal/config"
	"searchix/internal/index"
	"sync"
)

type Importer interface {
	FetchIfNeeded(context.Context) (bool, error)
	Import(context.Context, *index.WriteIndex) (bool, error)
}

func NewNixpkgsChannelImporter(
	source *config.Source,
	dataPath string,
	logger *slog.Logger,
) *NixpkgsChannelImporter {
	return &NixpkgsChannelImporter{
		DataPath: dataPath,
		Source:   source,
		Logger:   logger,
	}
}

func NewChannelImporter(
	source *config.Source,
	dataPath string,
	logger *slog.Logger,
) *ChannelImporter {
	return &ChannelImporter{
		DataPath: dataPath,
		Source:   source,
		Logger:   logger,
	}
}

func NewDownloadOptionsImporter(
	source *config.Source,
	dataPath string,
	logger *slog.Logger,
) *DownloadOptionsImporter {
	return &DownloadOptionsImporter{
		DataPath: dataPath,
		Source:   source,
		Logger:   logger,
	}
}

type importConfig struct {
	Filename string
	Source   *config.Source
	Logger   *slog.Logger
}

func processOptions(
	parent context.Context,
	indexer *index.WriteIndex,
	conf *importConfig,
) (bool, error) {
	ctx, cancel := context.WithTimeout(parent, conf.Source.ImportTimeout)
	defer cancel()

	conf.Logger.Debug("creating option processor", "filename", conf.Filename)
	processor, err := NewOptionProcessor(conf.Filename, conf.Source)
	if err != nil {
		return true, err
	}

	wg := sync.WaitGroup{}

	wg.Add(1)
	options, pErrs := processor.Process(ctx)

	wg.Add(1)
	iErrs := indexer.Import(ctx, options)

	var hadErrors bool
	go func() {
		for {
			select {
			case err, running := <-iErrs:
				if !running {
					wg.Done()
					iErrs = nil
					conf.Logger.Info("ingest completed")

					continue
				}
				hadErrors = true
				conf.Logger.Warn("error ingesting option", "error", err)
			case err, running := <-pErrs:
				if !running {
					wg.Done()
					pErrs = nil
					conf.Logger.Debug("processing completed")

					continue
				}
				hadErrors = true
				conf.Logger.Warn("error processing option", "error", err)
			}
		}
	}()

	conf.Logger.Debug("options processing", "state", "waiting")
	wg.Wait()
	conf.Logger.Debug("options processing", "state", "complete")

	return hadErrors, nil
}