all repos — searchix @ 347521a6dce587e6b51f33608744125d853c7078

Search engine for NixOS, nix-darwin, home-manager and NUR users

internal/importer/importer.go (view raw)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
package importer

import (
	"context"
	"sync"

	"alin.ovh/x/log"

	"alin.ovh/searchix/internal/config"
	"alin.ovh/searchix/internal/nix"
	"alin.ovh/searchix/internal/storage"
)

type Processor interface {
	Process(context.Context) (<-chan nix.Importable, <-chan error)
}

type (
	ImportSource      func(context.Context) (<-chan nix.Importable, <-chan error)
	ImportDestination func(context.Context, <-chan nix.Importable) <-chan error
)

func pipe(
	ctx context.Context,
	log *log.Logger,
	src ImportSource,
	dst ImportDestination,
) (bool, error) {
	wg := sync.WaitGroup{}

	wg.Add(1)
	objects, srcErrs := src(ctx)

	wg.Add(1)
	dstErrors := dst(ctx, objects)

	var hadObjectErrors bool
	var criticalError error
	go func() {
		for {
			select {
			case err, running := <-srcErrs:
				if !running {
					wg.Done()
					srcErrs = nil
					log.Debug("processing completed")

					continue
				}
				hadObjectErrors = true
				log.Warn("error processing object from source", "error", err)
			case err, running := <-dstErrors:
				if !running {
					wg.Done()
					dstErrors = nil

					continue
				}
				hadObjectErrors = true
				log.Warn("error writing object to target", "error", err)
			}
		}
	}()

	wg.Wait()

	return hadObjectErrors, criticalError
}

func (imp *Importer) indexSource(
	ctx context.Context,
	source *config.Source,
) (bool, error) {
	return pipe(
		ctx,
		imp.options.Logger,
		storage.MakeSourceExporter(imp.options.Storage, source),
		func(ctx context.Context, objects <-chan nix.Importable) <-chan error {
			return imp.options.WriteIndex.Import(ctx, objects)
		},
	)
}