all repos — searchix @ 0dc3ac14c8cd367a415bc14f5e0dd682420e5480

Search engine for NixOS, nix-darwin, home-manager and NUR users

cmd/searchix-web/ingest.go (view raw)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
package main

import (
	"context"
	"os"
	"os/signal"

	"github.com/Southclaws/fault"
	"github.com/Southclaws/fault/fmsg"

	"alin.ovh/searchix/internal/file"
	"alin.ovh/searchix/internal/importer"
	"alin.ovh/searchix/internal/index"
	"alin.ovh/searchix/internal/manpages"
	"alin.ovh/searchix/internal/storage"
)

type IngestOptions struct {
	Fetch   bool `long:"fetch"   description:"pre-fetch data"`
	Offline bool `long:"offline" description:"offline mode"`
	Replace bool `long:"replace" description:"replace existing storage"`
	Reindex bool `long:"reindex" description:"reindex existing index"`
}

func (opts *IngestOptions) Execute(_ []string) error {
	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
	defer cancel()

	root, err := file.CreateAndOpenRoot(cfg.DataPath)
	if err != nil {
		return fault.Wrap(err, fmsg.With("Failed to open data root"))
	}
	defer root.Close()

	store, err := storage.New(&storage.Options{
		Root:   root,
		Logger: logger.Named("store"),
	})
	if err != nil {
		return fault.Wrap(err, fmsg.With("Failed to create store"))
	}
	defer store.Close()

	_, write, exists, err := index.OpenOrCreate(
		&index.Options{
			Config:    cfg,
			Force:     opts.Reindex,
			LowMemory: cfg.Importer.LowMemory,
			BatchSize: cfg.Importer.BatchSize,
			Logger:    logger.Named("index"),
			Root:      root,
			Store:     store,
		},
	)
	if err != nil {
		return fault.Wrap(err, fmsg.With("Failed to open or create index"))
	}

	mdb := manpages.New(&manpages.Options{
		Logger: logger.Named("manpages"),
		Root:   root,
	})

	imp, err := importer.New(cfg, &importer.Options{
		Storage:    store,
		WriteIndex: write,
		LowMemory:  cfg.Importer.LowMemory,
		Logger:     logger.Named("importer"),
		Manpages:   mdb,
		Root:       root,
		Offline:    opts.Offline,
	})
	if err != nil {
		return fault.Wrap(err, fmsg.With("Failed to create importer"))
	}

	if !exists || opts.Replace || opts.Fetch {
		err = imp.Fetch(ctx, true, opts.Fetch && !opts.Replace, nil)
		if err != nil {
			return fault.Wrap(err, fmsg.With("Failed to start importer"))
		}
	}

	if !exists || opts.Reindex {
		err = imp.Index(ctx)
		if err != nil {
			return fault.Wrap(err, fmsg.With("Failed to index data"))
		}
	}

	return nil
}

func init() {
	var opts IngestOptions

	cmd, err := parser.AddCommand(
		"ingest",
		"Ingest data",
		"Fetch, store and index data",
		&opts,
	)
	if err != nil {
		panic(err)
	}

	cmd.Aliases = []string{"import"}
}