cmd/searchix-web/ingest.go (view raw)
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 | package main
import (
"context"
"os"
"os/signal"
"github.com/Southclaws/fault"
"github.com/Southclaws/fault/fmsg"
"alin.ovh/searchix/internal/file"
"alin.ovh/searchix/internal/importer"
"alin.ovh/searchix/internal/index"
"alin.ovh/searchix/internal/manpages"
)
type IngestOptions struct {
Fetch bool `long:"fetch" description:"pre-fetch data"`
Offline bool `long:"offline" description:"offline mode"`
Replace bool `long:"replace" description:"replace existing index"`
}
func (opts *IngestOptions) Execute(_ []string) error {
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
defer cancel()
root, err := file.CreateAndOpenRoot(cfg.DataPath)
if err != nil {
return fault.Wrap(err, fmsg.With("Failed to open data root"))
}
defer root.Close()
read, write, err := index.OpenOrCreate(
&index.Options{
Config: cfg,
Force: opts.Replace,
LowMemory: cfg.Importer.LowMemory,
BatchSize: cfg.Importer.BatchSize,
Logger: logger.Named("index"),
Root: root,
},
)
if err != nil {
return fault.Wrap(err, fmsg.With("Failed to open or create index"))
}
mdb := manpages.New(&manpages.Options{
Logger: logger.Named("manpages"),
Root: root,
})
imp, err := importer.New(cfg, &importer.Options{
ReadIndex: read,
WriteIndex: write,
LowMemory: cfg.Importer.LowMemory,
Logger: logger.Named("importer"),
Manpages: mdb,
Root: root,
Offline: opts.Offline,
})
if err != nil {
return fault.Wrap(err, fmsg.With("Failed to create importer"))
}
if !opts.Offline && (!write.Exists() || opts.Fetch || opts.Replace) {
importer.MarkImportStarted()
err = imp.Fetch(ctx, true, nil)
if err != nil {
return fault.Wrap(err, fmsg.With("Failed to start importer"))
}
importer.MarkImportFinished()
importer.MarkLastRun(write.Meta)
err = write.SaveMeta()
if err != nil {
return fault.Wrap(err, fmsg.With("Failed to save index metadata"))
}
}
if !write.Exists() || opts.Replace {
err = imp.Index(ctx, nil)
if err != nil {
return fault.Wrap(err, fmsg.With("Failed to index data"))
}
}
return nil
}
func init() {
var opts IngestOptions
cmd, err := parser.AddCommand(
"ingest",
"Ingest data",
"Fetch, store and index data",
&opts,
)
if err != nil {
panic(err)
}
cmd.Aliases = []string{"import"}
}
|