cmd/searchix-web/ingest.go (view raw)
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 | package main
import (
"context"
"os"
"os/signal"
"github.com/Southclaws/fault"
"github.com/Southclaws/fault/fmsg"
"alin.ovh/searchix/internal/file"
"alin.ovh/searchix/internal/importer"
"alin.ovh/searchix/internal/index"
"alin.ovh/searchix/internal/manpages"
"alin.ovh/searchix/internal/storage"
)
type IngestOptions struct {
Fetch bool `long:"fetch" description:"pre-fetch data"`
Offline bool `long:"offline" description:"offline mode"`
Replace bool `long:"replace" description:"replace existing storage"`
Reindex bool `long:"reindex" description:"reindex existing index"`
}
func (opts *IngestOptions) Execute(_ []string) error {
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
defer cancel()
root, err := file.CreateAndOpenRoot(cfg.DataPath)
if err != nil {
return fault.Wrap(err, fmsg.With("Failed to open data root"))
}
defer root.Close()
store, err := storage.New(&storage.Options{
Root: root,
Logger: logger.Named("store"),
})
if err != nil {
return fault.Wrap(err, fmsg.With("Failed to create store"))
}
defer store.Close()
_, write, exists, err := index.OpenOrCreate(
&index.Options{
Config: cfg,
Force: opts.Reindex,
LowMemory: cfg.Importer.LowMemory,
BatchSize: cfg.Importer.BatchSize,
Logger: logger.Named("index"),
Root: root,
Store: store,
},
)
if err != nil {
return fault.Wrap(err, fmsg.With("Failed to open or create index"))
}
mdb := manpages.New(&manpages.Options{
Logger: logger.Named("manpages"),
Root: root,
})
imp, err := importer.New(cfg, &importer.Options{
Storage: store,
WriteIndex: write,
LowMemory: cfg.Importer.LowMemory,
Logger: logger.Named("importer"),
Manpages: mdb,
Root: root,
Offline: opts.Offline,
})
if err != nil {
return fault.Wrap(err, fmsg.With("Failed to create importer"))
}
if !exists || opts.Replace || opts.Fetch {
importer.MarkImportStarted()
err = imp.Fetch(ctx, true, opts.Fetch && !opts.Replace, nil)
if err != nil {
return fault.Wrap(err, fmsg.With("Failed to start importer"))
}
importer.MarkImportFinished()
importer.MarkLastRun(write.Meta)
err = write.SaveMeta()
if err != nil {
return fault.Wrap(err, fmsg.With("Failed to save index metadata"))
}
}
if !exists || opts.Reindex {
err = imp.Index(ctx)
if err != nil {
return fault.Wrap(err, fmsg.With("Failed to index data"))
}
}
if opts.Replace || opts.Reindex {
err = imp.Prune(ctx)
if err != nil {
return fault.Wrap(err, fmsg.With("Failed to prune index"))
}
}
return nil
}
func init() {
var opts IngestOptions
cmd, err := parser.AddCommand(
"ingest",
"Ingest data",
"Fetch, store and index data",
&opts,
)
if err != nil {
panic(err)
}
cmd.Aliases = []string{"import"}
}
|