cmd/searchix-web/ingest.go (view raw)
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 | package main import ( "context" "os" "os/signal" "github.com/Southclaws/fault" "github.com/Southclaws/fault/fmsg" "alin.ovh/searchix/internal/file" "alin.ovh/searchix/internal/importer" "alin.ovh/searchix/internal/index" "alin.ovh/searchix/internal/manpages" "alin.ovh/searchix/internal/storage" ) type IngestOptions struct { Fetch bool `long:"fetch" description:"pre-fetch data"` Offline bool `long:"offline" description:"offline mode"` Replace bool `long:"replace" description:"replace existing storage"` Reindex bool `long:"reindex" description:"reindex existing index"` } func (opts *IngestOptions) Execute(_ []string) error { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) defer cancel() root, err := file.CreateAndOpenRoot(cfg.DataPath) if err != nil { return fault.Wrap(err, fmsg.With("Failed to open data root")) } defer root.Close() store, err := storage.New(&storage.Options{ Root: root, Logger: logger.Named("store"), }) if err != nil { return fault.Wrap(err, fmsg.With("Failed to create store")) } defer store.Close() _, write, exists, err := index.OpenOrCreate( &index.Options{ Config: cfg, Force: opts.Reindex, LowMemory: cfg.Importer.LowMemory, BatchSize: cfg.Importer.BatchSize, Logger: logger.Named("index"), Root: root, Store: store, }, ) if err != nil { return fault.Wrap(err, fmsg.With("Failed to open or create index")) } mdb := manpages.New(&manpages.Options{ Logger: logger.Named("manpages"), Root: root, }) imp, err := importer.New(cfg, &importer.Options{ Storage: store, WriteIndex: write, LowMemory: cfg.Importer.LowMemory, Logger: logger.Named("importer"), Manpages: mdb, Root: root, Offline: opts.Offline, }) if err != nil { return fault.Wrap(err, fmsg.With("Failed to create importer")) } if !exists || opts.Replace || opts.Fetch { err = imp.Fetch(ctx, true, opts.Fetch && !opts.Replace, nil) if err != nil { return fault.Wrap(err, fmsg.With("Failed to start importer")) } } if !exists || opts.Reindex { for _, source := range cfg.Importer.Sources { hadErrors, err := imp.IndexSource(ctx, source) if err != nil { return fault.Wrap(err, fmsg.Withf("Failed to import source %s", source.Name)) } if hadErrors { logger.Warn("Imported source encountered errors", "source", source.Name) } } } return nil } func init() { var opts IngestOptions cmd, err := parser.AddCommand( "ingest", "Ingest data", "Fetch, store and index data", &opts, ) if err != nil { panic(err) } cmd.Aliases = []string{"import"} } |