feat: stream files directly from fetcher to importer Use IndexMeta to store the information relevant to making conditional updates in future runs.
1 file changed, 22 insertions(+), 32 deletions(-)
changed files
M internal/fetcher/download.go → internal/fetcher/download.go
@@ -5,15 +5,13 @@ "context" "fmt" "log/slog" "net/url" - "path" "searchix/internal/config" - "searchix/internal/file" + "searchix/internal/index" "github.com/pkg/errors" ) type DownloadFetcher struct { - DataPath string Source *config.Source SourceFile string Logger *slog.Logger@@ -21,15 +19,13 @@ } func NewDownloadFetcher( source *config.Source, - dataPath string, logger *slog.Logger, ) (*DownloadFetcher, error) { switch source.Importer { case config.Options: return &DownloadFetcher{ - DataPath: dataPath, - Source: source, - Logger: logger, + Source: source, + Logger: logger, }, nil default: return nil, fmt.Errorf("unsupported importer type %s", source.Importer)@@ -42,22 +38,11 @@ "options": "options.json", } func (i *DownloadFetcher) FetchIfNeeded( - parent context.Context, -) (f FetchedFiles, updated bool, err error) { - ctx, cancel := context.WithTimeout(parent, i.Source.FetchTimeout.Duration) - defer cancel() - - root := i.DataPath - - err = file.Mkdirp(root) - if err != nil { - err = errors.WithMessagef(err, "error creating directory for data: %s", root) - - return - } - + ctx context.Context, + sourceMeta *index.SourceMeta, +) (f FetchedFiles, err error) { var fetchURL string - for _, filename := range files { + for key, filename := range files { fetchURL, err = url.JoinPath(i.Source.URL, filename) if err != nil { err = errors.WithMessagef(@@ -70,23 +55,28 @@ return } - outPath := path.Join(root, filename) - - i.Logger.Debug("preparing to fetch URL", "url", fetchURL, "path", outPath) + i.Logger.Debug("preparing to fetch URL", "url", fetchURL) - updated, err = fetchFileIfNeeded(ctx, outPath, fetchURL) + body, mtime, err := fetchFileIfNeeded(ctx, sourceMeta.Updated, fetchURL) if err != nil { - return + i.Logger.Warn("failed to fetch file", "url", fetchURL, "error", err) + + return f, err } // don't bother to issue requests for the later files - if !updated { + if mtime.Before(sourceMeta.Updated) { break } - } + sourceMeta.Updated = mtime - f = FetchedFiles{ - Revision: path.Join(root, "revision"), - Options: path.Join(root, "options.json"), + switch key { + case "revision": + f.Revision = body + case "options": + f.Options = body + default: + return f, errors.Errorf("unknown file kind %s", key) + } } return