all repos — searchix @ bfa2851e2a6d1800569ea5c451242bdf7f96dcac

Search engine for NixOS, nix-darwin, home-manager and NUR users

internal/fetcher/download.go (view raw)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
package fetcher

import (
	"context"
	"net/url"

	"github.com/Southclaws/fault"
	"github.com/Southclaws/fault/fmsg"

	"alin.ovh/searchix/internal/config"
	"alin.ovh/searchix/internal/fetcher/http"
	"alin.ovh/searchix/internal/index"
)

type DownloadFetcher struct {
	Source     config.Source
	SourceFile string
	*Options
}

func NewDownloadFetcher(
	source config.Source,
	options *Options,
) (*DownloadFetcher, error) {
	switch source.Importer {
	case config.Options, config.Packages:
		return &DownloadFetcher{
			Source:  source,
			Options: options,
		}, nil
	default:
		return nil, fault.Newf("unsupported importer type %s", source.Importer)
	}
}

var files = map[string]string{
	"revision": "revision",
	"options":  "options.json",
	"packages": "packages.json",
}

func (i *DownloadFetcher) FetchIfNeeded(
	ctx context.Context,
	sourceMeta *index.SourceMeta,
) (*FetchedFiles, error) {
	f := &FetchedFiles{}

	filesToFetch := make([]string, 2)
	filesToFetch[0] = files["revision"]
	switch i.Source.Importer {
	case config.Packages:
		filesToFetch[1] = files["packages"]
	case config.Options:
		filesToFetch[1] = files["options"]
	}

	fetcher := http.NewFetcher(&http.Options{
		Logger: i.Logger.Named("http"),
		Root:   i.Root,
	})

	for _, basename := range filesToFetch {
		target := i.Source.JoinPath(basename)
		fetchURL, baseErr := url.JoinPath(i.Source.URL, basename)
		if baseErr != nil {
			return nil, fault.Wrap(
				baseErr,
				fmsg.Withf(
					"could not build URL with elements %s and %s",
					i.Source.URL,
					basename,
				),
			)
		}

		i.Logger.Debug("preparing to fetch URL", "url", fetchURL, "target", target)

		body, err := fetcher.FetchFileIfNeeded(ctx, target, fetchURL)
		if err != nil {
			i.Logger.Warn("failed to fetch file", "url", fetchURL, "error", err)

			return nil, fault.Wrap(err, fmsg.Withf("could not fetch file %s", basename))
		}

		stat, err := i.Root.Stat(target)
		if err != nil {
			return nil, fault.Wrap(err, fmsg.Withf("could not stat file %s", target))
		}
		sourceMeta.UpdatedAt = stat.ModTime()

		switch basename {
		case files["revision"]:
			f.Revision = body
		case files["options"]:
			f.Options = body
		case files["packages"]:
			f.Packages = body
		default:
			return f, fault.Newf("unknown filename %s", basename)
		}
	}

	return f, nil
}