all repos — searchix @ e055ed6177bdec5fb286724a8e6b6e8ad715cca1

Search engine for NixOS, nix-darwin, home-manager and NUR users

refactor: use iterator instead of channel for import

Alan Pearce
commit

e055ed6177bdec5fb286724a8e6b6e8ad715cca1

parent

c4d3d6813a2ec8ba79a7923507db9df7f3406582

1 file changed, 99 insertions(+), 89 deletions(-)

changed files
M internal/importer/options.gointernal/importer/options.go
@@ -3,6 +3,7 @@
import ( "context" "io" + "iter" "reflect" "strings" "time"
@@ -55,6 +56,7 @@ dec *jstream.Decoder
log *log.Logger infile io.ReadCloser source config.Source + errs []error } func NewOptionProcessor(
@@ -72,112 +74,120 @@ }
func (i *OptionIngester) Process( ctx context.Context, - results chan<- index.Indexable, - errs chan<- error, -) { - defer i.infile.Close() +) iter.Seq[index.Indexable] { + return func(yield func(index.Indexable) bool) { + defer i.infile.Close() -outer: - for mv := range i.dec.Stream() { - select { - case <-ctx.Done(): - break outer - default: - } - if err := i.dec.Err(); err != nil { - errs <- fault.Wrap(err, fmsg.With("could not decode JSON")) + for mv := range i.dec.Stream() { + if err := i.dec.Err(); err != nil { + i.errs = append(i.errs, fault.Wrap(err, fmsg.With("could not decode JSON"))) - continue - } - if mv.ValueType != jstream.Object { - errs <- fault.Newf("unexpected object type %s", ValueTypeToString(mv.ValueType)) + continue + } + if mv.ValueType != jstream.Object { + i.errs = append( + i.errs, + fault.Newf("unexpected object type %s", ValueTypeToString(mv.ValueType)), + ) - continue - } - kv := mv.Value.(jstream.KV) - x := kv.Value.(map[string]any) + continue + } + kv := mv.Value.(jstream.KV) + x := kv.Value.(map[string]any) - var decls []nix.Link - for _, decl := range x["declarations"].([]any) { - switch decl := decl.(type) { - case string: - s := decl - link, err := MakeChannelLink(i.source.Repo, s) - if err != nil { - errs <- fault.Wrap(err, fmsg.Withf("could not make a channel link for channel %s, revision %s and subpath %s", - i.source.Channel, i.source.Repo.Revision, s, - )) + var decls []nix.Link + for _, decl := range x["declarations"].([]any) { + switch decl := decl.(type) { + case string: + s := decl + link, err := MakeChannelLink(i.source.Repo, s) + if err != nil { + i.errs = append(i.errs, fault.Wrap(err, fmsg.Withf("could not make a channel link for channel %s, revision %s and subpath %s", + i.source.Channel, i.source.Repo.Revision, s, + ))) + + continue + } + decls = append(decls, *link) + case map[string]any: + v := decl + link := nix.Link{ + Name: v["name"].(string), + URL: v["url"].(string), + } + decls = append(decls, link) + default: + i.errs = append(i.errs, fault.Newf("unexpected declaration type %s", reflect.TypeOf(decl).String())) continue } - decls = append(decls, *link) - case map[string]any: - v := decl - link := nix.Link{ - Name: v["name"].(string), - URL: v["url"].(string), - } - decls = append(decls, link) - default: - errs <- fault.Newf("unexpected declaration type %s", reflect.TypeOf(decl).String()) - - continue + } + var description string + if v, ok := x["description"].(string); ok { + description = v } - } - var description string - if v, ok := x["description"].(string); ok { - description = v - } - var optType string - if v, ok := x["type"].(string); ok { - optType = v - } + var optType string + if v, ok := x["type"].(string); ok { + optType = v + } - var relatedPackages string - if v, ok := x["relatedPackages"].(string); ok { - relatedPackages = v - } + var relatedPackages string + if v, ok := x["relatedPackages"].(string); ok { + relatedPackages = v + } - var loc []string - if v, ok := x["loc"].([]any); ok { - loc = make([]string, len(v)) - for i, item := range v { - if s, ok := item.(string); ok { - loc[i] = s + var loc []string + if v, ok := x["loc"].([]any); ok { + loc = make([]string, len(v)) + for i, item := range v { + if s, ok := item.(string); ok { + loc[i] = s + } } } - } - var defaultDocs *nix.Docs - if v, ok := x["default"].(map[string]any); ok { - defaultDocs = i.convertDocsValue(v) - } + var defaultDocs *nix.Docs + if v, ok := x["default"].(map[string]any); ok { + defaultDocs = i.convertDocsValue(v) + } - var exampleDocs *nix.Docs - if v, ok := x["example"].(map[string]any); ok { - exampleDocs = i.convertDocsValue(v) - } + var exampleDocs *nix.Docs + if v, ok := x["example"].(map[string]any); ok { + exampleDocs = i.convertDocsValue(v) + } - // Calculate parents - var parents string - if len(loc) > 1 { - parents = strings.Join(loc[:len(loc)-1], ".") + var parents string + if len(loc) > 1 { + parents = strings.Join(loc[:len(loc)-1], ".") + } + + // log.Debug("sending option", "name", kv.Key) + if !yield(nix.Option{ + Name: kv.Key, + Source: i.source.Key, + Declarations: decls, + Default: defaultDocs, + Description: nixdocs.Markdown(description), + Example: exampleDocs, + RelatedPackages: nixdocs.Markdown(relatedPackages), + Loc: loc, + Parents: parents, + Type: optType, + ImportedAt: time.Now(), + }) { + return + } } + } +} - // log.Debug("sending option", "name", kv.Key) - results <- nix.Option{ - Name: kv.Key, - Source: i.source.Key, - Declarations: decls, - Default: defaultDocs, - Description: nixdocs.Markdown(description), - Example: exampleDocs, - RelatedPackages: nixdocs.Markdown(relatedPackages), - Loc: loc, - Parents: parents, - Type: optType, - ImportedAt: time.Now(), - } +func (i *OptionIngester) Err() error { + if len(i.errs) == 0 { + return nil + } + if len(i.errs) == 1 { + return i.errs[0] } + return fault.Newf("encountered %d errors during processing", len(i.errs)) }