refactor: make data munging more explicit
5 files changed, 149 insertions(+), 153 deletions(-)
M go.mod → go.mod
@@ -17,7 +17,6 @@ github.com/ecnepsnai/sdnotify v1.0.0 github.com/fsnotify/fsnotify v1.9.0 github.com/getsentry/sentry-go v0.33.0 github.com/jessevdk/go-flags v1.6.1 - github.com/mitchellh/mapstructure v1.5.0 github.com/osdevisnot/sorvor v0.4.4 github.com/pelletier/go-toml/v2 v2.2.4 github.com/stefanfritsch/goldmark-fences v1.0.0
M go.sum → go.sum
@@ -93,8 +93,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
M gomod2nix.toml → gomod2nix.toml
@@ -115,9 +115,6 @@ hash = "sha256-To8A0h+lbfZ/6zM+2PpRpY3+L6725OPC66lffq6fUoM=" [mod."github.com/mattn/go-isatty"] version = "v0.0.20" hash = "sha256-qhw9hWtU5wnyFyuMbKx+7RB8ckQaFQ8D+8GKPkN3HHQ=" - [mod."github.com/mitchellh/mapstructure"] - version = "v1.5.0" - hash = "sha256-ztVhGQXs67MF8UadVvG72G3ly0ypQW0IRDdOOkjYwoE=" [mod."github.com/modern-go/concurrent"] version = "v0.0.0-20180306012644-bacd9c7ef1dd" hash = "sha256-OTySieAgPWR4oJnlohaFTeK1tRaVp/b0d1rYY8xKMzo="
M internal/importer/options.go → internal/importer/options.go
@@ -16,57 +16,44 @@ "github.com/Southclaws/fault" "github.com/Southclaws/fault/fmsg" "github.com/bcicen/jstream" - "github.com/mitchellh/mapstructure" ) -type nixDocJSON struct { - Type string `mapstructure:"_type"` - Text string -} +func (i *OptionIngester) convertDocsValue(docMap map[string]any) *nix.Docs { + if docMap == nil { + return nil + } -type linkJSON struct { - Name string - URL string `json:"url"` -} + var docType string + if t, ok := docMap["_type"].(string); ok { + docType = t + } -type nixOptionJSON struct { - Declarations []linkJSON - Default *nixDocJSON - Description string - Example *nixDocJSON - Loc []string - ReadOnly bool - RelatedPackages string - Type string -} + var text string + if t, ok := docMap["text"].(string); ok { + text = t + } -func (i *OptionIngester) convertDocsValue(nj *nixDocJSON) *nix.Docs { - if nj == nil { - return nil - } - switch nj.Type { + switch docType { case "", "literalExpression": return &nix.Docs{ - Plain: nj.Text, + Plain: text, } case "literalMD": return &nix.Docs{ - Markdown: nix.Markdown(nj.Text), + Markdown: nix.Markdown(text), } default: - i.log.Warn("got unexpected docs type", "type", nj.Type, "text", nj.Text) + i.log.Warn("got unexpected docs type", "type", docType, "text", text) return nil } } type OptionIngester struct { - dec *jstream.Decoder - ms *mapstructure.Decoder - log *log.Logger - optJSON nixOptionJSON - infile io.ReadCloser - source config.Source + dec *jstream.Decoder + log *log.Logger + infile io.ReadCloser + source config.Source } func NewOptionProcessor(@@ -74,27 +61,12 @@ infile io.ReadCloser, source config.Source, log *log.Logger, ) *OptionIngester { - i := &OptionIngester{ - dec: jstream.NewDecoder(infile, source.JSONDepth).EmitKV(), - log: log, - optJSON: nixOptionJSON{}, - infile: infile, - source: source, + return &OptionIngester{ + dec: jstream.NewDecoder(infile, source.JSONDepth).EmitKV(), + log: log, + infile: infile, + source: source, } - - ms, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ - ErrorUnused: true, - ZeroFields: true, - Result: &i.optJSON, - Squash: true, - DecodeHook: mapstructure.TextUnmarshallerHookFunc(), - }) - if err != nil { - panic("could not create mapstructure decoder: " + err.Error()) - } - i.ms = ms - - return i } func (i *OptionIngester) Process(@@ -124,11 +96,11 @@ } kv := mv.Value.(jstream.KV) x := kv.Value.(map[string]any) - var decls []*nix.Link + var decls []nix.Link for _, decl := range x["declarations"].([]any) { - switch decl := reflect.ValueOf(decl); decl.Kind() { - case reflect.String: - s := decl.String() + switch decl := decl.(type) { + case string: + s := decl link, err := MakeChannelLink(i.source.Repo, s) if err != nil { errs <- fault.Wrap(err, fmsg.Withf("could not make a channel link for channel %s, revision %s and subpath %s",@@ -137,49 +109,73 @@ )) continue } - decls = append(decls, link) - case reflect.Map: - v := decl.Interface().(map[string]any) + decls = append(decls, *link) + case map[string]any: + v := decl link := nix.Link{ Name: v["name"].(string), URL: v["url"].(string), } - decls = append(decls, &link) + decls = append(decls, link) default: - errs <- fault.Newf("unexpected declaration type %s", decl.Kind().String()) + errs <- fault.Newf("unexpected declaration type %s", reflect.TypeOf(decl).String()) continue } } - if len(decls) > 0 { - x["declarations"] = decls + var description string + if v, ok := x["description"].(string); ok { + description = v } - i.optJSON = nixOptionJSON{} - err := i.ms.Decode(x) // stores in optJSON - if err != nil { - errs <- fault.Wrap(err, fmsg.Withf("failed to decode option %#v", x)) + var optType string + if v, ok := x["type"].(string); ok { + optType = v + } - continue + var relatedPackages string + if v, ok := x["relatedPackages"].(string); ok { + relatedPackages = v } - decs := make([]nix.Link, len(i.optJSON.Declarations)) - for i, d := range i.optJSON.Declarations { - decs[i] = nix.Link(d) + var loc []string + if v, ok := x["loc"].([]any); ok { + loc = make([]string, len(v)) + for i, item := range v { + if s, ok := item.(string); ok { + loc[i] = s + } + } + } + + var defaultDocs *nix.Docs + if v, ok := x["default"].(map[string]any); ok { + defaultDocs = i.convertDocsValue(v) + } + + var exampleDocs *nix.Docs + if v, ok := x["example"].(map[string]any); ok { + exampleDocs = i.convertDocsValue(v) + } + + // Calculate parents + var parents string + if len(loc) > 1 { + parents = strings.Join(loc[:len(loc)-1], ".") } // log.Debug("sending option", "name", kv.Key) results <- nix.Option{ Name: kv.Key, Source: i.source.Key, - Declarations: decs, - Default: i.convertDocsValue(i.optJSON.Default), - Description: nix.Markdown(i.optJSON.Description), - Example: i.convertDocsValue(i.optJSON.Example), - RelatedPackages: nix.Markdown(i.optJSON.RelatedPackages), - Loc: i.optJSON.Loc, - Parents: strings.Join(i.optJSON.Loc[:len(i.optJSON.Loc)-1], "."), - Type: i.optJSON.Type, + Declarations: decls, + Default: defaultDocs, + Description: nix.Markdown(description), + Example: exampleDocs, + RelatedPackages: nix.Markdown(relatedPackages), + Loc: loc, + Parents: parents, + Type: optType, ImportedAt: time.Now(), } }
M internal/importer/package.go → internal/importer/package.go
@@ -19,36 +19,11 @@ "github.com/Southclaws/fault" "github.com/Southclaws/fault/fmsg" "github.com/bcicen/jstream" - "github.com/mitchellh/mapstructure" ) -type packageJSON struct { - Name string `mapstructure:"pname"` - Meta metaJSON - Version string -} - -type metaJSON struct { - Broken bool - Description string - LongDescription string - Homepages []string `mapstructure:"homepage"` - MainProgram string - Maintainers []maintainerJSON - Platforms []string - Position string -} - -type maintainerJSON struct { - Github string - Name string -} - type PackageIngester struct { dec *jstream.Decoder - ms *mapstructure.Decoder log *log.Logger - pkg packageJSON infile io.ReadCloser source config.Source programs *programs.DB@@ -75,27 +50,13 @@ source config.Source, log *log.Logger, programsDB *programs.DB, ) *PackageIngester { - i := &PackageIngester{ + return &PackageIngester{ dec: jstream.NewDecoder(infile, source.JSONDepth).EmitKV(), log: log, - pkg: packageJSON{}, infile: infile, source: source, programs: programsDB, } - - ms, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ - ZeroFields: true, - Result: &i.pkg, - Squash: true, - DecodeHook: mapstructure.TextUnmarshallerHookFunc(), - }) - if err != nil { - panic("could not create mapstructure decoder: " + err.Error()) - } - i.ms = ms - - return i } func convertToLicense(in map[string]any) *nix.License {@@ -276,25 +237,70 @@ } meta["maintainers"] = maints } - i.pkg = packageJSON{} - if err := i.ms.Decode(x); err != nil { // stores in i.pkg - errs <- fault.Wrap(err, fmsg.Withf("failed to decode package %#v", x)) + // Extract package name + var pkgName string + if pname, ok := x["pname"].(string); ok { + pkgName = pname + } + + // Extract version + var version string + if v, ok := x["version"].(string); ok { + version = v + } + + // Extract meta fields + var broken bool + if v, ok := meta["broken"].(bool); ok { + broken = v + } + + var description string + if v, ok := meta["description"].(string); ok { + description = v + } + + var longDescription string + if v, ok := meta["longDescription"].(string); ok { + longDescription = v + } + + var homepages []string + if v, ok := meta["homepage"].([]string); ok { + homepages = v + } else if v, ok := meta["homepage"].([]any); ok { + homepages = make([]string, len(v)) + for i, h := range v { + if s, ok := h.(string); ok { + homepages[i] = s + } + } + } + + var mainProgram string + if v, ok := meta["mainProgram"].(string); ok { + mainProgram = v + } + + var platforms []string + if v, ok := meta["platforms"].([]any); ok { + platforms = make([]string, len(v)) + for i, p := range v { + if s, ok := p.(string); ok { + platforms[i] = s + } + } + } - continue + var position string + if v, ok := meta["position"].(string); ok { + position = v } if i.source.Programs.Enable { programs, err = i.programs.GetPackagePrograms(ctx, kv.Key) if err != nil { - errs <- fault.Wrap(err, fmsg.Withf("failed to get programs for package %s", i.pkg.Name)) - } - } - - maintainers := make([]nix.Maintainer, len(i.pkg.Meta.Maintainers)) - for i, m := range i.pkg.Meta.Maintainers { - maintainers[i] = nix.Maintainer{ - Name: m.Name, - Github: m.Github, + errs <- fault.Wrap(err, fmsg.Withf("failed to get programs for package %s", pkgName)) } }@@ -304,36 +310,36 @@ pkgSet = "" } var definition string - if i.pkg.Meta.Position != "" { - defURL, err := url.Parse(i.pkg.Meta.Position) + if position != "" { + defURL, err := url.Parse(position) if err != nil { errs <- fault.Wrap(err, fmsg.Withf("failed to parse source URL %s", definition)) } if defURL.IsAbs() { - definition = i.pkg.Meta.Position + definition = position } else { - subpath, line, _ := strings.Cut(i.pkg.Meta.Position, ":") + subpath, line, _ := strings.Cut(position, ":") definition, err = i.source.Repo.GetFileURL(subpath, line) if err != nil { - errs <- fault.Wrap(err, fmsg.Withf("failed to make repo URL for package %s", i.pkg.Name)) + errs <- fault.Wrap(err, fmsg.Withf("failed to make repo URL for package %s", pkgName)) } } } results <- nix.Package{ - Name: i.pkg.Name, + Name: pkgName, Attribute: strings.TrimPrefix(kv.Key, "nur.repos."), Source: i.source.Key, PackageSet: pkgSet, - Version: i.pkg.Version, - Broken: i.pkg.Meta.Broken, - Description: i.pkg.Meta.Description, - LongDescription: nix.Markdown(i.pkg.Meta.LongDescription), - Homepages: i.pkg.Meta.Homepages, - MainProgram: i.pkg.Meta.MainProgram, - Platforms: i.pkg.Meta.Platforms, + Version: version, + Broken: broken, + Description: description, + LongDescription: nix.Markdown(longDescription), + Homepages: homepages, + MainProgram: mainProgram, + Platforms: platforms, Licenses: licenses, - Maintainers: maintainers, + Maintainers: maints, Definition: definition, Programs: programs, ImportedAt: time.Now(),