all repos — searchix @ d804ce1e5023d7c0ed5505736a3b553f6c81d7da

Search engine for NixOS, nix-darwin, home-manager and NUR users

refactor: re-arrange importer package

Alan Pearce
commit

d804ce1e5023d7c0ed5505736a3b553f6c81d7da

parent

347521a6dce587e6b51f33608744125d853c7078

1 file changed, 50 insertions(+), 0 deletions(-)

changed files
M internal/importer/utils.gointernal/importer/utils.go
@@ -1,13 +1,16 @@
package importer import ( + "context" "fmt" "io" "strings" + "sync" "alin.ovh/searchix/internal/config" "alin.ovh/searchix/internal/nix" + "alin.ovh/x/log" "github.com/Southclaws/fault" "github.com/Southclaws/fault/fmsg" "github.com/bcicen/jstream"
@@ -61,3 +64,50 @@ }
return nil } + +func pipe( + ctx context.Context, + log *log.Logger, + src ImportSource, + dst ImportDestination, +) (bool, error) { + wg := sync.WaitGroup{} + + wg.Add(1) + objects, srcErrs := src(ctx) + + wg.Add(1) + dstErrors := dst(ctx, objects) + + var hadObjectErrors bool + var criticalError error + go func() { + for { + select { + case err, running := <-srcErrs: + if !running { + wg.Done() + srcErrs = nil + log.Debug("processing completed") + + continue + } + hadObjectErrors = true + log.Warn("error processing object from source", "error", err) + case err, running := <-dstErrors: + if !running { + wg.Done() + dstErrors = nil + + continue + } + hadObjectErrors = true + log.Warn("error writing object to target", "error", err) + } + } + }() + + wg.Wait() + + return hadObjectErrors, criticalError +}