remove sqlite support
16 files changed, 10 insertions(+), 947 deletions(-)
D .sqlfluff
@@ -1,29 +0,0 @@ -[sqlfluff] -dialect = sqlite -templater = placeholder -processes = 0 - -[sqlfluff:indentation] -indented_joins = False -indented_using_on = True -template_blocks_indent = False - -[sqlfluff:rules:capitalisation.keywords] -capitalisation_policy = upper -[sqlfluff:rules:capitalisation.identifiers] -extended_capitalisation_policy = lower -[sqlfluff:rules:capitalisation.functions] -extended_capitalisation_policy = upper -[sqlfluff:rules:capitalisation.literals] -capitalisation_policy = upper -[sqlfluff:rules:capitalisation.types] -capitalisation_policy = upper - -[sqlfluff:rules:convention.not_equal] -preferred_not_equal_style = c_style - -[sqlfluff:templater] -unwrap_wrapped_queries = True - -[sqlfluff:templater:placeholder] -param_style = question_mark
M domain/content/builder/build/main.go → domain/content/builder/build/main.go
@@ -1,18 +1,13 @@ package main import ( - "database/sql" "errors" "fmt" "os" - "path/filepath" "alin.ovh/homestead/domain/content/builder" "alin.ovh/homestead/shared/config" - "alin.ovh/homestead/shared/file" - "alin.ovh/homestead/shared/storage" "alin.ovh/homestead/shared/storage/files" - "alin.ovh/homestead/shared/storage/sqlite" "alin.ovh/homestead/shared/vcs" "alin.ovh/x/log"@@ -23,7 +18,6 @@ type Options struct { *builder.Options Destination string `conf:"default:./public,short:d,flag:dest"` Compress bool `conf:"default:true"` - Writer string `conf:"default:files,help:Output type (files|sqlite)"` } const branch = "main"@@ -48,59 +42,16 @@ panic("could not open repository: " + err.Error()) } options.Repo = repo - var storage storage.Writer - switch options.Writer { - case "files": - storage, err = files.NewWriter(options.Destination, log.Named("storage"), &files.Options{ + options.Storage, err = files.NewWriter( + options.Destination, + log.Named("storage"), + &files.Options{ Compress: options.Compress, - }) - if err != nil { - panic("could not create storage: " + err.Error()) - } - case "sqlite": - var stat os.FileInfo - stat, err = file.Stat(options.Destination) - if err != nil && !errors.Is(err, os.ErrNotExist) { - panic("could not stat destination: " + err.Error()) - } - switch { - case stat == nil: - err := os.MkdirAll(options.Destination, 0o750) - if err != nil { - panic("could not make directory: " + err.Error()) - } - case !stat.IsDir(): - panic("destination is not a directory") - default: - err := file.CleanDir(options.Destination) - if err != nil { - panic("could not clean destination: " + err.Error()) - } - } - - var db *sql.DB - db, err = sqlite.OpenDB(filepath.Join(options.Destination, "site.db")) - if err != nil { - panic("could not open database: " + err.Error()) - } - defer db.Close() - storage, err = sqlite.NewWriter(db, log.Named("storage"), &sqlite.Options{ - Compress: options.Compress, - }) - if err != nil { - panic("could not create storage: " + err.Error()) - } - err = file.Copy( - filepath.Join(options.Source, "config.toml"), - filepath.Join(options.Destination, "config.toml"), - ) - if err != nil { - panic("could not copy configuration: " + err.Error()) - } - default: - panic("unknown storage type: " + options.Writer) + }, + ) + if err != nil { + panic("could not create storage: " + err.Error()) } - options.Storage = storage log.Debug("starting build process") cfg, err := config.GetConfig(options.Source, log)
M domain/web/website.go → domain/web/website.go
@@ -4,7 +4,6 @@ import ( "context" "net/http" "os" - "path/filepath" "sync" "time"@@ -21,7 +20,7 @@ "alin.ovh/homestead/shared/events" "alin.ovh/homestead/shared/file" ihttp "alin.ovh/homestead/shared/http" "alin.ovh/homestead/shared/storage" - "alin.ovh/homestead/shared/storage/sqlite" + "alin.ovh/homestead/shared/storage/files" "alin.ovh/x/log" "github.com/Southclaws/fault" "github.com/Southclaws/fault/fmsg"@@ -167,13 +166,7 @@ } website.Domain = cfg.BaseURL.Hostname() - siteDB := filepath.Join(root, "site.db") - db, err := sqlite.OpenDB(siteDB) - if err != nil { - log.Panic("could not open database", "error", err) - } - - website.reader, err = sqlite.NewReader(db, log.Named("reader")) + website.reader, err = files.NewReader(root, log.Named("reader")) if err != nil { log.Panic("could not create database reader", "error", err) }
M flake.nix → flake.nix
@@ -35,7 +35,6 @@ just ko flyctl redis - sqlc ]; devPackages = with pkgs; [ wgo@@ -43,7 +42,6 @@ gofumpt gopls gotools gomod2nix.legacyPackages.${system}.gomod2nix - sqlfluff systemfd modd@@ -57,8 +55,6 @@ mdformat.enable = true; gofumpt.enable = true; golines.enable = true; prettier.enable = true; - sqlfluff.enable = true; - sqlfluff.dialect = "sqlite"; }; settings.formatter = { taplo.excludes = [ "gomod2nix.toml" ];@@ -112,14 +108,6 @@ name = "gomod2nix"; description = "Import go.mod updates to nix"; types_or = [ "go-sum" ]; entry = "${pkgs.gomod2nix}/bin/gomod2nix"; - pass_filenames = false; - }; - sqlc = { - enable = true; - name = "sqlc"; - description = "Generate go code from SQL"; - types_or = [ "sql" ]; - entry = "${pkgs.sqlc}/bin/sqlc generate"; pass_filenames = false; }; };
M go.mod → go.mod
@@ -28,7 +28,6 @@ github.com/stefanfritsch/goldmark-fences v1.0.0 github.com/yuin/goldmark v1.7.13 go.hacdias.com/indielib v0.4.3 go.uber.org/zap v1.27.0 - modernc.org/sqlite v1.38.2 tailscale.com v1.84.2 vimagination.zapto.org/ics v1.0.0 )@@ -72,7 +71,6 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e // indirect - github.com/dustin/go-humanize v1.0.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/gaissmai/bart v0.18.0 // indirect@@ -104,12 +102,10 @@ github.com/mdlayher/socket v0.5.0 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/miekg/dns v1.1.58 // indirect github.com/mitchellh/go-ps v1.0.0 // indirect - github.com/ncruces/go-strftime v0.1.9 // indirect github.com/onsi/gomega v1.34.2 // indirect github.com/pjbgf/sha1cd v0.4.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus-community/pro-bing v0.4.0 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rs/zerolog v1.34.0 // indirect github.com/safchain/ethtool v0.3.0 // indirect github.com/sergi/go-diff v1.4.0 // indirect@@ -147,9 +143,6 @@ golang.zx2c4.com/wireguard/windows v0.5.3 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gvisor.dev/gvisor v0.0.0-20250205023644-9414b50a5633 // indirect - modernc.org/libc v1.66.7 // indirect - modernc.org/mathutil v1.7.1 // indirect - modernc.org/memory v1.11.0 // indirect moul.io/zapfilter v1.7.0 // indirect vimagination.zapto.org/parser v1.2.2 // indirect willnorris.com/go/microformats v1.2.1-0.20240301064101-b5d1b9d2120e // indirect
M go.sum → go.sum
@@ -130,8 +130,6 @@ github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= github.com/dsnet/try v0.0.3 h1:ptR59SsrcFUYbT/FhAbKTV6iLkeD6O18qfIWRml2fqI= github.com/dsnet/try v0.0.3/go.mod h1:WBM8tRpUmnXXhY1U6/S8dt6UWdHTQ7y8A5YSkRCkq40= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=@@ -183,8 +181,6 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/nftables v0.2.1-0.20240414091927-5e242ec57806 h1:wG8RYIyctLhdFk6Vl1yPGtSRtwGpVkWyZww1OCil2MI= github.com/google/nftables v0.2.1-0.20240414091927-5e242ec57806/go.mod h1:Beg6V6zZ3oEn0JuiUQ4wqwuyqqzasOltcoXPtgLbFp4= -github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= -github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg= github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4=@@ -253,8 +249,6 @@ github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= -github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8=@@ -282,8 +276,6 @@ github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= github.com/redis/go-redis/v9 v9.12.1 h1:k5iquqv27aBtnTm2tIkROUDp8JBXhXZIVu1InSgvovg= github.com/redis/go-redis/v9 v9.12.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=@@ -544,32 +536,6 @@ honnef.co/go/tools v0.5.1 h1:4bH5o3b5ZULQ4UrBmP+63W9r7qIkqJClEA9ko5YKx+I= honnef.co/go/tools v0.5.1/go.mod h1:e9irvo83WDG9/irijV44wr3tbhcFeRnfpVlRqVwpzMs= howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM= howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= -modernc.org/cc/v4 v4.26.3 h1:yEN8dzrkRFnn4PUUKXLYIqVf2PJYAEjMTFjO3BDGc3I= -modernc.org/cc/v4 v4.26.3/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= -modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU= -modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE= -modernc.org/fileutil v1.3.15 h1:rJAXTP6ilMW/1+kzDiqmBlHLWszheUFXIyGQIAvjJpY= -modernc.org/fileutil v1.3.15/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= -modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= -modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= -modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.66.7 h1:rjhZ8OSCybKWxS1CJr0hikpEi6Vg+944Ouyrd+bQsoY= -modernc.org/libc v1.66.7/go.mod h1:ln6tbWX0NH+mzApEoDRvilBvAWFt1HX7AUA4VDdVDPM= -modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= -modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= -modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= -modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= -modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= -modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= -modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= -modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.38.2 h1:Aclu7+tgjgcQVShZqim41Bbw9Cho0y/7WzYptXqkEek= -modernc.org/sqlite v1.38.2/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E= -modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= -modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= moul.io/zapfilter v1.7.0 h1:7aFrG4N72bDH9a2BtYUuUaDS981Dxu3qybWfeqaeBDU= moul.io/zapfilter v1.7.0/go.mod h1:M+N2s+qZiA+bzRoyKMVRxyuERijS2ovi2pnMyiOGMvc= software.sslmate.com/src/go-pkcs12 v0.4.0 h1:H2g08FrTvSFKUj+D309j1DPfk5APnIdAQAB8aEykJ5k=
M gomod2nix.toml → gomod2nix.toml
@@ -154,9 +154,6 @@ hash = "sha256-n/7xo5CQqo4yLaWMSzSN1Muk/oqK6O5dgDOFWapeDUI=" [mod."github.com/digitalocean/go-smbios"] version = "v0.0.0-20180907143718-390a4f403a8e" hash = "sha256-Hgx1ML3rigdYXx5ntnEYD5JEk6kJv1cL0+/GDbnPkbo=" - [mod."github.com/dustin/go-humanize"] - version = "v1.0.1" - hash = "sha256-yuvxYYngpfVkUg9yAmG99IUVmADTQA0tMbBXe0Fq0Mc=" [mod."github.com/emirpasic/gods"] version = "v1.18.1" hash = "sha256-hGDKddjLj+5dn2woHtXKUdd49/3xdsqnhx7VEdCu1m4="@@ -265,9 +262,6 @@ hash = "sha256-UGvyC1Abh2S5VaAUCV9AUuDMrCvpiWQy/UnYM9DfIB8=" [mod."github.com/mitchellh/go-ps"] version = "v1.0.0" hash = "sha256-HzxVHNLHZpnsBuPcub0G+9jjDcDOsxM/6wifbsxf7EY=" - [mod."github.com/ncruces/go-strftime"] - version = "v0.1.9" - hash = "sha256-T0iw+UEckzueWHT88PkTnZZixyKCEa+DTLzIiiohuWY=" [mod."github.com/onsi/gomega"] version = "v1.34.2" hash = "sha256-4QHv+D8hTc7y26LIkjn/SSQkT5bvwpYBGyzQIoV185g="@@ -286,9 +280,6 @@ hash = "sha256-3TH0wB85OITw3uzTcEva2EcEF6jNf98sAoSOsnL2G9g=" [mod."github.com/redis/go-redis/v9"] version = "v9.12.1" hash = "sha256-MHwOv3RaZ5Bq/pOL/hXd0kErXQeo1NwdCpRtRD6De1U=" - [mod."github.com/remyoudompheng/bigfft"] - version = "v0.0.0-20230129092748-24d4a6f8daec" - hash = "sha256-vYmpyCE37eBYP/navhaLV4oX4/nu0Z/StAocLIFqrmM=" [mod."github.com/rs/zerolog"] version = "v1.34.0" hash = "sha256-M503WwzPvqbOas3f70FQNXoWG17eV/XU6FubtB6P0uo="@@ -415,18 +406,6 @@ hash = "sha256-uVEGglIedjOIGZzHW4YwN1VoRSTK8o0eGZqzd+TNdd0=" [mod."gvisor.dev/gvisor"] version = "v0.0.0-20250205023644-9414b50a5633" hash = "sha256-XX3uGywgI/ihUyaGs3VbvoOayJBrGGKPZtO+KBZlCm8=" - [mod."modernc.org/libc"] - version = "v1.66.7" - hash = "sha256-xFkAPUAtzHWGmjcmnFbmh33XCnObvklH8Mem3qGyekU=" - [mod."modernc.org/mathutil"] - version = "v1.7.1" - hash = "sha256-COZ5rF2GhQVR1r6a0DanJ8qwQ94JSKdQxTMWrDzE0Cc=" - [mod."modernc.org/memory"] - version = "v1.11.0" - hash = "sha256-MkybF8vvrxXS5j7O8w3skwTo0aMo1yjWS0K440rYcHM=" - [mod."modernc.org/sqlite"] - version = "v1.38.2" - hash = "sha256-rI7ZxyE7ecVaZai+431IXx9uZec7Jo5O4c47mlYdaeY=" [mod."moul.io/zapfilter"] version = "v1.7.0" hash = "sha256-H6j5h8w123Y7d0zvKGkL5jiRYICtjmgzd2P/eeNaLrs="
D shared/storage/sqlite/db/db.go
@@ -1,128 +0,0 @@ -// Code generated by sqlc. DO NOT EDIT. -// versions: -// sqlc v1.29.0 - -package db - -import ( - "context" - "database/sql" - "fmt" -) - -type DBTX interface { - ExecContext(context.Context, string, ...interface{}) (sql.Result, error) - PrepareContext(context.Context, string) (*sql.Stmt, error) - QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) - QueryRowContext(context.Context, string, ...interface{}) *sql.Row -} - -func New(db DBTX) *Queries { - return &Queries{db: db} -} - -func Prepare(ctx context.Context, db DBTX) (*Queries, error) { - q := Queries{db: db} - var err error - if q.checkPathStmt, err = db.PrepareContext(ctx, checkPath); err != nil { - return nil, fmt.Errorf("error preparing query CheckPath: %w", err) - } - if q.getFileStmt, err = db.PrepareContext(ctx, getFile); err != nil { - return nil, fmt.Errorf("error preparing query GetFile: %w", err) - } - if q.insertContentStmt, err = db.PrepareContext(ctx, insertContent); err != nil { - return nil, fmt.Errorf("error preparing query InsertContent: %w", err) - } - if q.insertFileStmt, err = db.PrepareContext(ctx, insertFile); err != nil { - return nil, fmt.Errorf("error preparing query InsertFile: %w", err) - } - if q.insertURLStmt, err = db.PrepareContext(ctx, insertURL); err != nil { - return nil, fmt.Errorf("error preparing query InsertURL: %w", err) - } - return &q, nil -} - -func (q *Queries) Close() error { - var err error - if q.checkPathStmt != nil { - if cerr := q.checkPathStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing checkPathStmt: %w", cerr) - } - } - if q.getFileStmt != nil { - if cerr := q.getFileStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing getFileStmt: %w", cerr) - } - } - if q.insertContentStmt != nil { - if cerr := q.insertContentStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing insertContentStmt: %w", cerr) - } - } - if q.insertFileStmt != nil { - if cerr := q.insertFileStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing insertFileStmt: %w", cerr) - } - } - if q.insertURLStmt != nil { - if cerr := q.insertURLStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing insertURLStmt: %w", cerr) - } - } - return err -} - -func (q *Queries) exec(ctx context.Context, stmt *sql.Stmt, query string, args ...interface{}) (sql.Result, error) { - switch { - case stmt != nil && q.tx != nil: - return q.tx.StmtContext(ctx, stmt).ExecContext(ctx, args...) - case stmt != nil: - return stmt.ExecContext(ctx, args...) - default: - return q.db.ExecContext(ctx, query, args...) - } -} - -func (q *Queries) query(ctx context.Context, stmt *sql.Stmt, query string, args ...interface{}) (*sql.Rows, error) { - switch { - case stmt != nil && q.tx != nil: - return q.tx.StmtContext(ctx, stmt).QueryContext(ctx, args...) - case stmt != nil: - return stmt.QueryContext(ctx, args...) - default: - return q.db.QueryContext(ctx, query, args...) - } -} - -func (q *Queries) queryRow(ctx context.Context, stmt *sql.Stmt, query string, args ...interface{}) *sql.Row { - switch { - case stmt != nil && q.tx != nil: - return q.tx.StmtContext(ctx, stmt).QueryRowContext(ctx, args...) - case stmt != nil: - return stmt.QueryRowContext(ctx, args...) - default: - return q.db.QueryRowContext(ctx, query, args...) - } -} - -type Queries struct { - db DBTX - tx *sql.Tx - checkPathStmt *sql.Stmt - getFileStmt *sql.Stmt - insertContentStmt *sql.Stmt - insertFileStmt *sql.Stmt - insertURLStmt *sql.Stmt -} - -func (q *Queries) WithTx(tx *sql.Tx) *Queries { - return &Queries{ - db: tx, - tx: tx, - checkPathStmt: q.checkPathStmt, - getFileStmt: q.getFileStmt, - insertContentStmt: q.insertContentStmt, - insertFileStmt: q.insertFileStmt, - insertURLStmt: q.insertURLStmt, - } -}
D shared/storage/sqlite/db/models.go
@@ -1,27 +0,0 @@ -// Code generated by sqlc. DO NOT EDIT. -// versions: -// sqlc v1.29.0 - -package db - -type Content struct { - ContentID int64 - FileID int64 - Encoding string - Body []byte -} - -type File struct { - FileID int64 - UrlID int64 - ContentType string - LastModified int64 - Title string - Etag string - Headers []byte -} - -type Url struct { - UrlID int64 - Path string -}
D shared/storage/sqlite/db/query.sql.go
@@ -1,145 +0,0 @@ -// Code generated by sqlc. DO NOT EDIT. -// versions: -// sqlc v1.29.0 -// source: query.sql - -package db - -import ( - "context" -) - -const checkPath = `-- name: CheckPath :one -SELECT - EXISTS( - SELECT 1 - FROM url - WHERE path = ? - ) AS differs -` - -func (q *Queries) CheckPath(ctx context.Context, path string) (int64, error) { - row := q.queryRow(ctx, q.checkPathStmt, checkPath, path) - var differs int64 - err := row.Scan(&differs) - return differs, err -} - -const getFile = `-- name: GetFile :many -SELECT - file.file_id, file.url_id, file.content_type, file.last_modified, file.title, file.etag, file.headers, - content.content_id, content.file_id, content.encoding, content.body -FROM url -INNER JOIN file - USING (url_id) -INNER JOIN content - USING (file_id) -WHERE - url.path = ? -` - -type GetFileRow struct { - File File - Content Content -} - -func (q *Queries) GetFile(ctx context.Context, path string) ([]GetFileRow, error) { - rows, err := q.query(ctx, q.getFileStmt, getFile, path) - if err != nil { - return nil, err - } - defer rows.Close() - var items []GetFileRow - for rows.Next() { - var i GetFileRow - if err := rows.Scan( - &i.File.FileID, - &i.File.UrlID, - &i.File.ContentType, - &i.File.LastModified, - &i.File.Title, - &i.File.Etag, - &i.File.Headers, - &i.Content.ContentID, - &i.Content.FileID, - &i.Content.Encoding, - &i.Content.Body, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Close(); err != nil { - return nil, err - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - -const insertContent = `-- name: InsertContent :exec -INSERT INTO content (file_id, encoding, body) -VALUES (?1, ?2, ?3) -` - -type InsertContentParams struct { - Fileid int64 - Encoding string - Body []byte -} - -func (q *Queries) InsertContent(ctx context.Context, arg InsertContentParams) error { - _, err := q.exec(ctx, q.insertContentStmt, insertContent, arg.Fileid, arg.Encoding, arg.Body) - return err -} - -const insertFile = `-- name: InsertFile :execlastid -INSERT INTO file ( - url_id, content_type, last_modified, etag, title, headers -) -VALUES ( - ?1, - ?2, - ?3, - ?4, - ?5, - ?6 -) -` - -type InsertFileParams struct { - UrlID int64 - ContentType string - LastModified int64 - Etag string - Title string - Headers []byte -} - -func (q *Queries) InsertFile(ctx context.Context, arg InsertFileParams) (int64, error) { - result, err := q.exec(ctx, q.insertFileStmt, insertFile, - arg.UrlID, - arg.ContentType, - arg.LastModified, - arg.Etag, - arg.Title, - arg.Headers, - ) - if err != nil { - return 0, err - } - return result.LastInsertId() -} - -const insertURL = `-- name: InsertURL :execlastid -INSERT INTO url (path) VALUES (?) -` - -func (q *Queries) InsertURL(ctx context.Context, path string) (int64, error) { - result, err := q.exec(ctx, q.insertURLStmt, insertURL, path) - if err != nil { - return 0, err - } - return result.LastInsertId() -}
D shared/storage/sqlite/file.go
@@ -1,19 +0,0 @@ -package sqlite - -import ( - "strings" -) - -func pathNameToFileName(pathname string) string { - if strings.HasSuffix(pathname, "/") { - pathname += "index.html" - } - - return pathname -} - -func cleanPathName(pathname string) string { - pathname = strings.TrimSuffix(pathname, ".html") - - return pathname -}
D shared/storage/sqlite/query.sql
@@ -1,39 +0,0 @@ --- name: InsertURL :execlastid -INSERT INTO url (path) VALUES (?); - --- name: InsertFile :execlastid -INSERT INTO file ( - url_id, content_type, last_modified, etag, title, headers -) -VALUES ( - @url_id, - @content_type, - @last_modified, - @etag, - @title, - @headers -); - --- name: InsertContent :exec -INSERT INTO content (file_id, encoding, body) -VALUES (@fileid, @encoding, @body); - --- name: GetFile :many -SELECT - sqlc.embed(file), - sqlc.embed(content) -FROM url -INNER JOIN file - USING (url_id) -INNER JOIN content - USING (file_id) -WHERE - url.path = ?; - --- name: CheckPath :one -SELECT - EXISTS( - SELECT 1 - FROM url - WHERE path = ? - ) AS differs;
D shared/storage/sqlite/reader.go
@@ -1,104 +0,0 @@ -package sqlite - -import ( - "context" - "database/sql" - "encoding/json" - "strings" - "time" - - "alin.ovh/homestead/shared/buffer" - "alin.ovh/homestead/shared/storage" - "alin.ovh/homestead/shared/storage/sqlite/db" - "alin.ovh/x/log" - "github.com/Southclaws/fault" - "github.com/Southclaws/fault/fmsg" -) - -type Reader struct { - log *log.Logger - queries *db.Queries -} - -func NewReader(conn *sql.DB, log *log.Logger) (*Reader, error) { - r := &Reader{ - log: log, - queries: db.New(conn), - } - - return r, nil -} - -func (r *Reader) GetFile(filename string) (*storage.File, error) { - file := &storage.File{ - Encodings: make(map[string]*buffer.Buffer, 1), - } - - r.log.Debug("querying db for file", "filename", filename) - rows, err := r.queries.GetFile(context.TODO(), filename) - if err != nil { - return nil, fault.Wrap(err, fmsg.With("querying database")) - } - - count := 0 - for _, row := range rows { - count++ - file.ContentType = row.File.ContentType - file.LastModified = time.Unix(row.File.LastModified, 0) - file.Etag = row.File.Etag - file.Title = row.File.Title - - if len(row.File.Headers) > 2 { - err := json.Unmarshal(row.File.Headers, &file.Headers) - if err != nil { - return nil, fault.Wrap(err, fmsg.With("unmarshalling headers")) - } - } - - file.Encodings[row.Content.Encoding] = buffer.NewBuffer(row.Content.Body) - } - if count == 0 { - return nil, nil - } - - return file, nil -} - -func (r *Reader) CanonicalisePath(path string) (cPath string, differs bool) { - cPath = path - switch { - case path == "/": - differs = false - case strings.HasSuffix(path, "/index.xml"): - cPath, differs = strings.TrimSuffix(path, "index.xml")+"atom.xml", true - case strings.HasSuffix(path, "/index.html"): - cPath, differs = strings.CutSuffix(path, "index.html") - case strings.HasSuffix(path, ".html"): - cPath, differs = strings.CutSuffix(path, ".html") - case !strings.HasSuffix(path, "/"): - d, err := r.queries.CheckPath(context.TODO(), cPath+"/") - if err != nil { - r.log.Warn("error canonicalising path", "path", path, "error", err) - - return - } - differs = d == 1 - if differs { - cPath += "/" - } - case strings.HasSuffix(path, "/"): - tmp := strings.TrimSuffix(path, "/") - d, err := r.queries.CheckPath(context.TODO(), tmp) - if err != nil { - r.log.Warn("error canonicalising path", "path", path, "error", err) - - return - } - differs = d == 1 - if differs { - cPath = tmp - } - } - - return -}
D shared/storage/sqlite/schema.sql
@@ -1,32 +0,0 @@ -CREATE TABLE IF NOT EXISTS url ( - url_id INTEGER PRIMARY KEY, - path TEXT NOT NULL -) STRICT; - -CREATE UNIQUE INDEX IF NOT EXISTS url_path -ON url (path); - -CREATE TABLE IF NOT EXISTS file ( - file_id INTEGER PRIMARY KEY, - url_id INTEGER NOT NULL, - content_type TEXT NOT NULL, - last_modified INTEGER NOT NULL, - title TEXT NOT NULL, - etag TEXT NOT NULL, - headers BLOB NOT NULL, - FOREIGN KEY (url_id) REFERENCES url (url_id) -) STRICT; - -CREATE UNIQUE INDEX IF NOT EXISTS file_url_content_type -ON file (url_id, content_type); - -CREATE TABLE IF NOT EXISTS content ( - content_id INTEGER PRIMARY KEY, - file_id INTEGER NOT NULL, - encoding TEXT NOT NULL, - body BLOB NOT NULL, - FOREIGN KEY (file_id) REFERENCES file (file_id) -) STRICT; - -CREATE UNIQUE INDEX IF NOT EXISTS file_content -ON content (file_id, encoding);
D shared/storage/sqlite/writer.go
@@ -1,273 +0,0 @@ -package sqlite - -import ( - "context" - "database/sql" - _ "embed" - "encoding/json" - "fmt" - "hash/fnv" - "io" - "mime" - "net/http" - "path/filepath" - "time" - - "alin.ovh/homestead/domain/content" - "alin.ovh/homestead/shared/buffer" - "alin.ovh/homestead/shared/storage" - "alin.ovh/homestead/shared/storage/sqlite/db" - "alin.ovh/x/log" - "github.com/andybalholm/brotli" - "github.com/klauspost/compress/gzip" - "github.com/klauspost/compress/zstd" - - "github.com/Southclaws/fault" - "github.com/Southclaws/fault/fmsg" - _ "modernc.org/sqlite" // import registers db/SQL driver -) - -type Writer struct { - options *Options - log *log.Logger - queries *db.Queries -} - -type Options struct { - Compress bool -} - -var ( - encodings = []string{"gzip", "br", "zstd"} - //go:embed schema.sql - schema string -) - -func OpenDB(dbPath string) (*sql.DB, error) { - db, err := sql.Open( - "sqlite", - fmt.Sprintf( - "file:%s?mode=%s&_pragma=foreign_keys(1)&_pragma=mmap_size(%d)", - dbPath, - "rwc", - 16*1024*1024, - ), - ) - if err != nil { - return nil, fault.Wrap(err) - } - - return db, nil -} - -func NewWriter(conn *sql.DB, logger *log.Logger, opts *Options) (*Writer, error) { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - _, err := conn.ExecContext(ctx, schema) - if err != nil { - return nil, fault.Wrap(err, fmsg.With("creating tables")) - } - - w := &Writer{ - queries: db.New(conn), - log: logger, - options: opts, - } - - return w, nil -} - -func (s *Writer) Mkdirp(string) error { - return nil -} - -func (s *Writer) NewFileFromPost(post *content.Post) *storage.File { - file := &storage.File{ - Title: post.Title, - Path: post.URL, - FSPath: pathNameToFileName(post.URL), - LastModified: post.Date, - Encodings: map[string]*buffer.Buffer{}, - } - - return file -} - -func (s *Writer) WritePost(post *content.Post, content *buffer.Buffer) error { - s.log.Debug("storing post", "title", post.Title) - - return s.WriteFile(s.NewFileFromPost(post), content) -} - -func (s *Writer) Write(pathname string, title string, content *buffer.Buffer) error { - file := &storage.File{ - Title: title, - Path: cleanPathName(pathname), - FSPath: pathname, - LastModified: time.Now(), - Encodings: map[string]*buffer.Buffer{}, - } - - return s.WriteFile(file, content) -} - -func (s *Writer) WriteFile(file *storage.File, content *buffer.Buffer) error { - s.log.Debug("storing content", "pathname", file.Path) - - urlID, err := s.storeURL(file.Path) - if err != nil { - return fault.Wrap(err, fmsg.With("storing URL")) - } - - if file.Encodings == nil { - file.Encodings = map[string]*buffer.Buffer{} - } - file.Encodings["identity"] = content - - if file.ContentType == "" { - file.ContentType = contentType(file.FSPath) - } - - if file.Etag == "" { - file.Etag, err = etag(content.Bytes()) - if err != nil { - return fault.Wrap(err, fmsg.With("could not calculate file etag")) - } - } - - if err := content.SeekStart(); err != nil { - return fault.Wrap(err, fmsg.With("seeking content start")) - } - - err = file.CalculateStyleHash() - if err != nil { - return fault.Wrap(err, fmsg.With("calculating file hash")) - } - - fileID, err := s.storeFile(urlID, file) - if err != nil { - return fault.Wrap(err, fmsg.With("storing file")) - } - - err = s.storeEncoding(fileID, "identity", content.Bytes()) - if err != nil { - return err - } - - if s.options.Compress { - for _, enc := range encodings { - compressed, err := compress(enc, content) - if err != nil { - return fault.Wrap(err, fmsg.With("compressing file")) - } - - err = s.storeEncoding(fileID, enc, compressed.Bytes()) - if err != nil { - return err - } - } - } - - return nil -} - -func compress(encoding string, content *buffer.Buffer) (*buffer.Buffer, error) { - var w io.WriteCloser - compressed := new(buffer.Buffer) - switch encoding { - case "gzip": - w = gzip.NewWriter(compressed) - case "br": - w = brotli.NewWriter(compressed) - case "zstd": - var err error - w, err = zstd.NewWriter(compressed) - if err != nil { - return nil, fault.Wrap(err, fmsg.With("could not create zstd writer")) - } - } - defer w.Close() - - if err := content.SeekStart(); err != nil { - return nil, fault.Wrap(err, fmsg.With("seeking to start of content buffer")) - } - if _, err := io.Copy(w, content); err != nil { - return nil, fault.Wrap(err, fmsg.With("compressing file")) - } - - return compressed, nil -} -func (s *Writer) storeURL(path string) (int64, error) { - id, err := s.queries.InsertURL(context.TODO(), path) - if err != nil { - return 0, fault.Wrap(err, fmsg.With(fmt.Sprintf("inserting URL %s into database", path))) - } - - return id, nil -} - -func (s *Writer) storeFile(urlID int64, file *storage.File) (int64, error) { - if file.ContentType == "" { - file.ContentType = http.DetectContentType(file.Encodings["identity"].Bytes()) - s.log.Warn( - "file has no content type, sniffing", - "path", - file.Path, - "sniffed", - file.ContentType, - ) - } - params := db.InsertFileParams{ - UrlID: urlID, - ContentType: file.ContentType, - LastModified: file.LastModified.Unix(), - Etag: file.Etag, - Title: file.Title, - Headers: []byte{}, - } - if file.Headers != nil { - var err error - params.Headers, err = json.Marshal(file.Headers) - if err != nil { - return 0, fault.Wrap(err, fmsg.With("marshalling headers to JSON")) - } - } - id, err := s.queries.InsertFile(context.TODO(), params) - if err != nil { - return 0, fault.Wrap(err, fmsg.With("inserting file into database")) - } - - return id, nil -} - -func (s *Writer) storeEncoding(fileID int64, encoding string, data []byte) error { - err := s.queries.InsertContent(context.TODO(), db.InsertContentParams{ - Fileid: fileID, - Encoding: encoding, - Body: data, - }) - if err != nil { - return fault.Wrap( - err, - fmsg.With(fmt.Sprintf("inserting encoding into database file_id: %d encoding: %s", - fileID, - encoding)), - ) - } - - return nil -} - -func etag(content []byte) (string, error) { - hash := fnv.New64a() - _, err := hash.Write(content) - if err != nil { - return "", fault.Wrap(err) - } - - return fmt.Sprintf(`W/"%x"`, hash.Sum(nil)), nil -} - -func contentType(pathname string) string { - return mime.TypeByExtension(filepath.Ext(pathNameToFileName(pathname))) -}
D sqlc.yaml
@@ -1,11 +0,0 @@ -version: "2" -sql: - - engine: "sqlite" - queries: "shared/storage/sqlite/query.sql" - schema: "shared/storage/sqlite/schema.sql" - database: - uri: "file:db.sqlite3?mode=rwc" - gen: - go: - out: "shared/storage/sqlite/db" - emit_prepared_queries: true