switch to sqlite
1 file changed, 248 insertions(+), 0 deletions(-)
changed files
A internal/storage/sqlite/writer.go
@@ -0,0 +1,248 @@ +package sqlite + +import ( + "database/sql" + "fmt" + "hash/fnv" + "io" + "mime" + "path/filepath" + "time" + + "github.com/andybalholm/brotli" + "github.com/klauspost/compress/gzip" + "github.com/klauspost/compress/zstd" + "go.alanpearce.eu/website/internal/buffer" + "go.alanpearce.eu/website/internal/storage" + "go.alanpearce.eu/x/log" + + "gitlab.com/tozd/go/errors" + _ "modernc.org/sqlite" // import registers db/SQL driver +) + +var encodings = []string{"gzip", "br", "zstd"} + +type Writer struct { + db *sql.DB + + options *Options + log *log.Logger + queries struct { + insertURL *sql.Stmt + insertFile *sql.Stmt + insertContent *sql.Stmt + } +} + +type Options struct { + Compress bool +} + +func OpenDB(dbPath string) (*sql.DB, error) { + return sql.Open( + "sqlite", + fmt.Sprintf( + "file:%s?mode=%s&_pragma=foreign_keys(1)&_pragma=mmap_size(%d)", + dbPath, + "rwc", + 16*1024*1024, + ), + ) +} + +func NewWriter(db *sql.DB, logger *log.Logger, opts *Options) (*Writer, error) { + _, err := db.Exec(` + CREATE TABLE IF NOT EXISTS url ( + url_id INTEGER PRIMARY KEY, + path TEXT NOT NULL + ); + CREATE UNIQUE INDEX IF NOT EXISTS url_path + ON url (path); + + CREATE TABLE IF NOT EXISTS file ( + file_id INTEGER PRIMARY KEY, + url_id INTEGER NOT NULL, + content_type TEXT NOT NULL, + last_modified INTEGER NOT NULL, + etag TEXT NOT NULL, + FOREIGN KEY (url_id) REFERENCES url (url_id) + ); + CREATE UNIQUE INDEX IF NOT EXISTS file_url_content_type + ON file (url_id, content_type); + + CREATE TABLE IF NOT EXISTS content ( + content_id INTEGER PRIMARY KEY, + file_id INTEGER NOT NULL, + encoding TEXT NOT NULL, + body BLOB NOT NULL, + FOREIGN KEY (file_id) REFERENCES file (file_id) + ); + CREATE UNIQUE INDEX IF NOT EXISTS file_content + ON content (file_id, encoding); + `) + if err != nil { + return nil, errors.WithMessage(err, "creating tables") + } + + w := &Writer{ + db: db, + log: logger, + options: opts, + } + + w.queries.insertURL, err = db.Prepare(`INSERT INTO url (path) VALUES (?)`) + if err != nil { + return nil, errors.WithMessage(err, "preparing insert URL statement") + } + + w.queries.insertFile, err = db.Prepare(` + INSERT INTO file (url_id, content_type, last_modified, etag) + VALUES (:url_id, :content_type, :last_modified, :etag) + `) + if err != nil { + return nil, errors.WithMessage(err, "preparing insert file statement") + } + + w.queries.insertContent, err = db.Prepare(` + INSERT INTO content (file_id, encoding, body) + VALUES (:file_id, :encoding, :body) + `) + if err != nil { + return nil, errors.WithMessage(err, "preparing insert content statement") + } + + return w, nil +} + +func (s *Writer) Mkdirp(string) error { + return nil +} + +func (s *Writer) storeURL(path string) (int64, error) { + r, err := s.queries.insertURL.Exec(path) + if err != nil { + return 0, errors.WithMessagef(err, "inserting URL %s into database", path) + } + + return r.LastInsertId() +} + +func (s *Writer) storeFile(urlID int64, file *storage.File) (int64, error) { + r, err := s.queries.insertFile.Exec( + sql.Named("url_id", urlID), + sql.Named("content_type", file.ContentType), + sql.Named("last_modified", file.LastModified.Unix()), + sql.Named("etag", file.Etag), + ) + if err != nil { + return 0, errors.WithMessage(err, "inserting file into database") + } + + return r.LastInsertId() +} + +func (s *Writer) storeEncoding(fileID int64, encoding string, data []byte) error { + _, err := s.queries.insertContent.Exec( + sql.Named("file_id", fileID), + sql.Named("encoding", encoding), + sql.Named("body", data), + ) + if err != nil { + return errors.WithMessagef( + err, + "inserting encoding into database file_id: %d encoding: %s", + fileID, + encoding, + ) + } + + return nil +} + +func etag(content []byte) (string, error) { + hash := fnv.New64a() + hash.Write(content) + + return fmt.Sprintf(`W/"%x"`, hash.Sum(nil)), nil +} + +func contentType(pathname string) string { + return mime.TypeByExtension(filepath.Ext(pathNameToFileName(pathname))) +} + +func (s *Writer) Write(pathname string, content *buffer.Buffer) error { + s.log.Debug("storing content", "pathname", pathname) + bytes := content.Bytes() + + urlID, err := s.storeURL(pathname) + if err != nil { + return errors.WithMessage(err, "storing URL") + } + + etag, err := etag(bytes) + if err != nil { + return errors.WithMessage(err, "calculating etag") + } + + file := &storage.File{ + Path: pathname, + ContentType: contentType(pathname), + LastModified: time.Now(), + Etag: etag, + } + + fileID, err := s.storeFile(urlID, file) + if err != nil { + return errors.WithMessage(err, "storing file") + } + + err = s.storeEncoding(fileID, "identity", bytes) + if err != nil { + return err + } + + if s.options.Compress { + for _, enc := range encodings { + compressed, err := compress(enc, content) + if err != nil { + return errors.WithMessage(err, "compressing file") + } + + err = s.storeEncoding(fileID, enc, compressed.Bytes()) + if err != nil { + return err + } + + } + } + + return nil +} + +func compress(encoding string, content *buffer.Buffer) (compressed *buffer.Buffer, err error) { + var w io.WriteCloser + compressed = new(buffer.Buffer) + switch encoding { + case "gzip": + w = gzip.NewWriter(compressed) + case "br": + w = brotli.NewWriter(compressed) + case "zstd": + w, err = zstd.NewWriter(compressed) + if err != nil { + return nil, errors.WithMessage(err, "could not create zstd writer") + } + } + defer w.Close() + + err = content.SeekStart() + if err != nil { + return nil, errors.WithMessage(err, "seeking to start of content buffer") + } + _, err = io.Copy(w, content) + if err != nil { + return nil, errors.WithMessage(err, "compressing file") + } + + return compressed, nil +}