From fc397d83c2228e38babbc2fd44ef590d95a80bac Mon Sep 17 00:00:00 2001 From: Drew Bednar Date: Sat, 27 Jul 2024 21:31:09 -0400 Subject: [PATCH 1/2] Half way to migrations --- .env.template | 3 + .gitignore | 5 +- cmd/migrate/main.go | 12 ++ config/config.go | 22 +++ config/database.go | 17 ++ database/migrate/libsql/libsql.go | 269 ++++++++++++++++++++++++++++++ database/migrate/migrate.go | 7 + database/sqlx.go | 29 ++++ go.mod | 16 +- go.sum | 32 ++++ 10 files changed, 410 insertions(+), 2 deletions(-) create mode 100644 .env.template create mode 100644 cmd/migrate/main.go create mode 100644 config/config.go create mode 100644 config/database.go create mode 100644 database/migrate/libsql/libsql.go create mode 100644 database/migrate/migrate.go create mode 100644 database/sqlx.go create mode 100644 go.sum diff --git a/.env.template b/.env.template new file mode 100644 index 0000000..1175f41 --- /dev/null +++ b/.env.template @@ -0,0 +1,3 @@ +FLUX_DATABASE_DRIVER=libsql +FLUX_DATABASE_PATH=file:./flux-local.db +FLUX_MIGRATIONS_PATH=./migrations diff --git a/.gitignore b/.gitignore index f65ce7f..5cd1433 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,7 @@ # Go workspace file go.work -fluxfeed \ No newline at end of file +# Flux Feed +fluxfeed +flux-feed +.env \ No newline at end of file diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go new file mode 100644 index 0000000..9cbaa5d --- /dev/null +++ b/cmd/migrate/main.go @@ -0,0 +1,12 @@ +package main + +import ( + "git.runcible.io/androiddrew/flux-feed/config" + "git.runcible.io/androiddrew/flux-feed/database" +) + +func main() { + cfg := config.New() + dbHandle := database.NewSqlx(cfg.Database) + +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..db497ea --- /dev/null +++ b/config/config.go @@ -0,0 +1,22 @@ +package config + +import ( + "log" + + "github.com/joho/godotenv" +) + +type Config struct { + Database +} + +func New() *Config { + err := godotenv.Load() + if err != nil { + log.Println(err) + } + + return &Config{ + Database: DataStore(), + } +} diff --git a/config/database.go b/config/database.go new file mode 100644 index 0000000..c1778c4 --- /dev/null +++ b/config/database.go @@ -0,0 +1,17 @@ +package config + +import "github.com/kelseyhightower/envconfig" + +type Database struct { + DatabaseDriver string `split_words:"true"` + DatabasePath string `split_words:"true"` + MigrationsPath string `default:"head" split_words:"true"` + LibsqlConnectorType string `default:"local" split_words:"true"` +} + +func DataStore() Database { + var db Database + envconfig.MustProcess("flux", &db) + + return db +} diff --git a/database/migrate/libsql/libsql.go b/database/migrate/libsql/libsql.go new file mode 100644 index 0000000..259550d --- /dev/null +++ b/database/migrate/libsql/libsql.go @@ -0,0 +1,269 @@ +package libsql + +import ( + "database/sql" + "fmt" + "io" + nurl "net/url" + "strconv" + "strings" + + "go.uber.org/atomic" + + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database" + "github.com/hashicorp/go-multierror" + _ "github.com/tursodatabase/go-libsql" +) + +func init() { + database.Register("libsql", &Sqlite{}) +} + +var DefaultMigrationsTable = "schema_migrations" +var ( + ErrDatabaseDirty = fmt.Errorf("database is dirty") + ErrNilConfig = fmt.Errorf("no config") + ErrNoDatabaseName = fmt.Errorf("no database name") +) + +type Config struct { + MigrationsTable string + DatabaseName string + NoTxWrap bool +} + +type Sqlite struct { + db *sql.DB + isLocked atomic.Bool + + config *Config +} + +func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { + if config == nil { + return nil, ErrNilConfig + } + + if err := instance.Ping(); err != nil { + return nil, err + } + + if len(config.MigrationsTable) == 0 { + config.MigrationsTable = DefaultMigrationsTable + } + + mx := &Sqlite{ + db: instance, + config: config, + } + if err := mx.ensureVersionTable(); err != nil { + return nil, err + } + return mx, nil +} + +// ensureVersionTable checks if versions table exists and, if not, creates it. +// Note that this function locks the database, which deviates from the usual +// convention of "caller locks" in the Sqlite type. +func (m *Sqlite) ensureVersionTable() (err error) { + if err = m.Lock(); err != nil { + return err + } + + defer func() { + if e := m.Unlock(); e != nil { + if err == nil { + err = e + } else { + err = multierror.Append(err, e) + } + } + }() + + query := fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s (version uint64,dirty bool); + CREATE UNIQUE INDEX IF NOT EXISTS version_unique ON %s (version); + `, m.config.MigrationsTable, m.config.MigrationsTable) + + if _, err := m.db.Exec(query); err != nil { + return err + } + return nil +} + +func (m *Sqlite) Open(url string) (database.Driver, error) { + purl, err := nurl.Parse(url) + if err != nil { + return nil, err + } + dbfile := strings.Replace(migrate.FilterCustomQuery(purl).String(), "sqlite://", "", 1) + db, err := sql.Open("libsql", dbfile) + if err != nil { + return nil, err + } + + qv := purl.Query() + + migrationsTable := qv.Get("x-migrations-table") + if len(migrationsTable) == 0 { + migrationsTable = DefaultMigrationsTable + } + + noTxWrap := false + if v := qv.Get("x-no-tx-wrap"); v != "" { + noTxWrap, err = strconv.ParseBool(v) + if err != nil { + return nil, fmt.Errorf("x-no-tx-wrap: %s", err) + } + } + + mx, err := WithInstance(db, &Config{ + DatabaseName: purl.Path, + MigrationsTable: migrationsTable, + NoTxWrap: noTxWrap, + }) + if err != nil { + return nil, err + } + return mx, nil +} + +func (m *Sqlite) Close() error { + return m.db.Close() +} + +func (m *Sqlite) Drop() (err error) { + query := `SELECT name FROM sqlite_master WHERE type = 'table';` + tables, err := m.db.Query(query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + defer func() { + if errClose := tables.Close(); errClose != nil { + err = multierror.Append(err, errClose) + } + }() + + tableNames := make([]string, 0) + for tables.Next() { + var tableName string + if err := tables.Scan(&tableName); err != nil { + return err + } + if len(tableName) > 0 { + tableNames = append(tableNames, tableName) + } + } + if err := tables.Err(); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + if len(tableNames) > 0 { + for _, t := range tableNames { + query := "DROP TABLE " + t + err = m.executeQuery(query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + query := "VACUUM" + _, err = m.db.Query(query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + return nil +} + +func (m *Sqlite) Lock() error { + if !m.isLocked.CAS(false, true) { + return database.ErrLocked + } + return nil +} + +func (m *Sqlite) Unlock() error { + if !m.isLocked.CAS(true, false) { + return database.ErrNotLocked + } + return nil +} + +func (m *Sqlite) Run(migration io.Reader) error { + migr, err := io.ReadAll(migration) + if err != nil { + return err + } + query := string(migr[:]) + + if m.config.NoTxWrap { + return m.executeQueryNoTx(query) + } + return m.executeQuery(query) +} + +func (m *Sqlite) executeQuery(query string) error { + tx, err := m.db.Begin() + if err != nil { + return &database.Error{OrigErr: err, Err: "transaction start failed"} + } + if _, err := tx.Exec(query); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } + return &database.Error{OrigErr: err, Query: []byte(query)} + } + if err := tx.Commit(); err != nil { + return &database.Error{OrigErr: err, Err: "transaction commit failed"} + } + return nil +} + +func (m *Sqlite) executeQueryNoTx(query string) error { + if _, err := m.db.Exec(query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + return nil +} + +func (m *Sqlite) SetVersion(version int, dirty bool) error { + tx, err := m.db.Begin() + if err != nil { + return &database.Error{OrigErr: err, Err: "transaction start failed"} + } + + query := "DELETE FROM " + m.config.MigrationsTable + if _, err := tx.Exec(query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + // Also re-write the schema version for nil dirty versions to prevent + // empty schema version for failed down migration on the first migration + // See: https://github.com/golang-migrate/migrate/issues/330 + if version >= 0 || (version == database.NilVersion && dirty) { + query := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (?, ?)`, m.config.MigrationsTable) + if _, err := tx.Exec(query, version, dirty); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + if err := tx.Commit(); err != nil { + return &database.Error{OrigErr: err, Err: "transaction commit failed"} + } + + return nil +} + +func (m *Sqlite) Version() (version int, dirty bool, err error) { + query := "SELECT version, dirty FROM " + m.config.MigrationsTable + " LIMIT 1" + err = m.db.QueryRow(query).Scan(&version, &dirty) + if err != nil { + return database.NilVersion, false, nil + } + return version, dirty, nil +} diff --git a/database/migrate/migrate.go b/database/migrate/migrate.go new file mode 100644 index 0000000..fb400ab --- /dev/null +++ b/database/migrate/migrate.go @@ -0,0 +1,7 @@ +package database + +import "database/sql" + +type Migrator struct { + DB *sql.DB +} diff --git a/database/sqlx.go b/database/sqlx.go new file mode 100644 index 0000000..b5ece78 --- /dev/null +++ b/database/sqlx.go @@ -0,0 +1,29 @@ +package database + +import ( + "log" + + "git.runcible.io/androiddrew/flux-feed/config" + "github.com/jmoiron/sqlx" +) + +func NewSqlx(cfg config.Database) *sqlx.DB { + var dsn string + + // TODO add additional database driver support + switch cfg.DatabaseDriver { + case "libsql": + if cfg.DatabaseDriver == "local" { + dsn = cfg.DatabasePath + } + + default: + log.Fatal("Must choose a database driver") + + } + + db := sqlx.MustConnect(cfg.DatabaseDriver, dsn) + + return db + +} diff --git a/go.mod b/go.mod index aa70edb..c5d48af 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,17 @@ -module git.runcible.io/androiddrew/fluxfeed +module git.runcible.io/androiddrew/flux-feed go 1.22.5 + +require ( + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/golang-migrate/migrate/v4 v4.17.1 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/jmoiron/sqlx v1.4.0 // indirect + github.com/joho/godotenv v1.5.1 // indirect + github.com/kelseyhightower/envconfig v1.4.0 // indirect + github.com/libsql/sqlite-antlr4-parser v0.0.0-20240327125255-dbf53b6cbf06 // indirect + github.com/tursodatabase/go-libsql v0.0.0-20240725130945-f44f2b84c8c8 // indirect + go.uber.org/atomic v1.7.0 // indirect + golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..415ea53 --- /dev/null +++ b/go.sum @@ -0,0 +1,32 @@ +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/golang-migrate/migrate/v4 v4.17.1 h1:4zQ6iqL6t6AiItphxJctQb3cFqWiSpMnX7wLTPnnYO4= +github.com/golang-migrate/migrate/v4 v4.17.1/go.mod h1:m8hinFyWBn0SA4QKHuKh175Pm9wjmxj3S2Mia7dbXzM= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= +github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/libsql/sqlite-antlr4-parser v0.0.0-20240327125255-dbf53b6cbf06 h1:JLvn7D+wXjH9g4Jsjo+VqmzTUpl/LX7vfr6VOfSWTdM= +github.com/libsql/sqlite-antlr4-parser v0.0.0-20240327125255-dbf53b6cbf06/go.mod h1:FUkZ5OHjlGPjnM2UyGJz9TypXQFgYqw6AFNO1UiROTM= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/tursodatabase/go-libsql v0.0.0-20240725130945-f44f2b84c8c8 h1:nxpR20uTcKWd+IcojEUCCieKTmBhrEnIhl0SiwUMBPk= +github.com/tursodatabase/go-libsql v0.0.0-20240725130945-f44f2b84c8c8/go.mod h1:TjsB2miB8RW2Sse8sdxzVTdeGlx74GloD5zJYUC38d8= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= -- 2.38.4 From a03cf826819231cf1e81753c735323bad030ec5b Mon Sep 17 00:00:00 2001 From: Drew Bednar Date: Sun, 28 Jul 2024 14:51:12 -0400 Subject: [PATCH 2/2] Working migration cmd --- .env.template | 4 +- .gitignore | 3 +- cmd/migrate/main.go | 32 ++- config/config.go | 4 + config/database.go | 9 +- database/migrate/libsql/libsql.go | 269 -------------------------- database/migrate/migrate.go | 7 - database/sqlx.go | 14 +- go.mod | 16 +- go.sum | 18 +- main.go | 17 -- main_test.go | 18 -- migrations/10_initial_schema.down.sql | 3 + migrations/10_initial_schema.up.sql | 72 +++++++ 14 files changed, 145 insertions(+), 341 deletions(-) delete mode 100644 database/migrate/libsql/libsql.go delete mode 100644 database/migrate/migrate.go delete mode 100644 main.go delete mode 100644 main_test.go create mode 100644 migrations/10_initial_schema.down.sql create mode 100644 migrations/10_initial_schema.up.sql diff --git a/.env.template b/.env.template index 1175f41..2c38b8d 100644 --- a/.env.template +++ b/.env.template @@ -1,3 +1,3 @@ -FLUX_DATABASE_DRIVER=libsql -FLUX_DATABASE_PATH=file:./flux-local.db +FLUX_DATABASE_DRIVER=sqlite3 +FLUX_DATABASE_DSN=./flux-local.db FLUX_MIGRATIONS_PATH=./migrations diff --git a/.gitignore b/.gitignore index 5cd1433..2ac73c2 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,5 @@ go.work # Flux Feed fluxfeed flux-feed -.env \ No newline at end of file +.env +flux-local.db \ No newline at end of file diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go index 9cbaa5d..e26b0d6 100644 --- a/cmd/migrate/main.go +++ b/cmd/migrate/main.go @@ -1,12 +1,40 @@ +// Entry point for applying database migrations for flux-feed application package main import ( + "database/sql" + "log" + "git.runcible.io/androiddrew/flux-feed/config" - "git.runcible.io/androiddrew/flux-feed/database" + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database/sqlite3" + _ "github.com/golang-migrate/migrate/v4/source/file" + _ "github.com/mattn/go-sqlite3" ) func main() { cfg := config.New() - dbHandle := database.NewSqlx(cfg.Database) + db, err := sql.Open(cfg.DatabaseDriver, cfg.DatabaseDSN) + defer db.Close() + + err = db.Ping() + if err != nil { + log.Fatal(err) + } + log.Printf("Connected to Database: %s", cfg.DatabaseDSN) + + driver, err := sqlite3.WithInstance(db, &sqlite3.Config{}) + + log.Printf("Using migrations path: %s", cfg.MigrationsPath) + m, err := migrate.NewWithDatabaseInstance( + cfg.MigrationsPath, cfg.DatabaseDriver, driver) + if err != nil { + log.Fatal(err) + } + log.Printf("Migrating: %s", cfg.DatabaseDSN) + err = m.Up() + if err != nil { + log.Fatal(err) + } } diff --git a/config/config.go b/config/config.go index db497ea..3ea15dc 100644 --- a/config/config.go +++ b/config/config.go @@ -1,3 +1,5 @@ +// Package config provides configuration structures and methods to load +// environment variables for the application. package config import ( @@ -6,10 +8,12 @@ import ( "github.com/joho/godotenv" ) +// Config holds the configuration for the application, including the database settings. It contains multiple embedded configuration structs. type Config struct { Database } +// New loads environment variables and returns a new configured Config instance. func New() *Config { err := godotenv.Load() if err != nil { diff --git a/config/database.go b/config/database.go index c1778c4..bbdb1b0 100644 --- a/config/database.go +++ b/config/database.go @@ -2,13 +2,14 @@ package config import "github.com/kelseyhightower/envconfig" +// Database holds the configuration for the database connection. type Database struct { - DatabaseDriver string `split_words:"true"` - DatabasePath string `split_words:"true"` - MigrationsPath string `default:"head" split_words:"true"` - LibsqlConnectorType string `default:"local" split_words:"true"` + DatabaseDriver string `split_words:"true"` + DatabaseDSN string `split_words:"true"` + MigrationsPath string `default:"head" split_words:"true"` } +// DataStore processes environment variables and returns a configured Database configuration struct. func DataStore() Database { var db Database envconfig.MustProcess("flux", &db) diff --git a/database/migrate/libsql/libsql.go b/database/migrate/libsql/libsql.go deleted file mode 100644 index 259550d..0000000 --- a/database/migrate/libsql/libsql.go +++ /dev/null @@ -1,269 +0,0 @@ -package libsql - -import ( - "database/sql" - "fmt" - "io" - nurl "net/url" - "strconv" - "strings" - - "go.uber.org/atomic" - - "github.com/golang-migrate/migrate/v4" - "github.com/golang-migrate/migrate/v4/database" - "github.com/hashicorp/go-multierror" - _ "github.com/tursodatabase/go-libsql" -) - -func init() { - database.Register("libsql", &Sqlite{}) -} - -var DefaultMigrationsTable = "schema_migrations" -var ( - ErrDatabaseDirty = fmt.Errorf("database is dirty") - ErrNilConfig = fmt.Errorf("no config") - ErrNoDatabaseName = fmt.Errorf("no database name") -) - -type Config struct { - MigrationsTable string - DatabaseName string - NoTxWrap bool -} - -type Sqlite struct { - db *sql.DB - isLocked atomic.Bool - - config *Config -} - -func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { - if config == nil { - return nil, ErrNilConfig - } - - if err := instance.Ping(); err != nil { - return nil, err - } - - if len(config.MigrationsTable) == 0 { - config.MigrationsTable = DefaultMigrationsTable - } - - mx := &Sqlite{ - db: instance, - config: config, - } - if err := mx.ensureVersionTable(); err != nil { - return nil, err - } - return mx, nil -} - -// ensureVersionTable checks if versions table exists and, if not, creates it. -// Note that this function locks the database, which deviates from the usual -// convention of "caller locks" in the Sqlite type. -func (m *Sqlite) ensureVersionTable() (err error) { - if err = m.Lock(); err != nil { - return err - } - - defer func() { - if e := m.Unlock(); e != nil { - if err == nil { - err = e - } else { - err = multierror.Append(err, e) - } - } - }() - - query := fmt.Sprintf(` - CREATE TABLE IF NOT EXISTS %s (version uint64,dirty bool); - CREATE UNIQUE INDEX IF NOT EXISTS version_unique ON %s (version); - `, m.config.MigrationsTable, m.config.MigrationsTable) - - if _, err := m.db.Exec(query); err != nil { - return err - } - return nil -} - -func (m *Sqlite) Open(url string) (database.Driver, error) { - purl, err := nurl.Parse(url) - if err != nil { - return nil, err - } - dbfile := strings.Replace(migrate.FilterCustomQuery(purl).String(), "sqlite://", "", 1) - db, err := sql.Open("libsql", dbfile) - if err != nil { - return nil, err - } - - qv := purl.Query() - - migrationsTable := qv.Get("x-migrations-table") - if len(migrationsTable) == 0 { - migrationsTable = DefaultMigrationsTable - } - - noTxWrap := false - if v := qv.Get("x-no-tx-wrap"); v != "" { - noTxWrap, err = strconv.ParseBool(v) - if err != nil { - return nil, fmt.Errorf("x-no-tx-wrap: %s", err) - } - } - - mx, err := WithInstance(db, &Config{ - DatabaseName: purl.Path, - MigrationsTable: migrationsTable, - NoTxWrap: noTxWrap, - }) - if err != nil { - return nil, err - } - return mx, nil -} - -func (m *Sqlite) Close() error { - return m.db.Close() -} - -func (m *Sqlite) Drop() (err error) { - query := `SELECT name FROM sqlite_master WHERE type = 'table';` - tables, err := m.db.Query(query) - if err != nil { - return &database.Error{OrigErr: err, Query: []byte(query)} - } - defer func() { - if errClose := tables.Close(); errClose != nil { - err = multierror.Append(err, errClose) - } - }() - - tableNames := make([]string, 0) - for tables.Next() { - var tableName string - if err := tables.Scan(&tableName); err != nil { - return err - } - if len(tableName) > 0 { - tableNames = append(tableNames, tableName) - } - } - if err := tables.Err(); err != nil { - return &database.Error{OrigErr: err, Query: []byte(query)} - } - - if len(tableNames) > 0 { - for _, t := range tableNames { - query := "DROP TABLE " + t - err = m.executeQuery(query) - if err != nil { - return &database.Error{OrigErr: err, Query: []byte(query)} - } - } - query := "VACUUM" - _, err = m.db.Query(query) - if err != nil { - return &database.Error{OrigErr: err, Query: []byte(query)} - } - } - - return nil -} - -func (m *Sqlite) Lock() error { - if !m.isLocked.CAS(false, true) { - return database.ErrLocked - } - return nil -} - -func (m *Sqlite) Unlock() error { - if !m.isLocked.CAS(true, false) { - return database.ErrNotLocked - } - return nil -} - -func (m *Sqlite) Run(migration io.Reader) error { - migr, err := io.ReadAll(migration) - if err != nil { - return err - } - query := string(migr[:]) - - if m.config.NoTxWrap { - return m.executeQueryNoTx(query) - } - return m.executeQuery(query) -} - -func (m *Sqlite) executeQuery(query string) error { - tx, err := m.db.Begin() - if err != nil { - return &database.Error{OrigErr: err, Err: "transaction start failed"} - } - if _, err := tx.Exec(query); err != nil { - if errRollback := tx.Rollback(); errRollback != nil { - err = multierror.Append(err, errRollback) - } - return &database.Error{OrigErr: err, Query: []byte(query)} - } - if err := tx.Commit(); err != nil { - return &database.Error{OrigErr: err, Err: "transaction commit failed"} - } - return nil -} - -func (m *Sqlite) executeQueryNoTx(query string) error { - if _, err := m.db.Exec(query); err != nil { - return &database.Error{OrigErr: err, Query: []byte(query)} - } - return nil -} - -func (m *Sqlite) SetVersion(version int, dirty bool) error { - tx, err := m.db.Begin() - if err != nil { - return &database.Error{OrigErr: err, Err: "transaction start failed"} - } - - query := "DELETE FROM " + m.config.MigrationsTable - if _, err := tx.Exec(query); err != nil { - return &database.Error{OrigErr: err, Query: []byte(query)} - } - - // Also re-write the schema version for nil dirty versions to prevent - // empty schema version for failed down migration on the first migration - // See: https://github.com/golang-migrate/migrate/issues/330 - if version >= 0 || (version == database.NilVersion && dirty) { - query := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (?, ?)`, m.config.MigrationsTable) - if _, err := tx.Exec(query, version, dirty); err != nil { - if errRollback := tx.Rollback(); errRollback != nil { - err = multierror.Append(err, errRollback) - } - return &database.Error{OrigErr: err, Query: []byte(query)} - } - } - - if err := tx.Commit(); err != nil { - return &database.Error{OrigErr: err, Err: "transaction commit failed"} - } - - return nil -} - -func (m *Sqlite) Version() (version int, dirty bool, err error) { - query := "SELECT version, dirty FROM " + m.config.MigrationsTable + " LIMIT 1" - err = m.db.QueryRow(query).Scan(&version, &dirty) - if err != nil { - return database.NilVersion, false, nil - } - return version, dirty, nil -} diff --git a/database/migrate/migrate.go b/database/migrate/migrate.go deleted file mode 100644 index fb400ab..0000000 --- a/database/migrate/migrate.go +++ /dev/null @@ -1,7 +0,0 @@ -package database - -import "database/sql" - -type Migrator struct { - DB *sql.DB -} diff --git a/database/sqlx.go b/database/sqlx.go index b5ece78..937a731 100644 --- a/database/sqlx.go +++ b/database/sqlx.go @@ -1,3 +1,9 @@ +// Package database provides utility functions for connecting to and +// interacting with various database backends. +// +// This package supports multiple database drivers including: +// - sqlite3 +// - others coming soon...maybe package database import ( @@ -7,16 +13,14 @@ import ( "github.com/jmoiron/sqlx" ) +// NewSqlx configures a new sqlx.DB from application config func NewSqlx(cfg config.Database) *sqlx.DB { var dsn string // TODO add additional database driver support switch cfg.DatabaseDriver { - case "libsql": - if cfg.DatabaseDriver == "local" { - dsn = cfg.DatabasePath - } - + case "sqlite3": + dsn = cfg.DatabaseDSN default: log.Fatal("Must choose a database driver") diff --git a/go.mod b/go.mod index c5d48af..9a74ba8 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,15 @@ module git.runcible.io/androiddrew/flux-feed go 1.22.5 require ( - github.com/antlr4-go/antlr/v4 v4.13.0 // indirect - github.com/golang-migrate/migrate/v4 v4.17.1 // indirect + github.com/golang-migrate/migrate/v4 v4.17.1 + github.com/jmoiron/sqlx v1.4.0 + github.com/joho/godotenv v1.5.1 + github.com/kelseyhightower/envconfig v1.4.0 + github.com/mattn/go-sqlite3 v1.14.22 +) + +require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/jmoiron/sqlx v1.4.0 // indirect - github.com/joho/godotenv v1.5.1 // indirect - github.com/kelseyhightower/envconfig v1.4.0 // indirect - github.com/libsql/sqlite-antlr4-parser v0.0.0-20240327125255-dbf53b6cbf06 // indirect - github.com/tursodatabase/go-libsql v0.0.0-20240725130945-f44f2b84c8c8 // indirect go.uber.org/atomic v1.7.0 // indirect - golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect ) diff --git a/go.sum b/go.sum index 415ea53..73dd23f 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,9 @@ +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= -github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/golang-migrate/migrate/v4 v4.17.1 h1:4zQ6iqL6t6AiItphxJctQb3cFqWiSpMnX7wLTPnnYO4= github.com/golang-migrate/migrate/v4 v4.17.1/go.mod h1:m8hinFyWBn0SA4QKHuKh175Pm9wjmxj3S2Mia7dbXzM= @@ -17,16 +18,17 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/libsql/sqlite-antlr4-parser v0.0.0-20240327125255-dbf53b6cbf06 h1:JLvn7D+wXjH9g4Jsjo+VqmzTUpl/LX7vfr6VOfSWTdM= -github.com/libsql/sqlite-antlr4-parser v0.0.0-20240327125255-dbf53b6cbf06/go.mod h1:FUkZ5OHjlGPjnM2UyGJz9TypXQFgYqw6AFNO1UiROTM= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/tursodatabase/go-libsql v0.0.0-20240725130945-f44f2b84c8c8 h1:nxpR20uTcKWd+IcojEUCCieKTmBhrEnIhl0SiwUMBPk= -github.com/tursodatabase/go-libsql v0.0.0-20240725130945-f44f2b84c8c8/go.mod h1:TjsB2miB8RW2Sse8sdxzVTdeGlx74GloD5zJYUC38d8= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go deleted file mode 100644 index ead7e6a..0000000 --- a/main.go +++ /dev/null @@ -1,17 +0,0 @@ -package main - -import ( - "fmt" - "io" - "os" -) - -const FluxGreeting string = "Welcome to Flux Feed\n" - -func HelloFluxFeed(out io.Writer) { - fmt.Fprint(out, FluxGreeting) -} - -func main() { - HelloFluxFeed(os.Stdout) -} diff --git a/main_test.go b/main_test.go deleted file mode 100644 index 66d6cfc..0000000 --- a/main_test.go +++ /dev/null @@ -1,18 +0,0 @@ -package main - -import ( - "bytes" - "testing" -) - -func TestCliGreeting(t *testing.T) { - buffer := &bytes.Buffer{} - - HelloFluxFeed(buffer) - got := buffer.String() - want := FluxGreeting - - if got != want { - t.Errorf("Got %s but wanted %s", got, want) - } -} diff --git a/migrations/10_initial_schema.down.sql b/migrations/10_initial_schema.down.sql new file mode 100644 index 0000000..a5875e4 --- /dev/null +++ b/migrations/10_initial_schema.down.sql @@ -0,0 +1,3 @@ +DROP TABLE IF EXISTS user; +DROP TABLE IF EXISTS feed; +DROP TABLE IF EXISTS entry; \ No newline at end of file diff --git a/migrations/10_initial_schema.up.sql b/migrations/10_initial_schema.up.sql new file mode 100644 index 0000000..9b9013b --- /dev/null +++ b/migrations/10_initial_schema.up.sql @@ -0,0 +1,72 @@ +PRAGMA foreign_keys=1; +CREATE TABLE user ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + email TEXT UNIQUE NOT NULL, + password TEXT NOT NULL, + kindle_email TEXT +); + +CREATE TABLE feed ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER UNIQUE NOT NULL, + url TEXT, + type TEXT NOT NULL, + name TEXT UNIQUE, + icon_url TEXT, + created TIMESTAMP NOT NULL, + updated TIMESTAMP NOT NULL, + last_fetch TIMESTAMP, + raw_data TEXT, + folder TEXT, + etag TEXT, + modified_header TEXT, + filters TEXT, + FOREIGN KEY(user_id) REFERENCES user(id) +); + +CREATE TABLE entry ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + feed_id INTEGER UNIQUE, + user_id INTEGER NOT NULL, + remote_id TEXT UNIQUE NOT NULL, + title TEXT, + username TEXT, + user_url TEXT, + display_name TEXT, + avatar_url TEXT, + content_short TEXT, + content_full TEXT, + target_url TEXT, + content_url TEXT, + comments_url TEXT, + media_url TEXT, + created TIMESTAMP NOT NULL, + updated TIMESTAMP NOT NULL, + display_date TIMESTAMP NOT NULL, + sort_date TIMESTAMP NOT NULL, + viewed TIMESTAMP, + favorited TIMESTAMP, + pinned TIMESTAMP, + sent_to_kindle TIMESTAMP, + raw_data TEXT, + header TEXT, + icon_url TEXT, + FOREIGN KEY(feed_id) REFERENCES feed(id), + FOREIGN KEY(user_id) REFERENCES user(id) +); + + +CREATE INDEX ix_feed_folder ON feed(folder); +CREATE INDEX ix_feed_user_id ON feed(user_id); +CREATE INDEX ix_name_user ON feed(user_id, name); +CREATE INDEX ix_feed_created ON feed(created); + +CREATE INDEX ix_entry_favorited ON entry(favorited); +CREATE INDEX ix_entry_created ON entry(created); +CREATE INDEX ix_entry_sort_date ON entry(sort_date); +CREATE INDEX ix_entry_sent_to_kindle ON entry(sent_to_kindle); +CREATE INDEX entry_sort_ts ON entry(sort_date DESC); +CREATE INDEX ix_entry_viewed ON entry(viewed); +CREATE INDEX ix_entry_pinned ON entry(pinned); +CREATE INDEX ix_entry_user_id ON entry(user_id); +CREATE INDEX ix_entry_username ON entry(username); -- 2.38.4