Half way to migrations

6 months ago · fc397d83c2
parent 44982e3e0e
commit fc397d83c2
10 changed files with 410 additions and 2 deletions
--- a/.env.template
+++ b/.env.template
@ -0,0 +1,3 @@
+FLUX_DATABASE_DRIVER=libsql
+FLUX_DATABASE_PATH=file:./flux-local.db
+FLUX_MIGRATIONS_PATH=./migrations
--- a/.gitignore
+++ b/.gitignore
@ -21,4 +21,7 @@
 # Go workspace file
 go.work

+# Flux Feed
 fluxfeed
+flux-feed
+.env
--- a/cmd/migrate/main.go
+++ b/cmd/migrate/main.go
@ -0,0 +1,12 @@
+package main
+
+import (
+	"git.runcible.io/androiddrew/flux-feed/config"
+	"git.runcible.io/androiddrew/flux-feed/database"
+)
+
+func main() {
+	cfg := config.New()
+	dbHandle := database.NewSqlx(cfg.Database)
+
+}
--- a/config/config.go
+++ b/config/config.go
@ -0,0 +1,22 @@
+package config
+
+import (
+	"log"
+
+	"github.com/joho/godotenv"
+)
+
+type Config struct {
+	Database
+}
+
+func New() *Config {
+	err := godotenv.Load()
+	if err != nil {
+		log.Println(err)
+	}
+
+	return &Config{
+		Database: DataStore(),
+	}
+}
--- a/config/database.go
+++ b/config/database.go
@ -0,0 +1,17 @@
+package config
+
+import "github.com/kelseyhightower/envconfig"
+
+type Database struct {
+	DatabaseDriver      string `split_words:"true"`
+	DatabasePath        string `split_words:"true"`
+	MigrationsPath      string `default:"head" split_words:"true"`
+	LibsqlConnectorType string `default:"local" split_words:"true"`
+}
+
+func DataStore() Database {
+	var db Database
+	envconfig.MustProcess("flux", &db)
+
+	return db
+}
--- a/database/migrate/libsql/libsql.go
+++ b/database/migrate/libsql/libsql.go
@ -0,0 +1,269 @@
+package libsql
+
+import (
+	"database/sql"
+	"fmt"
+	"io"
+	nurl "net/url"
+	"strconv"
+	"strings"
+
+	"go.uber.org/atomic"
+
+	"github.com/golang-migrate/migrate/v4"
+	"github.com/golang-migrate/migrate/v4/database"
+	"github.com/hashicorp/go-multierror"
+	_ "github.com/tursodatabase/go-libsql"
+)
+
+func init() {
+	database.Register("libsql", &Sqlite{})
+}
+
+var DefaultMigrationsTable = "schema_migrations"
+var (
+	ErrDatabaseDirty  = fmt.Errorf("database is dirty")
+	ErrNilConfig      = fmt.Errorf("no config")
+	ErrNoDatabaseName = fmt.Errorf("no database name")
+)
+
+type Config struct {
+	MigrationsTable string
+	DatabaseName    string
+	NoTxWrap        bool
+}
+
+type Sqlite struct {
+	db       *sql.DB
+	isLocked atomic.Bool
+
+	config *Config
+}
+
+func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) {
+	if config == nil {
+		return nil, ErrNilConfig
+	}
+
+	if err := instance.Ping(); err != nil {
+		return nil, err
+	}
+
+	if len(config.MigrationsTable) == 0 {
+		config.MigrationsTable = DefaultMigrationsTable
+	}
+
+	mx := &Sqlite{
+		db:     instance,
+		config: config,
+	}
+	if err := mx.ensureVersionTable(); err != nil {
+		return nil, err
+	}
+	return mx, nil
+}
+
+// ensureVersionTable checks if versions table exists and, if not, creates it.
+// Note that this function locks the database, which deviates from the usual
+// convention of "caller locks" in the Sqlite type.
+func (m *Sqlite) ensureVersionTable() (err error) {
+	if err = m.Lock(); err != nil {
+		return err
+	}
+
+	defer func() {
+		if e := m.Unlock(); e != nil {
+			if err == nil {
+				err = e
+			} else {
+				err = multierror.Append(err, e)
+			}
+		}
+	}()
+
+	query := fmt.Sprintf(`
+	CREATE TABLE IF NOT EXISTS %s (version uint64,dirty bool);
+  CREATE UNIQUE INDEX IF NOT EXISTS version_unique ON %s (version);
+  `, m.config.MigrationsTable, m.config.MigrationsTable)
+
+	if _, err := m.db.Exec(query); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (m *Sqlite) Open(url string) (database.Driver, error) {
+	purl, err := nurl.Parse(url)
+	if err != nil {
+		return nil, err
+	}
+	dbfile := strings.Replace(migrate.FilterCustomQuery(purl).String(), "sqlite://", "", 1)
+	db, err := sql.Open("libsql", dbfile)
+	if err != nil {
+		return nil, err
+	}
+
+	qv := purl.Query()
+
+	migrationsTable := qv.Get("x-migrations-table")
+	if len(migrationsTable) == 0 {
+		migrationsTable = DefaultMigrationsTable
+	}
+
+	noTxWrap := false
+	if v := qv.Get("x-no-tx-wrap"); v != "" {
+		noTxWrap, err = strconv.ParseBool(v)
+		if err != nil {
+			return nil, fmt.Errorf("x-no-tx-wrap: %s", err)
+		}
+	}
+
+	mx, err := WithInstance(db, &Config{
+		DatabaseName:    purl.Path,
+		MigrationsTable: migrationsTable,
+		NoTxWrap:        noTxWrap,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return mx, nil
+}
+
+func (m *Sqlite) Close() error {
+	return m.db.Close()
+}
+
+func (m *Sqlite) Drop() (err error) {
+	query := `SELECT name FROM sqlite_master WHERE type = 'table';`
+	tables, err := m.db.Query(query)
+	if err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+	defer func() {
+		if errClose := tables.Close(); errClose != nil {
+			err = multierror.Append(err, errClose)
+		}
+	}()
+
+	tableNames := make([]string, 0)
+	for tables.Next() {
+		var tableName string
+		if err := tables.Scan(&tableName); err != nil {
+			return err
+		}
+		if len(tableName) > 0 {
+			tableNames = append(tableNames, tableName)
+		}
+	}
+	if err := tables.Err(); err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+
+	if len(tableNames) > 0 {
+		for _, t := range tableNames {
+			query := "DROP TABLE " + t
+			err = m.executeQuery(query)
+			if err != nil {
+				return &database.Error{OrigErr: err, Query: []byte(query)}
+			}
+		}
+		query := "VACUUM"
+		_, err = m.db.Query(query)
+		if err != nil {
+			return &database.Error{OrigErr: err, Query: []byte(query)}
+		}
+	}
+
+	return nil
+}
+
+func (m *Sqlite) Lock() error {
+	if !m.isLocked.CAS(false, true) {
+		return database.ErrLocked
+	}
+	return nil
+}
+
+func (m *Sqlite) Unlock() error {
+	if !m.isLocked.CAS(true, false) {
+		return database.ErrNotLocked
+	}
+	return nil
+}
+
+func (m *Sqlite) Run(migration io.Reader) error {
+	migr, err := io.ReadAll(migration)
+	if err != nil {
+		return err
+	}
+	query := string(migr[:])
+
+	if m.config.NoTxWrap {
+		return m.executeQueryNoTx(query)
+	}
+	return m.executeQuery(query)
+}
+
+func (m *Sqlite) executeQuery(query string) error {
+	tx, err := m.db.Begin()
+	if err != nil {
+		return &database.Error{OrigErr: err, Err: "transaction start failed"}
+	}
+	if _, err := tx.Exec(query); err != nil {
+		if errRollback := tx.Rollback(); errRollback != nil {
+			err = multierror.Append(err, errRollback)
+		}
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+	if err := tx.Commit(); err != nil {
+		return &database.Error{OrigErr: err, Err: "transaction commit failed"}
+	}
+	return nil
+}
+
+func (m *Sqlite) executeQueryNoTx(query string) error {
+	if _, err := m.db.Exec(query); err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+	return nil
+}
+
+func (m *Sqlite) SetVersion(version int, dirty bool) error {
+	tx, err := m.db.Begin()
+	if err != nil {
+		return &database.Error{OrigErr: err, Err: "transaction start failed"}
+	}
+
+	query := "DELETE FROM " + m.config.MigrationsTable
+	if _, err := tx.Exec(query); err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+
+	// Also re-write the schema version for nil dirty versions to prevent
+	// empty schema version for failed down migration on the first migration
+	// See: https://github.com/golang-migrate/migrate/issues/330
+	if version >= 0 || (version == database.NilVersion && dirty) {
+		query := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (?, ?)`, m.config.MigrationsTable)
+		if _, err := tx.Exec(query, version, dirty); err != nil {
+			if errRollback := tx.Rollback(); errRollback != nil {
+				err = multierror.Append(err, errRollback)
+			}
+			return &database.Error{OrigErr: err, Query: []byte(query)}
+		}
+	}
+
+	if err := tx.Commit(); err != nil {
+		return &database.Error{OrigErr: err, Err: "transaction commit failed"}
+	}
+
+	return nil
+}
+
+func (m *Sqlite) Version() (version int, dirty bool, err error) {
+	query := "SELECT version, dirty FROM " + m.config.MigrationsTable + " LIMIT 1"
+	err = m.db.QueryRow(query).Scan(&version, &dirty)
+	if err != nil {
+		return database.NilVersion, false, nil
+	}
+	return version, dirty, nil
+}
--- a/database/migrate/migrate.go
+++ b/database/migrate/migrate.go
@ -0,0 +1,7 @@
+package database
+
+import "database/sql"
+
+type Migrator struct {
+	DB *sql.DB
+}
--- a/database/sqlx.go
+++ b/database/sqlx.go
@ -0,0 +1,29 @@
+package database
+
+import (
+	"log"
+
+	"git.runcible.io/androiddrew/flux-feed/config"
+	"github.com/jmoiron/sqlx"
+)
+
+func NewSqlx(cfg config.Database) *sqlx.DB {
+	var dsn string
+
+	// TODO add additional database driver support
+	switch cfg.DatabaseDriver {
+	case "libsql":
+		if cfg.DatabaseDriver == "local" {
+			dsn = cfg.DatabasePath
+		}
+
+	default:
+		log.Fatal("Must choose a database driver")
+
+	}
+
+	db := sqlx.MustConnect(cfg.DatabaseDriver, dsn)
+
+	return db
+
+}
--- a/go.mod
+++ b/go.mod
@ -1,3 +1,17 @@
-module git.runcible.io/androiddrew/fluxfeed
+module git.runcible.io/androiddrew/flux-feed

 go 1.22.5
+
+require (
+	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
+	github.com/golang-migrate/migrate/v4 v4.17.1 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/jmoiron/sqlx v1.4.0 // indirect
+	github.com/joho/godotenv v1.5.1 // indirect
+	github.com/kelseyhightower/envconfig v1.4.0 // indirect
+	github.com/libsql/sqlite-antlr4-parser v0.0.0-20240327125255-dbf53b6cbf06 // indirect
+	github.com/tursodatabase/go-libsql v0.0.0-20240725130945-f44f2b84c8c8 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
+)
--- a/go.sum
+++ b/go.sum
@ -0,0 +1,32 @@
+filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
+github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
+github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
+github.com/golang-migrate/migrate/v4 v4.17.1 h1:4zQ6iqL6t6AiItphxJctQb3cFqWiSpMnX7wLTPnnYO4=
+github.com/golang-migrate/migrate/v4 v4.17.1/go.mod h1:m8hinFyWBn0SA4QKHuKh175Pm9wjmxj3S2Mia7dbXzM=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
+github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
+github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
+github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/libsql/sqlite-antlr4-parser v0.0.0-20240327125255-dbf53b6cbf06 h1:JLvn7D+wXjH9g4Jsjo+VqmzTUpl/LX7vfr6VOfSWTdM=
+github.com/libsql/sqlite-antlr4-parser v0.0.0-20240327125255-dbf53b6cbf06/go.mod h1:FUkZ5OHjlGPjnM2UyGJz9TypXQFgYqw6AFNO1UiROTM=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/tursodatabase/go-libsql v0.0.0-20240725130945-f44f2b84c8c8 h1:nxpR20uTcKWd+IcojEUCCieKTmBhrEnIhl0SiwUMBPk=
+github.com/tursodatabase/go-libsql v0.0.0-20240725130945-f44f2b84c8c8/go.mod h1:TjsB2miB8RW2Sse8sdxzVTdeGlx74GloD5zJYUC38d8=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU=
+golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=