Working migration cmd
parent
fc397d83c2
commit
a03cf82681
@ -1,3 +1,3 @@
|
||||
FLUX_DATABASE_DRIVER=libsql
|
||||
FLUX_DATABASE_PATH=file:./flux-local.db
|
||||
FLUX_DATABASE_DRIVER=sqlite3
|
||||
FLUX_DATABASE_DSN=./flux-local.db
|
||||
FLUX_MIGRATIONS_PATH=./migrations
|
||||
|
||||
@ -1,12 +1,40 @@
|
||||
// Entry point for applying database migrations for flux-feed application
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"log"
|
||||
|
||||
"git.runcible.io/androiddrew/flux-feed/config"
|
||||
"git.runcible.io/androiddrew/flux-feed/database"
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
"github.com/golang-migrate/migrate/v4/database/sqlite3"
|
||||
_ "github.com/golang-migrate/migrate/v4/source/file"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg := config.New()
|
||||
dbHandle := database.NewSqlx(cfg.Database)
|
||||
db, err := sql.Open(cfg.DatabaseDriver, cfg.DatabaseDSN)
|
||||
defer db.Close()
|
||||
|
||||
err = db.Ping()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
log.Printf("Connected to Database: %s", cfg.DatabaseDSN)
|
||||
|
||||
driver, err := sqlite3.WithInstance(db, &sqlite3.Config{})
|
||||
|
||||
log.Printf("Using migrations path: %s", cfg.MigrationsPath)
|
||||
|
||||
m, err := migrate.NewWithDatabaseInstance(
|
||||
cfg.MigrationsPath, cfg.DatabaseDriver, driver)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
log.Printf("Migrating: %s", cfg.DatabaseDSN)
|
||||
err = m.Up()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,269 +0,0 @@
|
||||
package libsql
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io"
|
||||
nurl "net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"go.uber.org/atomic"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
_ "github.com/tursodatabase/go-libsql"
|
||||
)
|
||||
|
||||
func init() {
|
||||
database.Register("libsql", &Sqlite{})
|
||||
}
|
||||
|
||||
var DefaultMigrationsTable = "schema_migrations"
|
||||
var (
|
||||
ErrDatabaseDirty = fmt.Errorf("database is dirty")
|
||||
ErrNilConfig = fmt.Errorf("no config")
|
||||
ErrNoDatabaseName = fmt.Errorf("no database name")
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
MigrationsTable string
|
||||
DatabaseName string
|
||||
NoTxWrap bool
|
||||
}
|
||||
|
||||
type Sqlite struct {
|
||||
db *sql.DB
|
||||
isLocked atomic.Bool
|
||||
|
||||
config *Config
|
||||
}
|
||||
|
||||
func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) {
|
||||
if config == nil {
|
||||
return nil, ErrNilConfig
|
||||
}
|
||||
|
||||
if err := instance.Ping(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(config.MigrationsTable) == 0 {
|
||||
config.MigrationsTable = DefaultMigrationsTable
|
||||
}
|
||||
|
||||
mx := &Sqlite{
|
||||
db: instance,
|
||||
config: config,
|
||||
}
|
||||
if err := mx.ensureVersionTable(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return mx, nil
|
||||
}
|
||||
|
||||
// ensureVersionTable checks if versions table exists and, if not, creates it.
|
||||
// Note that this function locks the database, which deviates from the usual
|
||||
// convention of "caller locks" in the Sqlite type.
|
||||
func (m *Sqlite) ensureVersionTable() (err error) {
|
||||
if err = m.Lock(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if e := m.Unlock(); e != nil {
|
||||
if err == nil {
|
||||
err = e
|
||||
} else {
|
||||
err = multierror.Append(err, e)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
query := fmt.Sprintf(`
|
||||
CREATE TABLE IF NOT EXISTS %s (version uint64,dirty bool);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS version_unique ON %s (version);
|
||||
`, m.config.MigrationsTable, m.config.MigrationsTable)
|
||||
|
||||
if _, err := m.db.Exec(query); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Sqlite) Open(url string) (database.Driver, error) {
|
||||
purl, err := nurl.Parse(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dbfile := strings.Replace(migrate.FilterCustomQuery(purl).String(), "sqlite://", "", 1)
|
||||
db, err := sql.Open("libsql", dbfile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
qv := purl.Query()
|
||||
|
||||
migrationsTable := qv.Get("x-migrations-table")
|
||||
if len(migrationsTable) == 0 {
|
||||
migrationsTable = DefaultMigrationsTable
|
||||
}
|
||||
|
||||
noTxWrap := false
|
||||
if v := qv.Get("x-no-tx-wrap"); v != "" {
|
||||
noTxWrap, err = strconv.ParseBool(v)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("x-no-tx-wrap: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
mx, err := WithInstance(db, &Config{
|
||||
DatabaseName: purl.Path,
|
||||
MigrationsTable: migrationsTable,
|
||||
NoTxWrap: noTxWrap,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return mx, nil
|
||||
}
|
||||
|
||||
func (m *Sqlite) Close() error {
|
||||
return m.db.Close()
|
||||
}
|
||||
|
||||
func (m *Sqlite) Drop() (err error) {
|
||||
query := `SELECT name FROM sqlite_master WHERE type = 'table';`
|
||||
tables, err := m.db.Query(query)
|
||||
if err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
defer func() {
|
||||
if errClose := tables.Close(); errClose != nil {
|
||||
err = multierror.Append(err, errClose)
|
||||
}
|
||||
}()
|
||||
|
||||
tableNames := make([]string, 0)
|
||||
for tables.Next() {
|
||||
var tableName string
|
||||
if err := tables.Scan(&tableName); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(tableName) > 0 {
|
||||
tableNames = append(tableNames, tableName)
|
||||
}
|
||||
}
|
||||
if err := tables.Err(); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
if len(tableNames) > 0 {
|
||||
for _, t := range tableNames {
|
||||
query := "DROP TABLE " + t
|
||||
err = m.executeQuery(query)
|
||||
if err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
}
|
||||
query := "VACUUM"
|
||||
_, err = m.db.Query(query)
|
||||
if err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Sqlite) Lock() error {
|
||||
if !m.isLocked.CAS(false, true) {
|
||||
return database.ErrLocked
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Sqlite) Unlock() error {
|
||||
if !m.isLocked.CAS(true, false) {
|
||||
return database.ErrNotLocked
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Sqlite) Run(migration io.Reader) error {
|
||||
migr, err := io.ReadAll(migration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
query := string(migr[:])
|
||||
|
||||
if m.config.NoTxWrap {
|
||||
return m.executeQueryNoTx(query)
|
||||
}
|
||||
return m.executeQuery(query)
|
||||
}
|
||||
|
||||
func (m *Sqlite) executeQuery(query string) error {
|
||||
tx, err := m.db.Begin()
|
||||
if err != nil {
|
||||
return &database.Error{OrigErr: err, Err: "transaction start failed"}
|
||||
}
|
||||
if _, err := tx.Exec(query); err != nil {
|
||||
if errRollback := tx.Rollback(); errRollback != nil {
|
||||
err = multierror.Append(err, errRollback)
|
||||
}
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return &database.Error{OrigErr: err, Err: "transaction commit failed"}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Sqlite) executeQueryNoTx(query string) error {
|
||||
if _, err := m.db.Exec(query); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Sqlite) SetVersion(version int, dirty bool) error {
|
||||
tx, err := m.db.Begin()
|
||||
if err != nil {
|
||||
return &database.Error{OrigErr: err, Err: "transaction start failed"}
|
||||
}
|
||||
|
||||
query := "DELETE FROM " + m.config.MigrationsTable
|
||||
if _, err := tx.Exec(query); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
// Also re-write the schema version for nil dirty versions to prevent
|
||||
// empty schema version for failed down migration on the first migration
|
||||
// See: https://github.com/golang-migrate/migrate/issues/330
|
||||
if version >= 0 || (version == database.NilVersion && dirty) {
|
||||
query := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (?, ?)`, m.config.MigrationsTable)
|
||||
if _, err := tx.Exec(query, version, dirty); err != nil {
|
||||
if errRollback := tx.Rollback(); errRollback != nil {
|
||||
err = multierror.Append(err, errRollback)
|
||||
}
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return &database.Error{OrigErr: err, Err: "transaction commit failed"}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Sqlite) Version() (version int, dirty bool, err error) {
|
||||
query := "SELECT version, dirty FROM " + m.config.MigrationsTable + " LIMIT 1"
|
||||
err = m.db.QueryRow(query).Scan(&version, &dirty)
|
||||
if err != nil {
|
||||
return database.NilVersion, false, nil
|
||||
}
|
||||
return version, dirty, nil
|
||||
}
|
||||
@ -1,7 +0,0 @@
|
||||
package database
|
||||
|
||||
import "database/sql"
|
||||
|
||||
type Migrator struct {
|
||||
DB *sql.DB
|
||||
}
|
||||
@ -1,17 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
const FluxGreeting string = "Welcome to Flux Feed\n"
|
||||
|
||||
func HelloFluxFeed(out io.Writer) {
|
||||
fmt.Fprint(out, FluxGreeting)
|
||||
}
|
||||
|
||||
func main() {
|
||||
HelloFluxFeed(os.Stdout)
|
||||
}
|
||||
@ -1,18 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCliGreeting(t *testing.T) {
|
||||
buffer := &bytes.Buffer{}
|
||||
|
||||
HelloFluxFeed(buffer)
|
||||
got := buffer.String()
|
||||
want := FluxGreeting
|
||||
|
||||
if got != want {
|
||||
t.Errorf("Got %s but wanted %s", got, want)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,3 @@
|
||||
DROP TABLE IF EXISTS user;
|
||||
DROP TABLE IF EXISTS feed;
|
||||
DROP TABLE IF EXISTS entry;
|
||||
@ -0,0 +1,72 @@
|
||||
PRAGMA foreign_keys=1;
|
||||
CREATE TABLE user (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
email TEXT UNIQUE NOT NULL,
|
||||
password TEXT NOT NULL,
|
||||
kindle_email TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE feed (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id INTEGER UNIQUE NOT NULL,
|
||||
url TEXT,
|
||||
type TEXT NOT NULL,
|
||||
name TEXT UNIQUE,
|
||||
icon_url TEXT,
|
||||
created TIMESTAMP NOT NULL,
|
||||
updated TIMESTAMP NOT NULL,
|
||||
last_fetch TIMESTAMP,
|
||||
raw_data TEXT,
|
||||
folder TEXT,
|
||||
etag TEXT,
|
||||
modified_header TEXT,
|
||||
filters TEXT,
|
||||
FOREIGN KEY(user_id) REFERENCES user(id)
|
||||
);
|
||||
|
||||
CREATE TABLE entry (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
feed_id INTEGER UNIQUE,
|
||||
user_id INTEGER NOT NULL,
|
||||
remote_id TEXT UNIQUE NOT NULL,
|
||||
title TEXT,
|
||||
username TEXT,
|
||||
user_url TEXT,
|
||||
display_name TEXT,
|
||||
avatar_url TEXT,
|
||||
content_short TEXT,
|
||||
content_full TEXT,
|
||||
target_url TEXT,
|
||||
content_url TEXT,
|
||||
comments_url TEXT,
|
||||
media_url TEXT,
|
||||
created TIMESTAMP NOT NULL,
|
||||
updated TIMESTAMP NOT NULL,
|
||||
display_date TIMESTAMP NOT NULL,
|
||||
sort_date TIMESTAMP NOT NULL,
|
||||
viewed TIMESTAMP,
|
||||
favorited TIMESTAMP,
|
||||
pinned TIMESTAMP,
|
||||
sent_to_kindle TIMESTAMP,
|
||||
raw_data TEXT,
|
||||
header TEXT,
|
||||
icon_url TEXT,
|
||||
FOREIGN KEY(feed_id) REFERENCES feed(id),
|
||||
FOREIGN KEY(user_id) REFERENCES user(id)
|
||||
);
|
||||
|
||||
|
||||
CREATE INDEX ix_feed_folder ON feed(folder);
|
||||
CREATE INDEX ix_feed_user_id ON feed(user_id);
|
||||
CREATE INDEX ix_name_user ON feed(user_id, name);
|
||||
CREATE INDEX ix_feed_created ON feed(created);
|
||||
|
||||
CREATE INDEX ix_entry_favorited ON entry(favorited);
|
||||
CREATE INDEX ix_entry_created ON entry(created);
|
||||
CREATE INDEX ix_entry_sort_date ON entry(sort_date);
|
||||
CREATE INDEX ix_entry_sent_to_kindle ON entry(sent_to_kindle);
|
||||
CREATE INDEX entry_sort_ts ON entry(sort_date DESC);
|
||||
CREATE INDEX ix_entry_viewed ON entry(viewed);
|
||||
CREATE INDEX ix_entry_pinned ON entry(pinned);
|
||||
CREATE INDEX ix_entry_user_id ON entry(user_id);
|
||||
CREATE INDEX ix_entry_username ON entry(username);
|
||||
Loading…
Reference in New Issue