memos/store/test/migrator_test.go

550 lines
19 KiB
Go

package test
import (
"context"
"database/sql"
"fmt"
"testing"
"time"
"github.com/stretchr/testify/require"
colorpb "google.golang.org/genproto/googleapis/type/color"
"google.golang.org/protobuf/encoding/protojson"
storepb "github.com/usememos/memos/proto/gen/store"
"github.com/usememos/memos/store"
)
// TestFreshInstall verifies that LATEST.sql applies correctly on a fresh database.
// This is essentially what NewTestingStore already does, but we make it explicit.
func TestFreshInstall(t *testing.T) {
t.Parallel()
ctx := context.Background()
// NewTestingStore creates a fresh database and runs Migrate()
// which applies LATEST.sql for uninitialized databases
ts := NewTestingStore(ctx, t)
// Verify migration completed successfully
currentSchemaVersion, err := ts.GetCurrentSchemaVersion()
require.NoError(t, err)
require.NotEmpty(t, currentSchemaVersion, "schema version should be set after fresh install")
// Verify we can read instance settings (basic sanity check)
instanceSetting, err := ts.GetInstanceBasicSetting(ctx)
require.NoError(t, err)
require.Equal(t, currentSchemaVersion, instanceSetting.SchemaVersion)
}
// TestMigrationReRun verifies that re-running the migration on an already
// migrated database does not fail or cause issues. This simulates a
// scenario where the server is restarted.
func TestMigrationReRun(t *testing.T) {
t.Parallel()
ctx := context.Background()
// Use the shared testing store which already runs migrations on init
ts := NewTestingStore(ctx, t)
// Get current version
initialVersion, err := ts.GetCurrentSchemaVersion()
require.NoError(t, err)
// Manually trigger migration again
err = ts.Migrate(ctx)
require.NoError(t, err, "re-running migration should not fail")
// Verify version hasn't changed (or at least is valid)
finalVersion, err := ts.GetCurrentSchemaVersion()
require.NoError(t, err)
require.Equal(t, initialVersion, finalVersion, "version should match after re-run")
}
// TestMigrationWithData verifies that migration preserves data integrity.
// Creates data, then re-runs migration and verifies data is still accessible.
func TestMigrationWithData(t *testing.T) {
t.Parallel()
ctx := context.Background()
ts := NewTestingStore(ctx, t)
// Create a user and memo before re-running migration
user, err := createTestingHostUser(ctx, ts)
require.NoError(t, err, "should create user")
originalMemo, err := ts.CreateMemo(ctx, &store.Memo{
UID: "migration-data-test",
CreatorID: user.ID,
Content: "Data before migration re-run",
Visibility: store.Public,
})
require.NoError(t, err, "should create memo")
// Re-run migration
err = ts.Migrate(ctx)
require.NoError(t, err, "re-running migration should not fail")
// Verify data is still accessible
memo, err := ts.GetMemo(ctx, &store.FindMemo{UID: &originalMemo.UID})
require.NoError(t, err, "should retrieve memo after migration")
require.Equal(t, "Data before migration re-run", memo.Content, "memo content should be preserved")
}
// TestMigrationMultipleReRuns verifies that migration is idempotent
// even when run multiple times in succession.
func TestMigrationMultipleReRuns(t *testing.T) {
t.Parallel()
ctx := context.Background()
ts := NewTestingStore(ctx, t)
// Get initial version
initialVersion, err := ts.GetCurrentSchemaVersion()
require.NoError(t, err)
// Run migration multiple times
for i := 0; i < 3; i++ {
err = ts.Migrate(ctx)
require.NoError(t, err, "migration run %d should not fail", i+1)
}
// Verify version is still correct
finalVersion, err := ts.GetCurrentSchemaVersion()
require.NoError(t, err)
require.Equal(t, initialVersion, finalVersion, "version should remain unchanged after multiple re-runs")
}
// TestMigrationMemoViewSetting verifies the 0.31 rename of the SHORTCUTS user setting
// to MEMO_VIEWS on every driver, since each one rewrites the stored JSON with its own
// dialect-specific functions.
func TestMigrationMemoViewSetting(t *testing.T) {
ctx := context.Background()
driver := getDriverFromEnv()
var dsn string
switch driver {
case "sqlite":
dsn = fmt.Sprintf("%s/memos_memo_view_migration.db", t.TempDir())
case "mysql":
dsn = GetMySQLDSN(t)
case "postgres":
dsn = GetPostgresDSN(t)
default:
t.Fatalf("unsupported driver: %s", driver)
}
db, err := sql.Open(driver, dsn)
require.NoError(t, err)
_, err = db.ExecContext(ctx, legacySchemaFixture(driver))
require.NoError(t, err)
basicSettingBytes, err := protojson.Marshal(&storepb.InstanceBasicSetting{SchemaVersion: "0.30.2"})
require.NoError(t, err)
insertBasicSetting := "INSERT INTO system_setting (name, value, description) VALUES ('BASIC', ?, '')"
if driver == "postgres" {
insertBasicSetting = "INSERT INTO system_setting (name, value, description) VALUES ('BASIC', $1, '')"
}
_, err = db.ExecContext(ctx, insertBasicSetting, string(basicSettingBytes))
require.NoError(t, err)
settingKeyColumn := "key"
if driver == "mysql" {
settingKeyColumn = "`key`"
}
// Bound rather than inlined: MySQL treats backslash as an escape character inside
// string literals, so an inlined \" would collapse and corrupt the fixture's JSON.
insertSetting := fmt.Sprintf("INSERT INTO user_setting (user_id, %[1]s, value) VALUES (?, ?, ?)", settingKeyColumn)
if driver == "postgres" {
insertSetting = fmt.Sprintf("INSERT INTO user_setting (user_id, %[1]s, value) VALUES ($1, $2, $3)", settingKeyColumn)
}
settingRows := []struct {
userID int
key string
value string
}{
// A normal legacy row, must be renamed and rewritten.
{1, "SHORTCUTS", `{"shortcuts":[{"id":"work","title":"shortcuts","filter":"tag in [\"work\"]"}]}`},
// A corrupt value, must be left alone rather than aborting the upgrade.
{2, "SHORTCUTS", `{oops}`},
// Already has a MEMO_VIEWS row, must not trip UNIQUE(user_id, key).
{3, "SHORTCUTS", `{"shortcuts":[{"id":"old","title":"old","filter":"tag in [\"old\"]"}]}`},
{3, "MEMO_VIEWS", `{"memoViews":[{"id":"new","title":"new","filter":"tag in [\"new\"]"}]}`},
}
for _, row := range settingRows {
_, err = db.ExecContext(ctx, insertSetting, row.userID, row.key, row.value)
require.NoError(t, err)
}
require.NoError(t, db.Close())
ts := NewTestingStoreWithDSN(ctx, t, driver, dsn)
require.NoError(t, ts.Migrate(ctx))
defer ts.Close()
migratedUserID := int32(1)
setting, err := ts.GetUserSetting(ctx, &store.FindUserSetting{
UserID: &migratedUserID,
Key: storepb.UserSetting_MEMO_VIEWS,
})
require.NoError(t, err)
require.NotNil(t, setting)
require.Len(t, setting.GetMemoViews().GetMemoViews(), 1)
require.Equal(t, "work", setting.GetMemoViews().GetMemoViews()[0].GetId())
require.Equal(t, "shortcuts", setting.GetMemoViews().GetMemoViews()[0].GetTitle())
require.Equal(t, `tag in ["work"]`, setting.GetMemoViews().GetMemoViews()[0].GetFilter())
// Malformed object-like JSON must neither abort the migration nor be renamed.
findCorruptSetting := fmt.Sprintf("SELECT value FROM user_setting WHERE user_id = ? AND %s = ?", settingKeyColumn)
if driver == "postgres" {
findCorruptSetting = fmt.Sprintf("SELECT value FROM user_setting WHERE user_id = $1 AND %s = $2", settingKeyColumn)
}
var corruptValue string
err = ts.GetDriver().GetDB().QueryRowContext(ctx, findCorruptSetting, 2, "SHORTCUTS").Scan(&corruptValue)
require.NoError(t, err)
require.Equal(t, `{oops}`, corruptValue)
// The pre-existing MEMO_VIEWS row wins; the legacy row is left behind untouched.
conflictUserID := int32(3)
conflicting, err := ts.GetUserSetting(ctx, &store.FindUserSetting{
UserID: &conflictUserID,
Key: storepb.UserSetting_MEMO_VIEWS,
})
require.NoError(t, err)
require.NotNil(t, conflicting)
require.Len(t, conflicting.GetMemoViews().GetMemoViews(), 1)
require.Equal(t, "new", conflicting.GetMemoViews().GetMemoViews()[0].GetId())
}
// TestMigrationCopiesInstanceTagsToUserSettings verifies instance tag metadata is copied into user settings.
func TestMigrationCopiesInstanceTagsToUserSettings(t *testing.T) {
if getDriverFromEnv() != "sqlite" {
t.Skip("skipping focused migration fixture for non-sqlite driver")
}
ctx := context.Background()
dsn := fmt.Sprintf("%s/memos_tag_migration.db", t.TempDir())
db, err := sql.Open("sqlite", dsn)
require.NoError(t, err)
_, err = db.ExecContext(ctx, `
CREATE TABLE system_setting (
name TEXT NOT NULL,
value TEXT NOT NULL,
description TEXT NOT NULL DEFAULT '',
UNIQUE(name)
);
CREATE TABLE user (
id INTEGER PRIMARY KEY AUTOINCREMENT,
created_ts BIGINT NOT NULL DEFAULT (strftime('%s', 'now')),
updated_ts BIGINT NOT NULL DEFAULT (strftime('%s', 'now')),
row_status TEXT NOT NULL CHECK (row_status IN ('NORMAL', 'ARCHIVED')) DEFAULT 'NORMAL',
username TEXT NOT NULL UNIQUE,
role TEXT NOT NULL DEFAULT 'USER',
email TEXT NOT NULL DEFAULT '',
nickname TEXT NOT NULL DEFAULT '',
password_hash TEXT NOT NULL,
avatar_url TEXT NOT NULL DEFAULT '',
description TEXT NOT NULL DEFAULT ''
);
CREATE TABLE user_setting (
user_id INTEGER NOT NULL,
key TEXT NOT NULL,
value TEXT NOT NULL,
UNIQUE(user_id, key)
);
CREATE TABLE memo (
id INTEGER PRIMARY KEY AUTOINCREMENT
);
`)
require.NoError(t, err)
basicSettingBytes, err := protojson.Marshal(&storepb.InstanceBasicSetting{SchemaVersion: "0.29.1"})
require.NoError(t, err)
tagsSettingBytes, err := protojson.Marshal(&storepb.InstanceTagsSetting{
Tags: map[string]*storepb.InstanceTagMetadata{
"bug": {
BackgroundColor: &colorpb.Color{Red: 0.9, Green: 0.1, Blue: 0.1},
},
"private/.*": {
BlurContent: true,
},
},
})
require.NoError(t, err)
existingUserTagsBytes, err := protojson.Marshal(&storepb.TagsUserSetting{
Tags: map[string]*storepb.UserTagMetadata{
"existing": {
BlurContent: true,
},
},
})
require.NoError(t, err)
_, err = db.ExecContext(ctx, "INSERT INTO system_setting (name, value) VALUES ('BASIC', ?), ('TAGS', ?)", string(basicSettingBytes), string(tagsSettingBytes))
require.NoError(t, err)
_, err = db.ExecContext(ctx, `
INSERT INTO user (id, username, role, password_hash, avatar_url)
VALUES (1, 'tag-owner', 'USER', 'legacy-hash', ''), (2, 'keeps-existing', 'USER', 'legacy-hash', '')
`)
require.NoError(t, err)
_, err = db.ExecContext(ctx, "INSERT INTO user_setting (user_id, key, value) VALUES (2, 'TAGS', ?)", string(existingUserTagsBytes))
require.NoError(t, err)
require.NoError(t, db.Close())
ts := NewTestingStoreWithDSN(ctx, t, "sqlite", dsn)
require.NoError(t, ts.Migrate(ctx))
defer ts.Close()
copiedUserID := int32(1)
copied, err := ts.GetUserSetting(ctx, &store.FindUserSetting{
UserID: &copiedUserID,
Key: storepb.UserSetting_TAGS,
})
require.NoError(t, err)
require.Contains(t, copied.GetTags().GetTags(), "bug")
bugMetadata := copied.GetTags().GetTags()["bug"]
require.NotNil(t, bugMetadata.GetBackgroundColor())
require.InDelta(t, 0.9, bugMetadata.GetBackgroundColor().GetRed(), 1e-6)
require.InDelta(t, 0.1, bugMetadata.GetBackgroundColor().GetGreen(), 1e-6)
require.InDelta(t, 0.1, bugMetadata.GetBackgroundColor().GetBlue(), 1e-6)
require.True(t, copied.GetTags().GetTags()["private/.*"].GetBlurContent())
existingUserID := int32(2)
existing, err := ts.GetUserSetting(ctx, &store.FindUserSetting{
UserID: &existingUserID,
Key: storepb.UserSetting_TAGS,
})
require.NoError(t, err)
require.Contains(t, existing.GetTags().GetTags(), "existing")
require.NotContains(t, existing.GetTags().GetTags(), "bug")
}
func TestCaseSensitiveUsernameMigration(t *testing.T) {
ctx := context.Background()
driver := getDriverFromEnv()
var dsn string
switch driver {
case "sqlite":
dsn = fmt.Sprintf("%s/memos_username_migration.db", t.TempDir())
case "mysql":
dsn = GetMySQLDSN(t)
case "postgres":
dsn = GetPostgresDSN(t)
default:
t.Fatalf("unsupported driver: %s", driver)
}
db, err := sql.Open(driver, dsn)
require.NoError(t, err)
_, err = db.ExecContext(ctx, legacySchemaFixture(driver))
require.NoError(t, err)
basicSettingBytes, err := protojson.Marshal(&storepb.InstanceBasicSetting{SchemaVersion: "0.30.1"})
require.NoError(t, err)
insertBasicSetting := "INSERT INTO system_setting (name, value, description) VALUES ('BASIC', ?, '')"
if driver == "postgres" {
insertBasicSetting = "INSERT INTO system_setting (name, value, description) VALUES ('BASIC', $1, '')"
}
_, err = db.ExecContext(ctx, insertBasicSetting, string(basicSettingBytes))
require.NoError(t, err)
insertUser := "INSERT INTO user (username, role, password_hash, avatar_url) VALUES ('Alice', 'USER', 'legacy-hash', '')"
if driver == "mysql" {
insertUser = "INSERT INTO `user` (username, role, password_hash, avatar_url) VALUES ('Alice', 'USER', 'legacy-hash', '')"
} else if driver == "postgres" {
insertUser = `INSERT INTO "user" (username, role, password_hash, avatar_url) VALUES ('Alice', 'USER', 'legacy-hash', '')`
}
_, err = db.ExecContext(ctx, insertUser)
require.NoError(t, err)
require.NoError(t, db.Close())
ts := NewTestingStoreWithDSN(ctx, t, driver, dsn)
require.NoError(t, ts.Migrate(ctx))
defer ts.Close()
lower, err := createTestingUserWithRole(ctx, ts, "alice", store.RoleUser)
require.NoError(t, err)
upperUsername := "Alice"
upper, err := ts.GetUser(ctx, &store.FindUser{Username: &upperUsername})
require.NoError(t, err)
require.NotEqual(t, upper.ID, lower.ID)
require.Equal(t, "Alice", upper.Username)
}
func legacySchemaFixture(driver string) string {
switch driver {
case "mysql":
return `
CREATE TABLE system_setting (
name VARCHAR(256) NOT NULL PRIMARY KEY,
value LONGTEXT NOT NULL,
description TEXT NOT NULL
);
CREATE TABLE ` + "`user`" + ` (
id INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
created_ts TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_ts TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
row_status VARCHAR(256) NOT NULL DEFAULT 'NORMAL',
username VARCHAR(256) NOT NULL UNIQUE,
role VARCHAR(256) NOT NULL DEFAULT 'USER',
email VARCHAR(256) NOT NULL DEFAULT '',
nickname VARCHAR(256) NOT NULL DEFAULT '',
password_hash VARCHAR(256) NOT NULL,
avatar_url LONGTEXT NOT NULL,
description VARCHAR(256) NOT NULL DEFAULT ''
);
CREATE TABLE user_setting (
user_id INT NOT NULL,
` + "`key`" + ` VARCHAR(256) NOT NULL,
value LONGTEXT NOT NULL,
UNIQUE(user_id, ` + "`key`" + `)
);
CREATE TABLE memo (id INT NOT NULL AUTO_INCREMENT PRIMARY KEY);
`
case "postgres":
return `
CREATE TABLE system_setting (
name TEXT NOT NULL PRIMARY KEY,
value TEXT NOT NULL,
description TEXT NOT NULL
);
CREATE TABLE "user" (
id SERIAL PRIMARY KEY,
created_ts BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW()),
updated_ts BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW()),
row_status TEXT NOT NULL DEFAULT 'NORMAL',
username TEXT NOT NULL UNIQUE,
role TEXT NOT NULL DEFAULT 'USER',
email TEXT NOT NULL DEFAULT '',
nickname TEXT NOT NULL DEFAULT '',
password_hash TEXT NOT NULL,
avatar_url TEXT NOT NULL,
description TEXT NOT NULL DEFAULT ''
);
CREATE TABLE user_setting (
user_id INTEGER NOT NULL,
key TEXT NOT NULL,
value TEXT NOT NULL,
UNIQUE(user_id, key)
);
CREATE TABLE memo (id SERIAL PRIMARY KEY);
`
case "sqlite":
return `
CREATE TABLE system_setting (
name TEXT NOT NULL,
value TEXT NOT NULL,
description TEXT NOT NULL DEFAULT '',
UNIQUE(name)
);
CREATE TABLE user (
id INTEGER PRIMARY KEY AUTOINCREMENT,
created_ts BIGINT NOT NULL DEFAULT (strftime('%s', 'now')),
updated_ts BIGINT NOT NULL DEFAULT (strftime('%s', 'now')),
row_status TEXT NOT NULL CHECK (row_status IN ('NORMAL', 'ARCHIVED')) DEFAULT 'NORMAL',
username TEXT NOT NULL UNIQUE,
role TEXT NOT NULL DEFAULT 'USER',
email TEXT NOT NULL DEFAULT '',
nickname TEXT NOT NULL DEFAULT '',
password_hash TEXT NOT NULL,
avatar_url TEXT NOT NULL DEFAULT '',
description TEXT NOT NULL DEFAULT ''
);
CREATE TABLE user_setting (
user_id INTEGER NOT NULL,
key TEXT NOT NULL,
value TEXT NOT NULL,
UNIQUE(user_id, key)
);
CREATE TABLE memo (id INTEGER PRIMARY KEY AUTOINCREMENT);
`
default:
return ""
}
}
// TestMigrationFromStableVersion verifies that upgrading from a stable Memos version
// to the current version works correctly. This is the critical upgrade path test.
//
// Test flow:
// 1. Start a stable Memos container to create a database with the old schema
// 2. Stop the container and wait for cleanup
// 3. Use the store directly to run migration with current code
// 4. Verify the migration succeeded and data can be written
//
// Note: This test is skipped when running with -race flag because testcontainers
// has known race conditions in its reaper code that are outside our control.
func TestMigrationFromStableVersion(t *testing.T) {
// Skip for non-SQLite drivers (simplifies the test)
if getDriverFromEnv() != "sqlite" {
t.Skip("skipping upgrade test for non-sqlite driver")
}
skipIfContainerProviderUnavailable(t)
ctx := context.Background()
dataDir := t.TempDir()
// 1. Start stable Memos container to create database with old schema
cfg := MemosContainerConfig{
Driver: "sqlite",
DataDir: dataDir,
Version: StableMemosVersion,
}
t.Logf("Starting Memos %s container to create old-schema database...", cfg.Version)
container, err := StartMemosContainer(ctx, cfg)
require.NoError(t, err, "failed to start stable memos container")
// Wait for the container to fully initialize the database
time.Sleep(10 * time.Second)
// Stop the container gracefully
t.Log("Stopping stable Memos container...")
err = container.Terminate(ctx)
require.NoError(t, err, "failed to stop memos container")
// Wait for file handles to be released
time.Sleep(2 * time.Second)
// 2. Connect to the database directly and run migration with current code
dsn := fmt.Sprintf("%s/memos_prod.db", dataDir)
t.Logf("Connecting to database at %s...", dsn)
ts := NewTestingStoreWithDSN(ctx, t, "sqlite", dsn)
// Get the schema version before migration
oldSetting, err := ts.GetInstanceBasicSetting(ctx)
require.NoError(t, err)
t.Logf("Old schema version: %s", oldSetting.SchemaVersion)
// 3. Run migration with current code
t.Log("Running migration with current code...")
err = ts.Migrate(ctx)
require.NoError(t, err, "migration from stable version should succeed")
// 4. Verify migration succeeded
newVersion, err := ts.GetCurrentSchemaVersion()
require.NoError(t, err)
t.Logf("New schema version: %s", newVersion)
newSetting, err := ts.GetInstanceBasicSetting(ctx)
require.NoError(t, err)
require.Equal(t, newVersion, newSetting.SchemaVersion, "schema version should be updated")
// Verify we can write data to the migrated database
user, err := createTestingHostUser(ctx, ts)
require.NoError(t, err, "should create user after migration")
memo, err := ts.CreateMemo(ctx, &store.Memo{
UID: "post-upgrade-memo",
CreatorID: user.ID,
Content: "Content after upgrade from stable",
Visibility: store.Public,
})
require.NoError(t, err, "should create memo after migration")
require.Equal(t, "Content after upgrade from stable", memo.Content)
t.Logf("Migration successful: %s -> %s", oldSetting.SchemaVersion, newVersion)
}