923 lines
33 KiB
Go
923 lines
33 KiB
Go
package test
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
colorpb "google.golang.org/genproto/googleapis/type/color"
|
|
"google.golang.org/protobuf/encoding/protojson"
|
|
|
|
"github.com/usememos/memos/internal/testutil/minio"
|
|
storepb "github.com/usememos/memos/proto/gen/store"
|
|
"github.com/usememos/memos/store"
|
|
)
|
|
|
|
// TestFreshInstall verifies that LATEST.sql applies correctly on a fresh database.
|
|
// This is essentially what NewTestingStore already does, but we make it explicit.
|
|
func TestFreshInstall(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := context.Background()
|
|
|
|
// NewTestingStore creates a fresh database and runs Migrate()
|
|
// which applies LATEST.sql for uninitialized databases
|
|
ts := NewTestingStore(ctx, t)
|
|
|
|
// Verify migration completed successfully
|
|
currentSchemaVersion, err := ts.GetCurrentSchemaVersion()
|
|
require.NoError(t, err)
|
|
require.NotEmpty(t, currentSchemaVersion, "schema version should be set after fresh install")
|
|
|
|
// Verify we can read instance settings (basic sanity check)
|
|
instanceSetting, err := ts.GetInstanceBasicSetting(ctx)
|
|
require.NoError(t, err)
|
|
require.Equal(t, currentSchemaVersion, instanceSetting.SchemaVersion)
|
|
}
|
|
|
|
// TestMigrationReRun verifies that re-running the migration on an already
|
|
// migrated database does not fail or cause issues. This simulates a
|
|
// scenario where the server is restarted.
|
|
func TestMigrationReRun(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := context.Background()
|
|
// Use the shared testing store which already runs migrations on init
|
|
ts := NewTestingStore(ctx, t)
|
|
|
|
// Get current version
|
|
initialVersion, err := ts.GetCurrentSchemaVersion()
|
|
require.NoError(t, err)
|
|
|
|
// Manually trigger migration again
|
|
err = ts.Migrate(ctx)
|
|
require.NoError(t, err, "re-running migration should not fail")
|
|
|
|
// Verify version hasn't changed (or at least is valid)
|
|
finalVersion, err := ts.GetCurrentSchemaVersion()
|
|
require.NoError(t, err)
|
|
require.Equal(t, initialVersion, finalVersion, "version should match after re-run")
|
|
}
|
|
|
|
// TestMigrationWithData verifies that migration preserves data integrity.
|
|
// Creates data, then re-runs migration and verifies data is still accessible.
|
|
func TestMigrationWithData(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := context.Background()
|
|
ts := NewTestingStore(ctx, t)
|
|
|
|
// Create a user and memo before re-running migration
|
|
user, err := createTestingHostUser(ctx, ts)
|
|
require.NoError(t, err, "should create user")
|
|
|
|
originalMemo, err := ts.CreateMemo(ctx, &store.Memo{
|
|
UID: "migration-data-test",
|
|
CreatorID: user.ID,
|
|
Content: "Data before migration re-run",
|
|
Visibility: store.Public,
|
|
})
|
|
require.NoError(t, err, "should create memo")
|
|
|
|
// Re-run migration
|
|
err = ts.Migrate(ctx)
|
|
require.NoError(t, err, "re-running migration should not fail")
|
|
|
|
// Verify data is still accessible
|
|
memo, err := ts.GetMemo(ctx, &store.FindMemo{UID: &originalMemo.UID})
|
|
require.NoError(t, err, "should retrieve memo after migration")
|
|
require.Equal(t, "Data before migration re-run", memo.Content, "memo content should be preserved")
|
|
}
|
|
|
|
// TestMigrationMultipleReRuns verifies that migration is idempotent
|
|
// even when run multiple times in succession.
|
|
func TestMigrationMultipleReRuns(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := context.Background()
|
|
ts := NewTestingStore(ctx, t)
|
|
|
|
// Get initial version
|
|
initialVersion, err := ts.GetCurrentSchemaVersion()
|
|
require.NoError(t, err)
|
|
|
|
// Run migration multiple times
|
|
for i := 0; i < 3; i++ {
|
|
err = ts.Migrate(ctx)
|
|
require.NoError(t, err, "migration run %d should not fail", i+1)
|
|
}
|
|
|
|
// Verify version is still correct
|
|
finalVersion, err := ts.GetCurrentSchemaVersion()
|
|
require.NoError(t, err)
|
|
require.Equal(t, initialVersion, finalVersion, "version should remain unchanged after multiple re-runs")
|
|
}
|
|
|
|
// TestMigrationStorageSetting verifies that the legacy singleton storage
|
|
// configuration is expanded into a named storage without removing fields used
|
|
// by older clients and rollback versions.
|
|
func TestMigrationStorageSetting(t *testing.T) {
|
|
ctx := context.Background()
|
|
driver := getDriverFromEnv()
|
|
|
|
legacyDatabase, err := protojson.Marshal(&storepb.InstanceStorageSetting{
|
|
StorageType: storepb.InstanceStorageSetting_DATABASE,
|
|
})
|
|
require.NoError(t, err)
|
|
legacyLocal, err := protojson.Marshal(&storepb.InstanceStorageSetting{
|
|
StorageType: storepb.InstanceStorageSetting_LOCAL,
|
|
FilepathTemplate: "assets/{filename}",
|
|
})
|
|
require.NoError(t, err)
|
|
legacyS3, err := protojson.Marshal(&storepb.InstanceStorageSetting{
|
|
StorageType: storepb.InstanceStorageSetting_S3,
|
|
S3Config: &storepb.StorageS3Config{
|
|
AccessKeyId: "access-key",
|
|
AccessKeySecret: "secret",
|
|
Endpoint: "https://s3.example.com",
|
|
Region: "us-east-1",
|
|
Bucket: "memos",
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
alreadyNamed, err := protojson.Marshal(&storepb.InstanceStorageSetting{
|
|
StorageType: storepb.InstanceStorageSetting_S3,
|
|
DefaultStorageId: "custom-s3",
|
|
S3Config: &storepb.StorageS3Config{
|
|
AccessKeyId: "access-key",
|
|
AccessKeySecret: "secret",
|
|
Endpoint: "https://s3.example.com",
|
|
Region: "us-east-1",
|
|
Bucket: "memos",
|
|
},
|
|
Storages: []*storepb.Storage{
|
|
{
|
|
Id: "custom-s3",
|
|
Name: "Primary",
|
|
Type: storepb.StorageType_STORAGE_TYPE_S3,
|
|
Config: &storepb.Storage_S3Config{S3Config: &storepb.StorageS3Config{
|
|
AccessKeyId: "access-key",
|
|
AccessKeySecret: "secret",
|
|
Endpoint: "https://s3.example.com",
|
|
Region: "us-east-1",
|
|
Bucket: "memos",
|
|
}},
|
|
},
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
tests := []struct {
|
|
name string
|
|
value string
|
|
wantStorageID string
|
|
wantStorageType storepb.StorageType
|
|
wantLegacyType storepb.InstanceStorageSetting_StorageType
|
|
wantName string
|
|
wantRawUnchanged bool
|
|
}{
|
|
{
|
|
name: "database",
|
|
value: string(legacyDatabase),
|
|
wantStorageID: "database",
|
|
wantStorageType: storepb.StorageType_STORAGE_TYPE_DATABASE,
|
|
wantLegacyType: storepb.InstanceStorageSetting_DATABASE,
|
|
wantName: "Database",
|
|
},
|
|
{
|
|
name: "local",
|
|
value: string(legacyLocal),
|
|
wantStorageID: "local",
|
|
wantStorageType: storepb.StorageType_STORAGE_TYPE_LOCAL,
|
|
wantLegacyType: storepb.InstanceStorageSetting_LOCAL,
|
|
wantName: "Local",
|
|
},
|
|
{
|
|
name: "s3",
|
|
value: string(legacyS3),
|
|
wantStorageID: "s3",
|
|
wantStorageType: storepb.StorageType_STORAGE_TYPE_S3,
|
|
wantLegacyType: storepb.InstanceStorageSetting_S3,
|
|
wantName: "S3",
|
|
},
|
|
{
|
|
name: "already named",
|
|
value: string(alreadyNamed),
|
|
wantStorageID: "custom-s3",
|
|
wantStorageType: storepb.StorageType_STORAGE_TYPE_S3,
|
|
wantLegacyType: storepb.InstanceStorageSetting_S3,
|
|
wantName: "Primary",
|
|
},
|
|
{
|
|
name: "invalid JSON",
|
|
value: `{oops}`,
|
|
wantRawUnchanged: true,
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
dsn := getTestingProfileForDriver(t, driver).DSN
|
|
db, err := sql.Open(driver, dsn)
|
|
require.NoError(t, err)
|
|
_, err = db.ExecContext(ctx, legacySchemaFixture(driver))
|
|
require.NoError(t, err)
|
|
|
|
basicSettingBytes, err := protojson.Marshal(&storepb.InstanceBasicSetting{SchemaVersion: "0.31.1"})
|
|
require.NoError(t, err)
|
|
insertSetting := "INSERT INTO system_setting (name, value, description) VALUES (?, ?, '')"
|
|
if driver == "postgres" {
|
|
insertSetting = "INSERT INTO system_setting (name, value, description) VALUES ($1, $2, '')"
|
|
}
|
|
_, err = db.ExecContext(ctx, insertSetting, "BASIC", string(basicSettingBytes))
|
|
require.NoError(t, err)
|
|
_, err = db.ExecContext(ctx, insertSetting, "STORAGE", test.value)
|
|
require.NoError(t, err)
|
|
if test.name == "s3" {
|
|
insertStorageMigrationAttachments(ctx, t, db, driver)
|
|
}
|
|
require.NoError(t, db.Close())
|
|
|
|
ts := NewTestingStoreWithDSN(ctx, t, driver, dsn)
|
|
require.NoError(t, ts.Migrate(ctx))
|
|
defer ts.Close()
|
|
if test.wantRawUnchanged {
|
|
query := "SELECT value FROM system_setting WHERE name = ?"
|
|
if driver == "postgres" {
|
|
query = "SELECT value FROM system_setting WHERE name = $1"
|
|
}
|
|
var raw string
|
|
require.NoError(t, ts.GetDriver().GetDB().QueryRowContext(ctx, query, "STORAGE").Scan(&raw))
|
|
require.Equal(t, test.value, raw)
|
|
return
|
|
}
|
|
|
|
stored, err := ts.GetStoredInstanceSetting(ctx, &store.FindInstanceSetting{Name: storepb.InstanceSettingKey_STORAGE.String()})
|
|
require.NoError(t, err)
|
|
require.NotNil(t, stored)
|
|
setting := stored.GetStorageSetting()
|
|
require.Equal(t, test.wantLegacyType, setting.GetStorageType(), "legacy storage type must be preserved")
|
|
require.Equal(t, test.wantStorageID, setting.GetDefaultStorageId())
|
|
require.Len(t, setting.GetStorages(), 1)
|
|
require.Equal(t, test.wantStorageID, setting.GetStorages()[0].GetId())
|
|
require.Equal(t, test.wantName, setting.GetStorages()[0].GetName())
|
|
require.Equal(t, test.wantStorageType, setting.GetStorages()[0].GetType())
|
|
if test.wantStorageType == storepb.StorageType_STORAGE_TYPE_S3 {
|
|
require.Equal(t, "secret", setting.GetS3Config().GetAccessKeySecret(), "legacy S3 config must be preserved")
|
|
require.Equal(t, "secret", setting.GetStorages()[0].GetS3Config().GetAccessKeySecret())
|
|
}
|
|
if test.name == "s3" {
|
|
assertStorageMigrationAttachments(ctx, t, ts)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func insertStorageMigrationAttachments(ctx context.Context, t *testing.T, db *sql.DB, driver string) {
|
|
t.Helper()
|
|
legacyPayload, err := protojson.Marshal(&storepb.AttachmentPayload{
|
|
Payload: &storepb.AttachmentPayload_S3Object_{
|
|
S3Object: &storepb.AttachmentPayload_S3Object{Key: "legacy/no-config.txt"},
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
embeddedPayload, err := protojson.Marshal(&storepb.AttachmentPayload{
|
|
Payload: &storepb.AttachmentPayload_S3Object_{
|
|
S3Object: &storepb.AttachmentPayload_S3Object{
|
|
Key: "legacy/embedded-config.txt",
|
|
S3Config: &storepb.StorageS3Config{
|
|
Endpoint: "https://old-s3.example.com",
|
|
Region: "us-east-1",
|
|
Bucket: "old-bucket",
|
|
},
|
|
},
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
namedPayload, err := protojson.Marshal(&storepb.AttachmentPayload{
|
|
Payload: &storepb.AttachmentPayload_S3Object_{
|
|
S3Object: &storepb.AttachmentPayload_S3Object{Key: "named/object.txt", StorageId: "existing-s3"},
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
insertAttachment := "INSERT INTO attachment (uid, creator_id, filename, type, size, storage_type, reference, payload) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
|
|
if driver == "postgres" {
|
|
insertAttachment = "INSERT INTO attachment (uid, creator_id, filename, type, size, storage_type, reference, payload) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)"
|
|
}
|
|
for _, attachment := range []struct {
|
|
uid string
|
|
payload []byte
|
|
}{
|
|
{uid: "legacy-s3-no-config", payload: legacyPayload},
|
|
{uid: "legacy-s3-embedded-config", payload: embeddedPayload},
|
|
{uid: "legacy-s3-existing-id", payload: namedPayload},
|
|
} {
|
|
_, err := db.ExecContext(ctx, insertAttachment, attachment.uid, 1, attachment.uid+".txt", "text/plain", 1, "S3", "", string(attachment.payload))
|
|
require.NoError(t, err)
|
|
}
|
|
}
|
|
|
|
func assertStorageMigrationAttachments(ctx context.Context, t *testing.T, ts *store.Store) {
|
|
t.Helper()
|
|
find := func(uid string) *storepb.AttachmentPayload_S3Object {
|
|
query := "SELECT payload FROM attachment WHERE uid = ?"
|
|
if getDriverFromEnv() == "postgres" {
|
|
query = "SELECT payload FROM attachment WHERE uid = $1"
|
|
}
|
|
var rawPayload []byte
|
|
err := ts.GetDriver().GetDB().QueryRowContext(ctx, query, uid).Scan(&rawPayload)
|
|
require.NoError(t, err)
|
|
payload := &storepb.AttachmentPayload{}
|
|
require.NoError(t, protojson.Unmarshal(rawPayload, payload))
|
|
require.NotNil(t, payload.GetS3Object())
|
|
return payload.GetS3Object()
|
|
}
|
|
|
|
legacy := find("legacy-s3-no-config")
|
|
require.Equal(t, "s3", legacy.GetStorageId())
|
|
require.Nil(t, legacy.GetS3Config())
|
|
|
|
embedded := find("legacy-s3-embedded-config")
|
|
require.Empty(t, embedded.GetStorageId(), "embedded namespace must not be rebound to the current storage")
|
|
require.Equal(t, "old-bucket", embedded.GetS3Config().GetBucket())
|
|
|
|
named := find("legacy-s3-existing-id")
|
|
require.Equal(t, "existing-s3", named.GetStorageId())
|
|
}
|
|
|
|
// TestMigrationLegacyS3AttachmentMinIO verifies the storage upgrade path for an
|
|
// attachment created before storage IDs existed. The migration must bind the
|
|
// payload to the migrated registry entry, after which the production resolver
|
|
// and driver must read and delete it using the real S3 protocol.
|
|
func TestMigrationLegacyS3AttachmentMinIO(t *testing.T) {
|
|
ctx := context.Background()
|
|
driver := getDriverFromEnv()
|
|
server := minio.New(t, "legacy-attachments")
|
|
config := server.Config("legacy-attachments")
|
|
content := []byte("attachment uploaded before named storage migration")
|
|
const objectKey = "legacy/no-storage-id.txt"
|
|
require.NoError(t, server.PutObject(config.Bucket, objectKey, "text/plain", content))
|
|
|
|
storageSetting, err := protojson.Marshal(&storepb.InstanceStorageSetting{
|
|
StorageType: storepb.InstanceStorageSetting_S3,
|
|
S3Config: config,
|
|
})
|
|
require.NoError(t, err)
|
|
attachmentPayload, err := protojson.Marshal(&storepb.AttachmentPayload{
|
|
Payload: &storepb.AttachmentPayload_S3Object_{
|
|
S3Object: &storepb.AttachmentPayload_S3Object{Key: objectKey},
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
dsn := getTestingProfileForDriver(t, driver).DSN
|
|
db, err := sql.Open(driver, dsn)
|
|
require.NoError(t, err)
|
|
_, err = db.ExecContext(ctx, legacySchemaFixture(driver))
|
|
require.NoError(t, err)
|
|
|
|
basicSetting, err := protojson.Marshal(&storepb.InstanceBasicSetting{SchemaVersion: "0.31.1"})
|
|
require.NoError(t, err)
|
|
insertSetting := "INSERT INTO system_setting (name, value, description) VALUES (?, ?, '')"
|
|
insertAttachment := "INSERT INTO attachment (uid, creator_id, filename, type, size, storage_type, reference, payload) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
|
|
if driver == "postgres" {
|
|
insertSetting = "INSERT INTO system_setting (name, value, description) VALUES ($1, $2, '')"
|
|
insertAttachment = "INSERT INTO attachment (uid, creator_id, filename, type, size, storage_type, reference, payload) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)"
|
|
}
|
|
_, err = db.ExecContext(ctx, insertSetting, "BASIC", string(basicSetting))
|
|
require.NoError(t, err)
|
|
_, err = db.ExecContext(ctx, insertSetting, "STORAGE", string(storageSetting))
|
|
require.NoError(t, err)
|
|
_, err = db.ExecContext(
|
|
ctx,
|
|
insertAttachment,
|
|
"legacy-s3-minio",
|
|
1,
|
|
"legacy.txt",
|
|
"text/plain",
|
|
len(content),
|
|
"S3",
|
|
"",
|
|
string(attachmentPayload),
|
|
)
|
|
require.NoError(t, err)
|
|
require.NoError(t, db.Close())
|
|
|
|
ts := NewTestingStoreWithDSN(ctx, t, driver, dsn)
|
|
require.NoError(t, ts.Migrate(ctx))
|
|
defer ts.Close()
|
|
|
|
query := "SELECT payload FROM attachment WHERE uid = ?"
|
|
if driver == "postgres" {
|
|
query = "SELECT payload FROM attachment WHERE uid = $1"
|
|
}
|
|
var rawPayload []byte
|
|
err = ts.GetDriver().GetDB().QueryRowContext(ctx, query, "legacy-s3-minio").Scan(&rawPayload)
|
|
require.NoError(t, err)
|
|
migratedPayload := &storepb.AttachmentPayload{}
|
|
require.NoError(t, protojson.Unmarshal(rawPayload, migratedPayload))
|
|
s3Object := migratedPayload.GetS3Object()
|
|
require.NotNil(t, s3Object)
|
|
require.Equal(t, "s3", s3Object.GetStorageId())
|
|
require.Nil(t, s3Object.GetS3Config())
|
|
|
|
storedSetting, err := ts.GetStoredInstanceSetting(ctx, &store.FindInstanceSetting{Name: storepb.InstanceSettingKey_STORAGE.String()})
|
|
require.NoError(t, err)
|
|
require.NotNil(t, storedSetting)
|
|
storageDriver, err := ts.ResolveStorageDriver(ctx, storedSetting.GetStorageSetting(), s3Object.StorageId, s3Object.S3Config)
|
|
require.NoError(t, err)
|
|
downloaded, err := storageDriver.GetObject(ctx, s3Object.Key)
|
|
require.NoError(t, err)
|
|
require.Equal(t, content, downloaded)
|
|
|
|
require.NoError(t, storageDriver.DeleteObject(ctx, s3Object.Key))
|
|
_, err = server.GetObject(config.Bucket, objectKey)
|
|
require.Error(t, err, "deleting the migrated attachment must remove its MinIO object")
|
|
}
|
|
|
|
// TestMigrationMemoViewSetting verifies the 0.31 rename of the SHORTCUTS user setting
|
|
// to MEMO_VIEWS on every driver, since each one rewrites the stored JSON with its own
|
|
// dialect-specific functions.
|
|
func TestMigrationMemoViewSetting(t *testing.T) {
|
|
ctx := context.Background()
|
|
driver := getDriverFromEnv()
|
|
var dsn string
|
|
switch driver {
|
|
case "sqlite":
|
|
dsn = fmt.Sprintf("%s/memos_memo_view_migration.db", t.TempDir())
|
|
case "mysql":
|
|
dsn = GetMySQLDSN(t)
|
|
case "postgres":
|
|
dsn = GetPostgresDSN(t)
|
|
default:
|
|
t.Fatalf("unsupported driver: %s", driver)
|
|
}
|
|
|
|
db, err := sql.Open(driver, dsn)
|
|
require.NoError(t, err)
|
|
|
|
_, err = db.ExecContext(ctx, legacySchemaFixture(driver))
|
|
require.NoError(t, err)
|
|
|
|
basicSettingBytes, err := protojson.Marshal(&storepb.InstanceBasicSetting{SchemaVersion: "0.30.2"})
|
|
require.NoError(t, err)
|
|
insertBasicSetting := "INSERT INTO system_setting (name, value, description) VALUES ('BASIC', ?, '')"
|
|
if driver == "postgres" {
|
|
insertBasicSetting = "INSERT INTO system_setting (name, value, description) VALUES ('BASIC', $1, '')"
|
|
}
|
|
_, err = db.ExecContext(ctx, insertBasicSetting, string(basicSettingBytes))
|
|
require.NoError(t, err)
|
|
|
|
settingKeyColumn := "key"
|
|
if driver == "mysql" {
|
|
settingKeyColumn = "`key`"
|
|
}
|
|
// Bound rather than inlined: MySQL treats backslash as an escape character inside
|
|
// string literals, so an inlined \" would collapse and corrupt the fixture's JSON.
|
|
insertSetting := fmt.Sprintf("INSERT INTO user_setting (user_id, %[1]s, value) VALUES (?, ?, ?)", settingKeyColumn)
|
|
if driver == "postgres" {
|
|
insertSetting = fmt.Sprintf("INSERT INTO user_setting (user_id, %[1]s, value) VALUES ($1, $2, $3)", settingKeyColumn)
|
|
}
|
|
settingRows := []struct {
|
|
userID int
|
|
key string
|
|
value string
|
|
}{
|
|
// A normal legacy row, must be renamed and rewritten.
|
|
{1, "SHORTCUTS", `{"shortcuts":[{"id":"work","title":"shortcuts","filter":"tag in [\"work\"]"}]}`},
|
|
// A corrupt value, must be left alone rather than aborting the upgrade.
|
|
{2, "SHORTCUTS", `{oops}`},
|
|
// Already has a MEMO_VIEWS row, must not trip UNIQUE(user_id, key).
|
|
{3, "SHORTCUTS", `{"shortcuts":[{"id":"old","title":"old","filter":"tag in [\"old\"]"}]}`},
|
|
{3, "MEMO_VIEWS", `{"memoViews":[{"id":"new","title":"new","filter":"tag in [\"new\"]"}]}`},
|
|
}
|
|
for _, row := range settingRows {
|
|
_, err = db.ExecContext(ctx, insertSetting, row.userID, row.key, row.value)
|
|
require.NoError(t, err)
|
|
}
|
|
require.NoError(t, db.Close())
|
|
|
|
ts := NewTestingStoreWithDSN(ctx, t, driver, dsn)
|
|
require.NoError(t, ts.Migrate(ctx))
|
|
defer ts.Close()
|
|
|
|
migratedUserID := int32(1)
|
|
setting, err := ts.GetUserSetting(ctx, &store.FindUserSetting{
|
|
UserID: &migratedUserID,
|
|
Key: storepb.UserSetting_MEMO_VIEWS,
|
|
})
|
|
require.NoError(t, err)
|
|
require.NotNil(t, setting)
|
|
require.Len(t, setting.GetMemoViews().GetMemoViews(), 1)
|
|
require.Equal(t, "work", setting.GetMemoViews().GetMemoViews()[0].GetId())
|
|
require.Equal(t, "shortcuts", setting.GetMemoViews().GetMemoViews()[0].GetTitle())
|
|
require.Equal(t, `tag in ["work"]`, setting.GetMemoViews().GetMemoViews()[0].GetFilter())
|
|
|
|
// Malformed object-like JSON must neither abort the migration nor be renamed.
|
|
findCorruptSetting := fmt.Sprintf("SELECT value FROM user_setting WHERE user_id = ? AND %s = ?", settingKeyColumn)
|
|
if driver == "postgres" {
|
|
findCorruptSetting = fmt.Sprintf("SELECT value FROM user_setting WHERE user_id = $1 AND %s = $2", settingKeyColumn)
|
|
}
|
|
var corruptValue string
|
|
err = ts.GetDriver().GetDB().QueryRowContext(ctx, findCorruptSetting, 2, "SHORTCUTS").Scan(&corruptValue)
|
|
require.NoError(t, err)
|
|
require.Equal(t, `{oops}`, corruptValue)
|
|
|
|
// The pre-existing MEMO_VIEWS row wins; the legacy row is left behind untouched.
|
|
conflictUserID := int32(3)
|
|
conflicting, err := ts.GetUserSetting(ctx, &store.FindUserSetting{
|
|
UserID: &conflictUserID,
|
|
Key: storepb.UserSetting_MEMO_VIEWS,
|
|
})
|
|
require.NoError(t, err)
|
|
require.NotNil(t, conflicting)
|
|
require.Len(t, conflicting.GetMemoViews().GetMemoViews(), 1)
|
|
require.Equal(t, "new", conflicting.GetMemoViews().GetMemoViews()[0].GetId())
|
|
}
|
|
|
|
// TestMigrationCopiesInstanceTagsToUserSettings verifies instance tag metadata is copied into user settings.
|
|
func TestMigrationCopiesInstanceTagsToUserSettings(t *testing.T) {
|
|
if getDriverFromEnv() != "sqlite" {
|
|
t.Skip("skipping focused migration fixture for non-sqlite driver")
|
|
}
|
|
|
|
ctx := context.Background()
|
|
dsn := fmt.Sprintf("%s/memos_tag_migration.db", t.TempDir())
|
|
|
|
db, err := sql.Open("sqlite", dsn)
|
|
require.NoError(t, err)
|
|
|
|
_, err = db.ExecContext(ctx, `
|
|
CREATE TABLE system_setting (
|
|
name TEXT NOT NULL,
|
|
value TEXT NOT NULL,
|
|
description TEXT NOT NULL DEFAULT '',
|
|
UNIQUE(name)
|
|
);
|
|
CREATE TABLE user (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
created_ts BIGINT NOT NULL DEFAULT (strftime('%s', 'now')),
|
|
updated_ts BIGINT NOT NULL DEFAULT (strftime('%s', 'now')),
|
|
row_status TEXT NOT NULL CHECK (row_status IN ('NORMAL', 'ARCHIVED')) DEFAULT 'NORMAL',
|
|
username TEXT NOT NULL UNIQUE,
|
|
role TEXT NOT NULL DEFAULT 'USER',
|
|
email TEXT NOT NULL DEFAULT '',
|
|
nickname TEXT NOT NULL DEFAULT '',
|
|
password_hash TEXT NOT NULL,
|
|
avatar_url TEXT NOT NULL DEFAULT '',
|
|
description TEXT NOT NULL DEFAULT ''
|
|
);
|
|
CREATE TABLE user_setting (
|
|
user_id INTEGER NOT NULL,
|
|
key TEXT NOT NULL,
|
|
value TEXT NOT NULL,
|
|
UNIQUE(user_id, key)
|
|
);
|
|
CREATE TABLE memo (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT
|
|
);
|
|
CREATE TABLE attachment (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
storage_type TEXT NOT NULL DEFAULT '',
|
|
payload TEXT NOT NULL DEFAULT '{}'
|
|
);
|
|
`)
|
|
require.NoError(t, err)
|
|
|
|
basicSettingBytes, err := protojson.Marshal(&storepb.InstanceBasicSetting{SchemaVersion: "0.29.1"})
|
|
require.NoError(t, err)
|
|
tagsSettingBytes, err := protojson.Marshal(&storepb.InstanceTagsSetting{
|
|
Tags: map[string]*storepb.InstanceTagMetadata{
|
|
"bug": {
|
|
BackgroundColor: &colorpb.Color{Red: 0.9, Green: 0.1, Blue: 0.1},
|
|
},
|
|
"private/.*": {
|
|
BlurContent: true,
|
|
},
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
existingUserTagsBytes, err := protojson.Marshal(&storepb.TagsUserSetting{
|
|
Tags: map[string]*storepb.UserTagMetadata{
|
|
"existing": {
|
|
BlurContent: true,
|
|
},
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
_, err = db.ExecContext(ctx, "INSERT INTO system_setting (name, value) VALUES ('BASIC', ?), ('TAGS', ?)", string(basicSettingBytes), string(tagsSettingBytes))
|
|
require.NoError(t, err)
|
|
_, err = db.ExecContext(ctx, `
|
|
INSERT INTO user (id, username, role, password_hash, avatar_url)
|
|
VALUES (1, 'tag-owner', 'USER', 'legacy-hash', ''), (2, 'keeps-existing', 'USER', 'legacy-hash', '')
|
|
`)
|
|
require.NoError(t, err)
|
|
_, err = db.ExecContext(ctx, "INSERT INTO user_setting (user_id, key, value) VALUES (2, 'TAGS', ?)", string(existingUserTagsBytes))
|
|
require.NoError(t, err)
|
|
require.NoError(t, db.Close())
|
|
|
|
ts := NewTestingStoreWithDSN(ctx, t, "sqlite", dsn)
|
|
require.NoError(t, ts.Migrate(ctx))
|
|
defer ts.Close()
|
|
|
|
copiedUserID := int32(1)
|
|
copied, err := ts.GetUserSetting(ctx, &store.FindUserSetting{
|
|
UserID: &copiedUserID,
|
|
Key: storepb.UserSetting_TAGS,
|
|
})
|
|
require.NoError(t, err)
|
|
require.Contains(t, copied.GetTags().GetTags(), "bug")
|
|
bugMetadata := copied.GetTags().GetTags()["bug"]
|
|
require.NotNil(t, bugMetadata.GetBackgroundColor())
|
|
require.InDelta(t, 0.9, bugMetadata.GetBackgroundColor().GetRed(), 1e-6)
|
|
require.InDelta(t, 0.1, bugMetadata.GetBackgroundColor().GetGreen(), 1e-6)
|
|
require.InDelta(t, 0.1, bugMetadata.GetBackgroundColor().GetBlue(), 1e-6)
|
|
require.True(t, copied.GetTags().GetTags()["private/.*"].GetBlurContent())
|
|
|
|
existingUserID := int32(2)
|
|
existing, err := ts.GetUserSetting(ctx, &store.FindUserSetting{
|
|
UserID: &existingUserID,
|
|
Key: storepb.UserSetting_TAGS,
|
|
})
|
|
require.NoError(t, err)
|
|
require.Contains(t, existing.GetTags().GetTags(), "existing")
|
|
require.NotContains(t, existing.GetTags().GetTags(), "bug")
|
|
}
|
|
|
|
func TestCaseSensitiveUsernameMigration(t *testing.T) {
|
|
ctx := context.Background()
|
|
driver := getDriverFromEnv()
|
|
var dsn string
|
|
switch driver {
|
|
case "sqlite":
|
|
dsn = fmt.Sprintf("%s/memos_username_migration.db", t.TempDir())
|
|
case "mysql":
|
|
dsn = GetMySQLDSN(t)
|
|
case "postgres":
|
|
dsn = GetPostgresDSN(t)
|
|
default:
|
|
t.Fatalf("unsupported driver: %s", driver)
|
|
}
|
|
|
|
db, err := sql.Open(driver, dsn)
|
|
require.NoError(t, err)
|
|
|
|
_, err = db.ExecContext(ctx, legacySchemaFixture(driver))
|
|
require.NoError(t, err)
|
|
|
|
basicSettingBytes, err := protojson.Marshal(&storepb.InstanceBasicSetting{SchemaVersion: "0.30.1"})
|
|
require.NoError(t, err)
|
|
insertBasicSetting := "INSERT INTO system_setting (name, value, description) VALUES ('BASIC', ?, '')"
|
|
if driver == "postgres" {
|
|
insertBasicSetting = "INSERT INTO system_setting (name, value, description) VALUES ('BASIC', $1, '')"
|
|
}
|
|
_, err = db.ExecContext(ctx, insertBasicSetting, string(basicSettingBytes))
|
|
require.NoError(t, err)
|
|
insertUser := "INSERT INTO user (username, role, password_hash, avatar_url) VALUES ('Alice', 'USER', 'legacy-hash', '')"
|
|
if driver == "mysql" {
|
|
insertUser = "INSERT INTO `user` (username, role, password_hash, avatar_url) VALUES ('Alice', 'USER', 'legacy-hash', '')"
|
|
} else if driver == "postgres" {
|
|
insertUser = `INSERT INTO "user" (username, role, password_hash, avatar_url) VALUES ('Alice', 'USER', 'legacy-hash', '')`
|
|
}
|
|
_, err = db.ExecContext(ctx, insertUser)
|
|
require.NoError(t, err)
|
|
require.NoError(t, db.Close())
|
|
|
|
ts := NewTestingStoreWithDSN(ctx, t, driver, dsn)
|
|
require.NoError(t, ts.Migrate(ctx))
|
|
defer ts.Close()
|
|
|
|
lower, err := createTestingUserWithRole(ctx, ts, "alice", store.RoleUser)
|
|
require.NoError(t, err)
|
|
|
|
upperUsername := "Alice"
|
|
upper, err := ts.GetUser(ctx, &store.FindUser{Username: &upperUsername})
|
|
require.NoError(t, err)
|
|
require.NotEqual(t, upper.ID, lower.ID)
|
|
require.Equal(t, "Alice", upper.Username)
|
|
}
|
|
|
|
func legacySchemaFixture(driver string) string {
|
|
switch driver {
|
|
case "mysql":
|
|
return `
|
|
CREATE TABLE system_setting (
|
|
name VARCHAR(256) NOT NULL PRIMARY KEY,
|
|
value LONGTEXT NOT NULL,
|
|
description TEXT NOT NULL
|
|
);
|
|
CREATE TABLE ` + "`user`" + ` (
|
|
id INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
|
|
created_ts TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
updated_ts TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
row_status VARCHAR(256) NOT NULL DEFAULT 'NORMAL',
|
|
username VARCHAR(256) NOT NULL UNIQUE,
|
|
role VARCHAR(256) NOT NULL DEFAULT 'USER',
|
|
email VARCHAR(256) NOT NULL DEFAULT '',
|
|
nickname VARCHAR(256) NOT NULL DEFAULT '',
|
|
password_hash VARCHAR(256) NOT NULL,
|
|
avatar_url LONGTEXT NOT NULL,
|
|
description VARCHAR(256) NOT NULL DEFAULT ''
|
|
);
|
|
CREATE TABLE user_setting (
|
|
user_id INT NOT NULL,
|
|
` + "`key`" + ` VARCHAR(256) NOT NULL,
|
|
value LONGTEXT NOT NULL,
|
|
UNIQUE(user_id, ` + "`key`" + `)
|
|
);
|
|
CREATE TABLE memo (id INT NOT NULL AUTO_INCREMENT PRIMARY KEY);
|
|
CREATE TABLE attachment (
|
|
id INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
|
|
uid VARCHAR(256) NOT NULL UNIQUE,
|
|
creator_id INT NOT NULL,
|
|
created_ts TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
updated_ts TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
filename TEXT NOT NULL,
|
|
type VARCHAR(256) NOT NULL DEFAULT '',
|
|
size INT NOT NULL DEFAULT 0,
|
|
memo_id INT DEFAULT NULL,
|
|
storage_type VARCHAR(256) NOT NULL DEFAULT '',
|
|
reference TEXT NOT NULL,
|
|
payload TEXT NOT NULL
|
|
);
|
|
`
|
|
case "postgres":
|
|
return `
|
|
CREATE TABLE system_setting (
|
|
name TEXT NOT NULL PRIMARY KEY,
|
|
value TEXT NOT NULL,
|
|
description TEXT NOT NULL
|
|
);
|
|
CREATE TABLE "user" (
|
|
id SERIAL PRIMARY KEY,
|
|
created_ts BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW()),
|
|
updated_ts BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW()),
|
|
row_status TEXT NOT NULL DEFAULT 'NORMAL',
|
|
username TEXT NOT NULL UNIQUE,
|
|
role TEXT NOT NULL DEFAULT 'USER',
|
|
email TEXT NOT NULL DEFAULT '',
|
|
nickname TEXT NOT NULL DEFAULT '',
|
|
password_hash TEXT NOT NULL,
|
|
avatar_url TEXT NOT NULL,
|
|
description TEXT NOT NULL DEFAULT ''
|
|
);
|
|
CREATE TABLE user_setting (
|
|
user_id INTEGER NOT NULL,
|
|
key TEXT NOT NULL,
|
|
value TEXT NOT NULL,
|
|
UNIQUE(user_id, key)
|
|
);
|
|
CREATE TABLE memo (id SERIAL PRIMARY KEY);
|
|
CREATE TABLE attachment (
|
|
id SERIAL PRIMARY KEY,
|
|
uid TEXT NOT NULL UNIQUE,
|
|
creator_id INTEGER NOT NULL,
|
|
created_ts BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW()),
|
|
updated_ts BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW()),
|
|
filename TEXT NOT NULL,
|
|
blob BYTEA,
|
|
type TEXT NOT NULL DEFAULT '',
|
|
size INTEGER NOT NULL DEFAULT 0,
|
|
memo_id INTEGER DEFAULT NULL,
|
|
storage_type TEXT NOT NULL DEFAULT '',
|
|
reference TEXT NOT NULL DEFAULT '',
|
|
payload TEXT NOT NULL DEFAULT '{}'
|
|
);
|
|
`
|
|
case "sqlite":
|
|
return `
|
|
CREATE TABLE system_setting (
|
|
name TEXT NOT NULL,
|
|
value TEXT NOT NULL,
|
|
description TEXT NOT NULL DEFAULT '',
|
|
UNIQUE(name)
|
|
);
|
|
CREATE TABLE user (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
created_ts BIGINT NOT NULL DEFAULT (strftime('%s', 'now')),
|
|
updated_ts BIGINT NOT NULL DEFAULT (strftime('%s', 'now')),
|
|
row_status TEXT NOT NULL CHECK (row_status IN ('NORMAL', 'ARCHIVED')) DEFAULT 'NORMAL',
|
|
username TEXT NOT NULL UNIQUE,
|
|
role TEXT NOT NULL DEFAULT 'USER',
|
|
email TEXT NOT NULL DEFAULT '',
|
|
nickname TEXT NOT NULL DEFAULT '',
|
|
password_hash TEXT NOT NULL,
|
|
avatar_url TEXT NOT NULL DEFAULT '',
|
|
description TEXT NOT NULL DEFAULT ''
|
|
);
|
|
CREATE TABLE user_setting (
|
|
user_id INTEGER NOT NULL,
|
|
key TEXT NOT NULL,
|
|
value TEXT NOT NULL,
|
|
UNIQUE(user_id, key)
|
|
);
|
|
CREATE TABLE memo (id INTEGER PRIMARY KEY AUTOINCREMENT);
|
|
CREATE TABLE attachment (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
uid TEXT NOT NULL UNIQUE,
|
|
creator_id INTEGER NOT NULL,
|
|
created_ts BIGINT NOT NULL DEFAULT (strftime('%s', 'now')),
|
|
updated_ts BIGINT NOT NULL DEFAULT (strftime('%s', 'now')),
|
|
filename TEXT NOT NULL DEFAULT '',
|
|
blob BLOB DEFAULT NULL,
|
|
type TEXT NOT NULL DEFAULT '',
|
|
size INTEGER NOT NULL DEFAULT 0,
|
|
memo_id INTEGER,
|
|
storage_type TEXT NOT NULL DEFAULT '',
|
|
reference TEXT NOT NULL DEFAULT '',
|
|
payload TEXT NOT NULL DEFAULT '{}'
|
|
);
|
|
`
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
// TestMigrationFromStableVersion verifies that upgrading from a stable Memos version
|
|
// to the current version works correctly. This is the critical upgrade path test.
|
|
//
|
|
// Test flow:
|
|
// 1. Start a stable Memos container to create a database with the old schema
|
|
// 2. Stop the container and wait for cleanup
|
|
// 3. Use the store directly to run migration with current code
|
|
// 4. Verify the migration succeeded and data can be written
|
|
//
|
|
// Note: This test is skipped when running with -race flag because testcontainers
|
|
// has known race conditions in its reaper code that are outside our control.
|
|
func TestMigrationFromStableVersion(t *testing.T) {
|
|
// Skip for non-SQLite drivers (simplifies the test)
|
|
if getDriverFromEnv() != "sqlite" {
|
|
t.Skip("skipping upgrade test for non-sqlite driver")
|
|
}
|
|
|
|
skipIfContainerProviderUnavailable(t)
|
|
|
|
ctx := context.Background()
|
|
dataDir := t.TempDir()
|
|
|
|
// 1. Start stable Memos container to create database with old schema
|
|
cfg := MemosContainerConfig{
|
|
Driver: "sqlite",
|
|
DataDir: dataDir,
|
|
Version: StableMemosVersion,
|
|
}
|
|
|
|
t.Logf("Starting Memos %s container to create old-schema database...", cfg.Version)
|
|
container, err := StartMemosContainer(ctx, cfg)
|
|
require.NoError(t, err, "failed to start stable memos container")
|
|
|
|
// Wait for the container to fully initialize the database
|
|
time.Sleep(10 * time.Second)
|
|
|
|
// Stop the container gracefully
|
|
t.Log("Stopping stable Memos container...")
|
|
err = container.Terminate(ctx)
|
|
require.NoError(t, err, "failed to stop memos container")
|
|
|
|
// Wait for file handles to be released
|
|
time.Sleep(2 * time.Second)
|
|
|
|
// 2. Connect to the database directly and run migration with current code
|
|
dsn := fmt.Sprintf("%s/memos_prod.db", dataDir)
|
|
t.Logf("Connecting to database at %s...", dsn)
|
|
|
|
ts := NewTestingStoreWithDSN(ctx, t, "sqlite", dsn)
|
|
|
|
// Get the schema version before migration
|
|
oldSetting, err := ts.GetInstanceBasicSetting(ctx)
|
|
require.NoError(t, err)
|
|
t.Logf("Old schema version: %s", oldSetting.SchemaVersion)
|
|
|
|
// 3. Run migration with current code
|
|
t.Log("Running migration with current code...")
|
|
err = ts.Migrate(ctx)
|
|
require.NoError(t, err, "migration from stable version should succeed")
|
|
|
|
// 4. Verify migration succeeded
|
|
newVersion, err := ts.GetCurrentSchemaVersion()
|
|
require.NoError(t, err)
|
|
t.Logf("New schema version: %s", newVersion)
|
|
|
|
newSetting, err := ts.GetInstanceBasicSetting(ctx)
|
|
require.NoError(t, err)
|
|
require.Equal(t, newVersion, newSetting.SchemaVersion, "schema version should be updated")
|
|
|
|
// Verify we can write data to the migrated database
|
|
user, err := createTestingHostUser(ctx, ts)
|
|
require.NoError(t, err, "should create user after migration")
|
|
|
|
memo, err := ts.CreateMemo(ctx, &store.Memo{
|
|
UID: "post-upgrade-memo",
|
|
CreatorID: user.ID,
|
|
Content: "Content after upgrade from stable",
|
|
Visibility: store.Public,
|
|
})
|
|
require.NoError(t, err, "should create memo after migration")
|
|
require.Equal(t, "Content after upgrade from stable", memo.Content)
|
|
|
|
t.Logf("Migration successful: %s -> %s", oldSetting.SchemaVersion, newVersion)
|
|
}
|