fix(store): harden 0.31 migration upgrades (#6247)

This commit is contained in:
boojack 2026-08-30 09:24:56 +08:00 committed by GitHub
parent d5849bdf52
commit a7208701d1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 178 additions and 107 deletions

View file

@ -18,7 +18,10 @@ SET
ELSE value
END
WHERE `key` = 'SHORTCUTS'
AND JSON_VALID(value)
AND CASE
WHEN JSON_VALID(value) THEN JSON_TYPE(value) = 'OBJECT'
ELSE FALSE
END
AND user_id NOT IN (
SELECT user_id FROM (
SELECT user_id FROM user_setting WHERE `key` = 'MEMO_VIEWS'

View file

@ -1,24 +1,18 @@
-- Replaces the memo resource name stored in reaction.content_id with the
-- memo's stable internal ID. The inner join intentionally drops orphaned
-- reactions whose resource name no longer resolves to an existing memo.
CREATE TABLE `reaction_new` (
`id` INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
`created_ts` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
`creator_id` INT NOT NULL,
`memo_id` INT NOT NULL,
`reaction_type` VARCHAR(256) NOT NULL,
UNIQUE(`creator_id`, `memo_id`, `reaction_type`)
);
-- memo's stable internal ID. Rows that cannot be resolved are intentionally
-- deleted before memo_id is made non-nullable. Altering the table in place
-- preserves its AUTO_INCREMENT high-water mark.
ALTER TABLE `reaction`
ADD COLUMN `memo_id` INT DEFAULT NULL AFTER `creator_id`;
INSERT INTO `reaction_new` (`id`, `created_ts`, `creator_id`, `memo_id`, `reaction_type`)
SELECT
`reaction`.`id`,
`reaction`.`created_ts`,
`reaction`.`creator_id`,
`memo`.`id`,
`reaction`.`reaction_type`
FROM `reaction`
JOIN `memo` ON `reaction`.`content_id` = CONCAT('memos/', `memo`.`uid`);
UPDATE `reaction`
JOIN `memo` ON `reaction`.`content_id` = CONCAT('memos/', `memo`.`uid`)
SET `reaction`.`memo_id` = `memo`.`id`;
DROP TABLE `reaction`;
RENAME TABLE `reaction_new` TO `reaction`;
DELETE FROM `reaction` WHERE `memo_id` IS NULL;
ALTER TABLE `reaction`
DROP INDEX `creator_id`,
DROP COLUMN `content_id`,
MODIFY COLUMN `memo_id` INT NOT NULL,
ADD UNIQUE (`creator_id`, `memo_id`, `reaction_type`);

View file

@ -1,24 +0,0 @@
ALTER TABLE user RENAME TO user_old;
CREATE TABLE user (
id INTEGER PRIMARY KEY AUTOINCREMENT,
created_ts BIGINT NOT NULL DEFAULT (strftime('%s', 'now')),
updated_ts BIGINT NOT NULL DEFAULT (strftime('%s', 'now')),
row_status TEXT NOT NULL CHECK (row_status IN ('NORMAL', 'ARCHIVED')) DEFAULT 'NORMAL',
username TEXT COLLATE BINARY NOT NULL UNIQUE,
role TEXT NOT NULL DEFAULT 'USER',
email TEXT NOT NULL DEFAULT '',
nickname TEXT NOT NULL DEFAULT '',
password_hash TEXT NOT NULL,
avatar_url TEXT NOT NULL DEFAULT '',
description TEXT NOT NULL DEFAULT ''
);
INSERT INTO user (
id, created_ts, updated_ts, row_status, username, role, email, nickname, password_hash, avatar_url, description
)
SELECT
id, created_ts, updated_ts, row_status, username, role, email, nickname, password_hash, avatar_url, description
FROM user_old;
DROP TABLE user_old;

View file

@ -16,7 +16,10 @@ SET
ELSE value
END
WHERE key = 'SHORTCUTS'
AND json_valid(value)
AND CASE
WHEN json_valid(value) THEN json_type(value) = 'object'
ELSE 0
END
AND user_id NOT IN (
SELECT user_id FROM user_setting WHERE key = 'MEMO_VIEWS'
);

View file

@ -20,5 +20,16 @@ SELECT
FROM reaction
JOIN memo ON reaction.content_id = 'memos/' || memo.uid;
-- Keep IDs issued for deleted or orphaned reactions from being reused.
INSERT INTO sqlite_sequence (name, seq)
SELECT 'reaction_new', seq
FROM sqlite_sequence
WHERE name = 'reaction'
AND NOT EXISTS (SELECT 1 FROM sqlite_sequence WHERE name = 'reaction_new');
UPDATE sqlite_sequence
SET seq = MAX(seq, (SELECT seq FROM sqlite_sequence WHERE name = 'reaction'))
WHERE name = 'reaction_new';
DROP TABLE reaction;
ALTER TABLE reaction_new RENAME TO reaction;

View file

@ -37,6 +37,18 @@ SELECT
visibility, pinned, payload, NULL
FROM memo;
-- Preserve the largest ID ever issued, including IDs belonging to deleted
-- memos that were not copied into the rebuilt table.
INSERT INTO sqlite_sequence (name, seq)
SELECT 'memo_new', seq
FROM sqlite_sequence
WHERE name = 'memo'
AND NOT EXISTS (SELECT 1 FROM sqlite_sequence WHERE name = 'memo_new');
UPDATE sqlite_sequence
SET seq = MAX(seq, (SELECT seq FROM sqlite_sequence WHERE name = 'memo'))
WHERE name = 'memo_new';
DROP TABLE memo;
ALTER TABLE memo_new RENAME TO memo;

View file

@ -37,7 +37,7 @@ const (
// cannot change what CI verifies. Bump this when a new stable ships.
// scripts/release_smoke_test.sh detects the previous release from Git tags
// instead, so the black-box tier still follows "stable" automatically.
StableMemosVersion = "0.29.1"
StableMemosVersion = "0.30.0"
mysqlNetworkAlias = "memos-mysql"
postgresNetworkAlias = "memos-postgres"
@ -104,7 +104,7 @@ func GetMySQLDSN(t *testing.T) string {
}
container, err := mysql.Run(ctx,
"mysql:8",
"mysql:8.4",
mysql.WithDatabase("init_db"),
mysql.WithUsername("root"),
mysql.WithPassword(testPassword),

View file

@ -2,26 +2,27 @@ package test
import (
"context"
"fmt"
"testing"
"time"
"github.com/stretchr/testify/require"
colorpb "google.golang.org/genproto/googleapis/type/color"
storepb "github.com/usememos/memos/proto/gen/store"
"github.com/usememos/memos/store"
)
// TestUpgradeFromPreviousStableCopiesTagsToUserSettings covers the upgrade path
// users actually take: previous stable release to the current build, with data
// already in the database.
// TestUpgradeFromPreviousStableRenamesShortcutsToMemoViews covers the upgrade
// path users actually take: previous stable release to the current build, with
// data already in the database.
//
// The 0.30 tag migration is hand-written per driver with three different
// conflict forms (INSERT OR IGNORE, INSERT IGNORE, ON CONFLICT DO NOTHING) and
// different quoting of the reserved words "user" and "key". The existing
// fixture test for it only runs on SQLite, so this test drives the real
// previous-stable schema on whichever driver DRIVER selects.
func TestUpgradeFromPreviousStableCopiesTagsToUserSettings(t *testing.T) {
// The 0.31 SHORTCUTS rename is hand-written per driver with dialect-specific
// JSON rewriting (MySQL JSON_* functions and backtick quoting, SQLite json_*,
// Postgres jsonb operators) and per-driver validity guards. The existing
// fixture test drives a hand-maintained legacy schema; this test drives the
// schema the real previous-stable binary created on whichever driver DRIVER
// selects.
func TestUpgradeFromPreviousStableRenamesShortcutsToMemoViews(t *testing.T) {
if testing.Short() {
t.Skip("skipping container-based upgrade test in short mode")
}
@ -53,53 +54,56 @@ func TestUpgradeFromPreviousStableCopiesTagsToUserSettings(t *testing.T) {
require.NoError(t, container.Terminate(ctx), "failed to stop memos %s container", StableMemosVersion)
container = nil
// Seed through the store API. 0.29 shipped no migrations and 0.30 adds no
// DDL, so current store code reads and writes the previous stable schema.
// Seed through the store API where the schema is shared by v0.30.0 and the
// current build. The legacy SHORTCUTS rows predate the current proto, so
// they are seeded with raw SQL exactly as the previous release stored them.
seedStore := NewTestingStoreWithDSN(ctx, t, driver, hostDSN)
copiedUser, err := createTestingUserWithRole(ctx, seedStore, "tagcopy", store.RoleUser)
renamedUser, err := createTestingUserWithRole(ctx, seedStore, "legacyviews", store.RoleUser)
require.NoError(t, err)
keepsOwnUser, err := createTestingUserWithRole(ctx, seedStore, "keepsown", store.RoleUser)
require.NoError(t, err)
_, err = seedStore.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{
Key: storepb.InstanceSettingKey_TAGS,
Value: &storepb.InstanceSetting_TagsSetting{
TagsSetting: &storepb.InstanceTagsSetting{
Tags: map[string]*storepb.InstanceTagMetadata{
"bug": {
BackgroundColor: &colorpb.Color{Red: 0.9, Green: 0.1, Blue: 0.1},
},
"private/.*": {
BlurContent: true,
},
},
},
},
})
require.NoError(t, err, "should seed the instance-level TAGS setting")
settingKeyColumn := "key"
if driver == "mysql" {
settingKeyColumn = "`key`"
}
// Bound rather than inlined: MySQL treats backslash as an escape character
// inside string literals, so an inlined \" would collapse and corrupt the
// seeded JSON.
insertSetting := fmt.Sprintf("INSERT INTO user_setting (user_id, %[1]s, value) VALUES (?, ?, ?)", settingKeyColumn)
if driver == "postgres" {
insertSetting = fmt.Sprintf("INSERT INTO user_setting (user_id, %[1]s, value) VALUES ($1, $2, $3)", settingKeyColumn)
}
db := seedStore.GetDriver().GetDB()
_, err = db.ExecContext(ctx, insertSetting, renamedUser.ID, "SHORTCUTS",
`{"shortcuts":[{"id":"legacy","title":"legacy shortcut","filter":"tag in [\"legacy\"]"}]}`)
require.NoError(t, err, "should seed a legacy SHORTCUTS setting")
_, err = db.ExecContext(ctx, insertSetting, keepsOwnUser.ID, "SHORTCUTS",
`{"shortcuts":[{"id":"stale","title":"stale","filter":"tag in [\"stale\"]"}]}`)
require.NoError(t, err, "should seed a conflicting legacy SHORTCUTS setting")
// One user already has their own TAGS setting; the migration must not clobber it.
// One user already has a MEMO_VIEWS setting; the migration must not clobber it.
_, err = seedStore.UpsertUserSetting(ctx, &storepb.UserSetting{
UserId: keepsOwnUser.ID,
Key: storepb.UserSetting_TAGS,
Value: &storepb.UserSetting_Tags{
Tags: &storepb.TagsUserSetting{
Tags: map[string]*storepb.UserTagMetadata{
"existing": {BlurContent: true},
Key: storepb.UserSetting_MEMO_VIEWS,
Value: &storepb.UserSetting_MemoViews{
MemoViews: &storepb.MemoViewsUserSetting{
MemoViews: []*storepb.MemoViewsUserSetting_MemoView{
{Id: "own", Title: "own view", Filter: `tag in ["own"]`},
},
},
},
})
require.NoError(t, err, "should seed a pre-existing user TAGS setting")
require.NoError(t, err, "should seed a pre-existing MEMO_VIEWS setting")
// The user that should receive a copy must not have one yet.
// The user whose setting should be renamed must not have one yet.
preExisting, err := seedStore.GetUserSetting(ctx, &store.FindUserSetting{
UserID: &copiedUser.ID,
Key: storepb.UserSetting_TAGS,
UserID: &renamedUser.ID,
Key: storepb.UserSetting_MEMO_VIEWS,
})
require.NoError(t, err)
require.Nil(t, preExisting, "the seeded user should not have a TAGS setting before the upgrade")
require.Nil(t, preExisting, "the seeded user should not have a MEMO_VIEWS setting before the upgrade")
// Upgrade with current code.
ts := NewTestingStoreWithDSN(ctx, t, driver, hostDSN)
@ -111,30 +115,27 @@ func TestUpgradeFromPreviousStableCopiesTagsToUserSettings(t *testing.T) {
require.NoError(t, err)
require.Equal(t, currentVersion, upgradedSetting.SchemaVersion, "schema version should advance")
// The instance tags should now exist on the user that had none.
copied, err := ts.GetUserSetting(ctx, &store.FindUserSetting{
UserID: &copiedUser.ID,
Key: storepb.UserSetting_TAGS,
// The legacy SHORTCUTS setting is now readable as MEMO_VIEWS.
renamed, err := ts.GetUserSetting(ctx, &store.FindUserSetting{
UserID: &renamedUser.ID,
Key: storepb.UserSetting_MEMO_VIEWS,
})
require.NoError(t, err)
require.NotNil(t, copied, "instance tags should be copied to the user")
require.Contains(t, copied.GetTags().GetTags(), "bug")
bugMetadata := copied.GetTags().GetTags()["bug"]
require.NotNil(t, bugMetadata.GetBackgroundColor())
require.InDelta(t, 0.9, bugMetadata.GetBackgroundColor().GetRed(), 1e-6)
require.InDelta(t, 0.1, bugMetadata.GetBackgroundColor().GetGreen(), 1e-6)
require.InDelta(t, 0.1, bugMetadata.GetBackgroundColor().GetBlue(), 1e-6)
require.True(t, copied.GetTags().GetTags()["private/.*"].GetBlurContent())
require.NotNil(t, renamed, "legacy shortcuts should be renamed to memo views")
require.Len(t, renamed.GetMemoViews().GetMemoViews(), 1)
require.Equal(t, "legacy", renamed.GetMemoViews().GetMemoViews()[0].GetId())
require.Equal(t, "legacy shortcut", renamed.GetMemoViews().GetMemoViews()[0].GetTitle())
require.Equal(t, `tag in ["legacy"]`, renamed.GetMemoViews().GetMemoViews()[0].GetFilter())
// The user with their own setting keeps it untouched.
kept, err := ts.GetUserSetting(ctx, &store.FindUserSetting{
UserID: &keepsOwnUser.ID,
Key: storepb.UserSetting_TAGS,
Key: storepb.UserSetting_MEMO_VIEWS,
})
require.NoError(t, err)
require.NotNil(t, kept)
require.Contains(t, kept.GetTags().GetTags(), "existing")
require.NotContains(t, kept.GetTags().GetTags(), "bug", "existing user tags should not be overwritten")
require.Len(t, kept.GetMemoViews().GetMemoViews(), 1)
require.Equal(t, "own", kept.GetMemoViews().GetMemoViews()[0].GetId(), "existing memo views should not be overwritten")
// Re-running the upgrade must stay a no-op, which is what a container
// restart on an already-upgraded volume does.
@ -143,7 +144,7 @@ func TestUpgradeFromPreviousStableCopiesTagsToUserSettings(t *testing.T) {
// The upgraded database must still accept writes.
postUpgradeMemo, err := ts.CreateMemo(ctx, &store.Memo{
UID: "post-stable-upgrade-memo",
CreatorID: copiedUser.ID,
CreatorID: renamedUser.ID,
Content: "created after upgrading from previous stable",
Visibility: store.Private,
})

View file

@ -187,6 +187,14 @@ func TestMigrationMultiSpacesPreservesMemosAndRelations(t *testing.T) {
_, err = db.ExecContext(ctx, insertMemo, memo.id, memo.uid, memo.uid, memo.visibility, memo.rowStatus, memo.pinned, memo.payload)
require.NoError(t, err)
}
_, err = db.ExecContext(ctx, insertMemo, 100, "deleted-high-memo", "deleted", store.Private, store.Normal, false, `{}`)
require.NoError(t, err)
_, err = db.ExecContext(ctx, "DELETE FROM memo WHERE id = 100")
require.NoError(t, err)
if driver == "postgres" {
_, err = db.ExecContext(ctx, "SELECT setval(pg_get_serial_sequence('memo', 'id'), 100, true)")
require.NoError(t, err)
}
for _, relation := range []struct {
memoID, relatedMemoID int32
typeName store.MemoRelationType
@ -256,6 +264,23 @@ func TestMigrationMultiSpacesPreservesMemosAndRelations(t *testing.T) {
{memoID: 3, relatedMemoID: 2, typeName: store.MemoRelationComment},
}, relations)
insertNextMemo := "INSERT INTO memo (uid, creator_id, content, visibility, payload) VALUES (?, ?, ?, ?, ?)"
if driver == "postgres" {
var nextID int64
err = ts.GetDriver().GetDB().QueryRowContext(ctx,
"INSERT INTO memo (uid, creator_id, content, visibility, payload) VALUES ($1, $2, $3, $4, $5) RETURNING id",
"after-migration", 0, "after", store.Private, `{}`,
).Scan(&nextID)
require.NoError(t, err)
require.Equal(t, int64(101), nextID)
} else {
result, err := ts.GetDriver().GetDB().ExecContext(ctx, insertNextMemo, "after-migration", 0, "after", store.Private, `{}`)
require.NoError(t, err)
nextID, err := result.LastInsertId()
require.NoError(t, err)
require.Equal(t, int64(101), nextID)
}
requireQueryError(ctx, t, ts.GetDriver().GetDB(), "SELECT parent_memo_id, root_memo_id FROM memo LIMIT 0", "memo-local schema must not add canonical-root columns")
requireQueryError(ctx, t, ts.GetDriver().GetDB(), "SELECT row_status FROM space LIMIT 0", "Space has no archived state")
if driver == "sqlite" {
@ -660,6 +685,10 @@ func TestMigrationReactionMemoID(t *testing.T) {
require.NoError(t, err)
_, err = db.ExecContext(ctx, insertReaction, 101, 7, "memos/missing-target", "orphan")
require.NoError(t, err)
if driver == "postgres" {
_, err = db.ExecContext(ctx, "SELECT setval(pg_get_serial_sequence('reaction', 'id'), 101, true)")
require.NoError(t, err)
}
require.NoError(t, db.Close())
ts := NewTestingStoreWithDSN(ctx, t, driver, dsn)
@ -682,6 +711,19 @@ func TestMigrationReactionMemoID(t *testing.T) {
err = ts.GetDriver().GetDB().QueryRowContext(ctx, "SELECT COUNT(*) FROM reaction").Scan(&reactionCount)
require.NoError(t, err)
require.Equal(t, 1, reactionCount, "orphaned reactions must be discarded")
insertNextReaction := "INSERT INTO reaction (creator_id, memo_id, reaction_type) VALUES (?, ?, ?)"
findNextReaction := "SELECT id FROM reaction WHERE creator_id = ? AND memo_id = ? AND reaction_type = ?"
if driver == "postgres" {
insertNextReaction = "INSERT INTO reaction (creator_id, memo_id, reaction_type) VALUES ($1, $2, $3)"
findNextReaction = "SELECT id FROM reaction WHERE creator_id = $1 AND memo_id = $2 AND reaction_type = $3"
}
_, err = ts.GetDriver().GetDB().ExecContext(ctx, insertNextReaction, 8, 42, "after-migration")
require.NoError(t, err)
var nextReactionID int32
err = ts.GetDriver().GetDB().QueryRowContext(ctx, findNextReaction, 8, 42, "after-migration").Scan(&nextReactionID)
require.NoError(t, err)
require.Equal(t, int32(102), nextReactionID)
}
// TestMigrationLegacyS3AttachmentMinIO verifies the storage upgrade path for an
@ -798,7 +840,7 @@ func TestMigrationMemoViewSetting(t *testing.T) {
_, err = db.ExecContext(ctx, legacySchemaFixture(driver))
require.NoError(t, err)
basicSettingBytes, err := protojson.Marshal(&storepb.InstanceBasicSetting{SchemaVersion: "0.30.2"})
basicSettingBytes, err := protojson.Marshal(&storepb.InstanceBasicSetting{SchemaVersion: "0.30.1"})
require.NoError(t, err)
insertBasicSetting := "INSERT INTO system_setting (name, value, description) VALUES ('BASIC', ?, '')"
if driver == "postgres" {
@ -829,6 +871,9 @@ func TestMigrationMemoViewSetting(t *testing.T) {
// Already has a MEMO_VIEWS row, must not trip UNIQUE(user_id, key).
{3, "SHORTCUTS", `{"shortcuts":[{"id":"old","title":"old","filter":"tag in [\"old\"]"}]}`},
{3, "MEMO_VIEWS", `{"memoViews":[{"id":"new","title":"new","filter":"tag in [\"new\"]"}]}`},
{4, "SHORTCUTS", `"scalar"`},
{5, "SHORTCUTS", `[]`},
{6, "SHORTCUTS", `null`},
}
for _, row := range settingRows {
_, err = db.ExecContext(ctx, insertSetting, row.userID, row.key, row.value)
@ -861,6 +906,12 @@ func TestMigrationMemoViewSetting(t *testing.T) {
err = ts.GetDriver().GetDB().QueryRowContext(ctx, findCorruptSetting, 2, "SHORTCUTS").Scan(&corruptValue)
require.NoError(t, err)
require.Equal(t, `{oops}`, corruptValue)
for userID, rawValue := range map[int]string{4: `"scalar"`, 5: `[]`, 6: `null`} {
var value string
err = ts.GetDriver().GetDB().QueryRowContext(ctx, findCorruptSetting, userID, "SHORTCUTS").Scan(&value)
require.NoError(t, err)
require.Equal(t, rawValue, value)
}
// The pre-existing MEMO_VIEWS row wins; the legacy row is left behind untouched.
conflictUserID := int32(3)
@ -950,7 +1001,9 @@ func TestMigrationCopiesInstanceTagsToUserSettings(t *testing.T) {
require.NotContains(t, existing.GetTags().GetTags(), "bug")
}
func TestCaseSensitiveUsernameMigration(t *testing.T) {
// TestMigrationCaseSensitiveUsername is named to match the smoke workflow's
// -run 'TestMigration|...' filter so its per-driver assertions run on every driver.
func TestMigrationCaseSensitiveUsername(t *testing.T) {
ctx := context.Background()
driver := getDriverFromEnv()
var dsn string
@ -987,6 +1040,23 @@ func TestCaseSensitiveUsernameMigration(t *testing.T) {
}
_, err = db.ExecContext(ctx, insertUser)
require.NoError(t, err)
insertHighUser := "INSERT INTO user (id, username, role, password_hash, avatar_url) VALUES (100, 'deleted-high-user', 'USER', 'legacy-hash', '')"
deleteHighUser := "DELETE FROM user WHERE id = 100"
if driver == "mysql" {
insertHighUser = "INSERT INTO `user` (id, username, role, password_hash, avatar_url) VALUES (100, 'deleted-high-user', 'USER', 'legacy-hash', '')"
deleteHighUser = "DELETE FROM `user` WHERE id = 100"
} else if driver == "postgres" {
insertHighUser = `INSERT INTO "user" (id, username, role, password_hash, avatar_url) VALUES (100, 'deleted-high-user', 'USER', 'legacy-hash', '')`
deleteHighUser = `DELETE FROM "user" WHERE id = 100`
}
_, err = db.ExecContext(ctx, insertHighUser)
require.NoError(t, err)
_, err = db.ExecContext(ctx, deleteHighUser)
require.NoError(t, err)
if driver == "postgres" {
_, err = db.ExecContext(ctx, `SELECT setval(pg_get_serial_sequence('"user"', 'id'), 100, true)`)
require.NoError(t, err)
}
require.NoError(t, db.Close())
ts := NewTestingStoreWithDSN(ctx, t, driver, dsn)
@ -995,6 +1065,7 @@ func TestCaseSensitiveUsernameMigration(t *testing.T) {
lower, err := createTestingUserWithRole(ctx, ts, "alice", store.RoleUser)
require.NoError(t, err)
require.Equal(t, int32(101), lower.ID)
upperUsername := "Alice"
upper, err := ts.GetUser(ctx, &store.FindUser{Username: &upperUsername})