From e24f102dc736ecd87d4e3c08c0a36fbc68da60fe Mon Sep 17 00:00:00 2001 From: Zakk Date: Sun, 20 Sep 2026 05:48:45 +1000 Subject: [PATCH] fix(db): an upgrade from BoltDB keeps its groups and can always restart The migration wrote each outbound's connected servers and setting into the outbound_connections and outbound_settings tables, while the runtime reads both from system_config under outbound.:connectedServers and outbound.:setting. Every upgrade from a BoltDB release therefore came up with no group members and default outbound settings. The migration now stores the Bolt values under the keys the runtime reads (the ordinals are the ones configure.Which resolves at runtime) and the verification checks those keys. The two tables are gone from the schema; a database that still has them is left alone. The migration also built v2raya.db in place, so an error before the rename left a partial database that made the next start skip the migration and then fail on the remaining bolt.db. It builds a temporary file and renames it in after verification; a bolt.db found next to an existing v2raya.db is moved to a numbered backup instead of blocking every start. The v4 import no longer copies boltv4.db to bolt.db, which produced exactly that pair. A subscription update rewrote the connected ordinals and then replaced the subscription in two commits; they are one transaction. --- service/db/configure/configure.go | 16 ++ service/db/configure/configure_test.go | 47 +++++ service/db/listOp.go | 50 ++--- service/db/migrate.go | 280 ++++++++++++++----------- service/db/migrate_test.go | 160 ++++++++++++++ service/db/plainOp.go | 15 +- service/db/schema.go | 16 -- service/db/schema_test.go | 48 +++++ service/db/setOp.go | 4 - service/pre_database.go | 16 +- service/server/service/subscription.go | 5 +- 11 files changed, 463 insertions(+), 194 deletions(-) create mode 100644 service/db/migrate_test.go create mode 100644 service/db/schema_test.go diff --git a/service/db/configure/configure.go b/service/db/configure/configure.go index 86805c40..0485e194 100644 --- a/service/db/configure/configure.go +++ b/service/db/configure/configure.go @@ -2,6 +2,7 @@ package configure import ( "bytes" + "database/sql" "encoding/hex" "fmt" "sort" @@ -127,6 +128,21 @@ func SetServer(index int, server *ServerRaw) (err error) { func SetSubscription(index int, subscription *SubscriptionRaw) (err error) { return db.ListSet("touch", "subscriptions", index, subscription) } + +func SetSubscriptionAndConnects(index int, subscription *SubscriptionRaw, ws *Whiches) error { + return db.ListSetWithTransaction("touch", "subscriptions", index, subscription, func(tx *sql.Tx) error { + outboundWhiches := make(map[string][]*Which) + for _, which := range ws.Get() { + outboundWhiches[which.Outbound] = append(outboundWhiches[which.Outbound], which) + } + for outbound, touches := range outboundWhiches { + if err := db.SetTx(tx, fmt.Sprintf("outbound.%v", outbound), "connectedServers", &Whiches{Touches: touches}); err != nil { + return err + } + } + return nil + }) +} func SetSetting(setting *Setting) (err error) { return db.Set("system", "setting", setting) } diff --git a/service/db/configure/configure_test.go b/service/db/configure/configure_test.go index 5f712b6a..58e95250 100644 --- a/service/db/configure/configure_test.go +++ b/service/db/configure/configure_test.go @@ -26,3 +26,50 @@ func TestGetLenSubscriptionServersPanicsOnMissingSubscription(t *testing.T) { }() GetLenSubscriptionServers(-1) } + +func TestSetSubscriptionAndConnectsRollsBackConnections(t *testing.T) { + const outbound = "transaction-test" + if err := AddOutbound(outbound); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = RemoveOutbound(outbound) }) + + index := GetLenSubscriptions() + subscription := &SubscriptionRaw{Address: "https://example.test/subscription"} + if err := AppendSubscriptions([]*SubscriptionRaw{subscription}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = RemoveSubscriptions([]int{index}) }) + + initial := &Whiches{Touches: []*Which{{ + TYPE: SubscriptionServerType, + ID: 1, + Sub: index, + Outbound: outbound, + }}} + if err := OverwriteConnects(initial); err != nil { + t.Fatal(err) + } + replacement := &Whiches{Touches: []*Which{{ + TYPE: SubscriptionServerType, + ID: 2, + Sub: index, + Outbound: outbound, + }}} + if err := SetSubscriptionAndConnects(index+1, subscription, replacement); err == nil { + t.Fatal("updating a missing subscription succeeded") + } + + got := GetConnectedServersByOutbound(outbound) + if got == nil || got.Len() != 1 || *got.Get()[0] != *initial.Get()[0] { + t.Fatalf("connections changed after rollback: %+v", got) + } + + if err := SetSubscriptionAndConnects(index, subscription, replacement); err != nil { + t.Fatalf("replace subscription and connections: %v", err) + } + got = GetConnectedServersByOutbound(outbound) + if got == nil || got.Len() != 1 || *got.Get()[0] != *replacement.Get()[0] { + t.Fatalf("connections were not committed with subscription: %+v", got) + } +} diff --git a/service/db/listOp.go b/service/db/listOp.go index f7c6f7c2..d4653050 100644 --- a/service/db/listOp.go +++ b/service/db/listOp.go @@ -32,21 +32,34 @@ func ListSet(bucket string, key string, index int, val interface{}) (err error) return nil case "touch/subscriptions": - b, err := jsoniter.Marshal(val) - if err != nil { - return err - } - // One transaction: the node list is deleted and re-inserted, and a - // crash between the two must not leave the subscription empty. - return ReadModifyWrite(func(tx *sql.Tx) error { - return setSubscription(tx, index, gjson.ParseBytes(b)) - }) + return ListSetWithTransaction(bucket, key, index, val, nil) default: return fmt.Errorf("ListSet: unsupported bucket/key: %s/%s", bucket, key) } } +// ListSetWithTransaction runs beforeSet and the list replacement in one +// transaction. The callback may write related state through transaction-aware +// database operations. +func ListSetWithTransaction(bucket string, key string, index int, val interface{}, beforeSet func(tx *sql.Tx) error) error { + if bucket+"/"+key != "touch/subscriptions" { + return fmt.Errorf("ListSetWithTransaction: unsupported bucket/key: %s/%s", bucket, key) + } + b, err := jsoniter.Marshal(val) + if err != nil { + return err + } + return ReadModifyWrite(func(tx *sql.Tx) error { + if beforeSet != nil { + if err := beforeSet(tx); err != nil { + return err + } + } + return setSubscription(tx, index, gjson.ParseBytes(b)) + }) +} + func setSubscription(tx *sql.Tx, index int, parsed gjson.Result) error { address := parsed.Get("address").String() remarks := parsed.Get("remarks").String() @@ -72,15 +85,6 @@ func setSubscription(tx *sql.Tx, index int, parsed gjson.Result) error { return err } - // Update servers within this subscription. - // Clean up outbound_connections first to satisfy foreign key constraint; - // otherwise the delete fails and the insert below duplicates the list. - if _, err := tx.Exec(` - DELETE FROM outbound_connections - WHERE server_id IN (SELECT id FROM servers WHERE type = 'subscription_server' AND sub_id = ?) - `, subID); err != nil { - return fmt.Errorf("ListSet: failed to clear outbound connections of subscription %d: %w", index, err) - } if _, err := tx.Exec("DELETE FROM servers WHERE type = 'subscription_server' AND sub_id = ?", subID); err != nil { return fmt.Errorf("ListSet: failed to clear old servers of subscription %d: %w", index, err) } @@ -320,11 +324,6 @@ func ListRemove(bucket, key string, indexes []int) error { switch bucket + "/" + key { case "touch/servers": for _, idx := range indexes { - // Clean up outbound_connections first to satisfy foreign key constraint - _, _ = db.Exec(` - DELETE FROM outbound_connections - WHERE server_id IN (SELECT id FROM servers WHERE type = 'server' AND sort = ?) - `, idx) _, err := db.Exec("DELETE FROM servers WHERE type = 'server' AND sort = ?", idx) if err != nil { return err @@ -349,11 +348,6 @@ func ListRemove(bucket, key string, indexes []int) error { } return err } - // Clean up outbound_connections first to satisfy foreign key constraint - _, _ = db.Exec(` - DELETE FROM outbound_connections - WHERE server_id IN (SELECT id FROM servers WHERE type = 'subscription_server' AND sub_id = ?) - `, subID) _, err = db.Exec("DELETE FROM servers WHERE type = 'subscription_server' AND sub_id = ?", subID) if err != nil { return err diff --git a/service/db/migrate.go b/service/db/migrate.go index 63212a7d..ba91a6d6 100644 --- a/service/db/migrate.go +++ b/service/db/migrate.go @@ -2,9 +2,11 @@ package db import ( "database/sql" + "encoding/json" "fmt" "os" "path/filepath" + "strings" "github.com/tidwall/gjson" "github.com/v2rayA/v2rayA/conf" @@ -14,7 +16,7 @@ import ( // MigrateFromBoltDB migrates data from BoltDB to SQLite. // It opens bolt.db, reads all data, creates a fresh SQLite database, -// writes data to SQLite, verifies integrity, then renames bolt.db to bolt.db.bak. +// writes data to SQLite, verifies integrity, then renames bolt.db to a backup. // This function does NOT use GetDB() — it creates its own SQLite connection // so that migration can happen before the normal SQLite initialization. func MigrateFromBoltDB() error { @@ -22,16 +24,23 @@ func MigrateFromBoltDB() error { boltPath := filepath.Join(confPath, "bolt.db") sqlitePath := filepath.Join(confPath, "v2raya.db") - // Check if BoltDB file exists - if _, err := os.Stat(boltPath); os.IsNotExist(err) { - log.Info("No BoltDB file found at %s, skipping migration", boltPath) - return nil + if _, err := os.Stat(boltPath); err != nil { + if os.IsNotExist(err) { + log.Info("No BoltDB file found at %s, skipping migration", boltPath) + return nil + } + return fmt.Errorf("failed to inspect BoltDB: %w", err) } - // Check if SQLite already exists (migration already done or fresh start) if _, err := os.Stat(sqlitePath); err == nil { - log.Info("SQLite database already exists at %s, skipping migration", sqlitePath) + backupPath, err := moveBoltAside(boltPath) + if err != nil { + return fmt.Errorf("SQLite database already exists but bolt.db could not be moved aside: %w", err) + } + log.Warn("SQLite database already exists; moved stale bolt.db to %s", backupPath) return nil + } else if !os.IsNotExist(err) { + return fmt.Errorf("failed to inspect SQLite database: %w", err) } log.Warn("Migrating from BoltDB to SQLite...") @@ -41,14 +50,37 @@ func MigrateFromBoltDB() error { if err != nil { return fmt.Errorf("failed to open BoltDB: %w", err) } - defer boltDB.Close() + defer func() { + if boltDB != nil { + _ = boltDB.Close() + } + }() - // Create a fresh SQLite database independently (do NOT use GetDB) - sqldb, err := createSQLiteDB(sqlitePath) + tempFile, err := os.CreateTemp(confPath, ".v2raya.db.migrate-*") + if err != nil { + return fmt.Errorf("failed to create temporary SQLite database: %w", err) + } + tempPath := tempFile.Name() + if err := tempFile.Close(); err != nil { + _ = os.Remove(tempPath) + return fmt.Errorf("failed to close temporary SQLite database: %w", err) + } + defer func() { + _ = os.Remove(tempPath) + _ = os.Remove(tempPath + "-shm") + _ = os.Remove(tempPath + "-wal") + }() + + // Create a fresh SQLite database independently (do NOT use GetDB). + sqldb, err := createSQLiteDB(tempPath) if err != nil { return fmt.Errorf("failed to create SQLite database for migration: %w", err) } - defer sqldb.Close() + defer func() { + if sqldb != nil { + _ = sqldb.Close() + } + }() // Perform migration in a single transaction tx, err := sqldb.Begin() @@ -69,9 +101,7 @@ func MigrateFromBoltDB() error { } // Migrate touch bucket (servers and subscriptions) - // subIDMap maps BoltDB subscription index (0-based) -> SQLite subscription ID - subIDMap, err := migrateTouchBucket(boltDB, tx) - if err != nil { + if err := migrateTouchBucket(boltDB, tx); err != nil { _ = tx.Rollback() return fmt.Errorf("failed to migrate touch bucket: %w", err) } @@ -81,7 +111,7 @@ func MigrateFromBoltDB() error { // and requiring re-registration ensures users set up fresh bcrypt-based credentials. // Migrate outbounds bucket - if err := migrateOutboundsBucket(boltDB, tx, subIDMap); err != nil { + if err := migrateOutboundsBucket(boltDB, tx); err != nil { _ = tx.Rollback() return fmt.Errorf("failed to migrate outbounds bucket: %w", err) } @@ -96,19 +126,46 @@ func MigrateFromBoltDB() error { return fmt.Errorf("migration verification failed: %w", err) } - // Close BoltDB before renaming - boltDB.Close() + if err := sqldb.Close(); err != nil { + return fmt.Errorf("failed to close migrated SQLite database: %w", err) + } + sqldb = nil + if err := boltDB.Close(); err != nil { + return fmt.Errorf("failed to close BoltDB: %w", err) + } + boltDB = nil - // Rename old BoltDB to .bak as backup - backupPath := boltPath + ".bak" - if err := os.Rename(boltPath, backupPath); err != nil { - return fmt.Errorf("failed to rename bolt.db to bolt.db.bak: %w", err) + if err := os.Rename(tempPath, sqlitePath); err != nil { + return fmt.Errorf("failed to publish migrated SQLite database: %w", err) + } + backupPath, err := moveBoltAside(boltPath) + if err != nil { + if rollbackErr := os.Rename(sqlitePath, tempPath); rollbackErr != nil { + return fmt.Errorf("failed to back up bolt.db: %v; failed to remove published SQLite database: %w", err, rollbackErr) + } + return fmt.Errorf("failed to back up bolt.db: %w", err) } - log.Warn("Migration completed successfully. Old BoltDB backed up to bolt.db.bak") + log.Warn("Migration completed successfully. Old BoltDB backed up to %s", backupPath) return nil } +func moveBoltAside(boltPath string) (string, error) { + backupPath := boltPath + ".bak" + for suffix := 1; ; suffix++ { + if _, err := os.Stat(backupPath); os.IsNotExist(err) { + break + } else if err != nil { + return "", err + } + backupPath = fmt.Sprintf("%s.bak.%d", boltPath, suffix) + } + if err := os.Rename(boltPath, backupPath); err != nil { + return "", err + } + return backupPath, nil +} + // createSQLiteDB creates a new SQLite database file at the given path, // opens a connection, configures PRAGMAs, and initializes the schema. func createSQLiteDB(dbPath string) (*sql.DB, error) { @@ -180,12 +237,9 @@ func migrateSystemBucket(boltDB *bbolt.DB, tx *sql.Tx) error { }) } -// migrateTouchBucket migrates the touch bucket (servers and subscriptions) -// Returns a map of BoltDB subscription index (0-based) -> SQLite subscription ID -// for use in outbound connection migration. -func migrateTouchBucket(boltDB *bbolt.DB, tx *sql.Tx) (map[int64]int64, error) { - var subIDMap map[int64]int64 - err := boltDB.View(func(btx *bbolt.Tx) error { +// migrateTouchBucket migrates the touch bucket (servers and subscriptions). +func migrateTouchBucket(boltDB *bbolt.DB, tx *sql.Tx) error { + return boltDB.View(func(btx *bbolt.Tx) error { bkt := btx.Bucket([]byte("touch")) if bkt == nil { log.Info("No touch bucket found, skipping") @@ -203,19 +257,13 @@ func migrateTouchBucket(boltDB *bbolt.DB, tx *sql.Tx) (map[int64]int64, error) { // Migrate subscriptions subsJSON := bkt.Get([]byte("subscriptions")) if subsJSON != nil { - var err error - subIDMap, err = migrateSubscriptions(subsJSON, tx) - if err != nil { + if err := migrateSubscriptions(subsJSON, tx); err != nil { return err } } return nil }) - if err != nil { - return nil, err - } - return subIDMap, nil } // migrateServers migrates the servers JSON array to the servers table @@ -255,30 +303,27 @@ func migrateServers(data []byte, tx *sql.Tx) error { } // migrateSubscriptions migrates the subscriptions JSON array to the subscriptions table. -// Returns a map of BoltDB subscription index (0-based) -> SQLite subscription ID. -func migrateSubscriptions(data []byte, tx *sql.Tx) (map[int64]int64, error) { +func migrateSubscriptions(data []byte, tx *sql.Tx) error { if !gjson.ValidBytes(data) { - return nil, fmt.Errorf("invalid JSON for subscriptions") + return fmt.Errorf("invalid JSON for subscriptions") } parsed := gjson.ParseBytes(data) if !parsed.IsArray() { - return nil, fmt.Errorf("subscriptions data is not an array") + return fmt.Errorf("subscriptions data is not an array") } results := parsed.Array() if len(results) == 0 { - return nil, nil + return nil } - subIDMap := make(map[int64]int64, len(results)) - subStmt, err := tx.Prepare(` INSERT INTO subscriptions (address, remarks, status, info, auto_select, sort) VALUES (?, ?, ?, ?, ?, ?) `) if err != nil { - return nil, err + return err } defer subStmt.Close() @@ -287,7 +332,7 @@ func migrateSubscriptions(data []byte, tx *sql.Tx) (map[int64]int64, error) { VALUES ('subscription_server', ?, '', 0, '', ?, '', '', '', ?) `) if err != nil { - return nil, err + return err } defer serverStmt.Close() @@ -303,34 +348,31 @@ func migrateSubscriptions(data []byte, tx *sql.Tx) (map[int64]int64, error) { res, err := subStmt.Exec(address, remarks, status, info, autoSelect, i) if err != nil { - return nil, fmt.Errorf("failed to insert subscription %d: %w", i, err) + return fmt.Errorf("failed to insert subscription %d: %w", i, err) } subID, err := res.LastInsertId() if err != nil { - return nil, fmt.Errorf("failed to get last insert id for subscription %d: %w", i, err) + return fmt.Errorf("failed to get last insert id for subscription %d: %w", i, err) } - // Record the mapping: BoltDB index (0-based) -> SQLite subscription ID - subIDMap[int64(i)] = subID - // Migrate servers within this subscription servers := r.Get("servers").Array() for j, s := range servers { configJSON := s.Raw if _, err := serverStmt.Exec(subID, configJSON, j); err != nil { - return nil, fmt.Errorf("failed to insert subscription server %d/%d: %w", i, j, err) + return fmt.Errorf("failed to insert subscription server %d/%d: %w", i, j, err) } } log.Info("Migrated subscription %d with %d servers", i, len(servers)) } - return subIDMap, nil + return nil } // migrateOutboundsBucket migrates the outbounds bucket -func migrateOutboundsBucket(boltDB *bbolt.DB, tx *sql.Tx, subIDMap map[int64]int64) error { +func migrateOutboundsBucket(boltDB *bbolt.DB, tx *sql.Tx) error { return boltDB.View(func(btx *bbolt.Tx) error { // Migrate outbound names from outbounds/names set outboundsBkt := btx.Bucket([]byte("outbounds")) @@ -364,7 +406,7 @@ func migrateOutboundsBucket(boltDB *bbolt.DB, tx *sql.Tx, subIDMap map[int64]int // Migrate connectedServers connData := outboundBkt.Get([]byte("connectedServers")) if connData != nil { - if err := migrateOutboundConnections(outboundName, connData, tx, subIDMap); err != nil { + if err := migrateOutboundConnections(outboundName, connData, tx); err != nil { return err } } @@ -424,88 +466,25 @@ func migrateOutboundSetting(outboundName string, data []byte, tx *sql.Tx) error return err } - if _, err := tx.Exec("INSERT OR REPLACE INTO outbound_settings (outbound_name, setting_json) VALUES (?, ?)", - outboundName, string(data)); err != nil { + key := makeKey("outbound."+outboundName, "setting") + if _, err := tx.Exec("INSERT OR REPLACE INTO system_config (key, value) VALUES (?, ?)", key, string(data)); err != nil { return fmt.Errorf("failed to insert outbound setting for %s: %w", outboundName, err) } log.Info("Migrated outbound setting: %s", outboundName) return nil } -// migrateOutboundConnections migrates connected servers for an outbound. -// subIDMap maps BoltDB subscription index (0-based) -> SQLite subscription ID. -func migrateOutboundConnections(outboundName string, data []byte, tx *sql.Tx, subIDMap map[int64]int64) error { - if !gjson.ValidBytes(data) { - return nil - } - - // Ensure the outbound name exists in outbound_names table to satisfy - // the FOREIGN KEY constraint on outbound_connections.outbound_name. - // This is critical because migrateOutboundSetting may not be called - // if the outbound bucket has connectedServers but no setting data. +// migrateOutboundConnections preserves Bolt ordinals because configure.Which +// resolves them against the ordered server and subscription lists at runtime. +func migrateOutboundConnections(outboundName string, data []byte, tx *sql.Tx) error { if _, err := tx.Exec("INSERT OR IGNORE INTO outbound_names (name, sort) VALUES (?, 0)", outboundName); err != nil { return fmt.Errorf("failed to ensure outbound name %s exists: %w", outboundName, err) } - parsed := gjson.ParseBytes(data) - touches := parsed.Get("touches").Array() - - stmt, err := tx.Prepare(` - INSERT INTO outbound_connections (outbound_name, server_id, sort) - VALUES (?, ?, ?) - `) - if err != nil { - return err + key := makeKey("outbound."+outboundName, "connectedServers") + if _, err := tx.Exec("INSERT OR REPLACE INTO system_config (key, value) VALUES (?, ?)", key, string(data)); err != nil { + return fmt.Errorf("failed to insert outbound connections for %s: %w", outboundName, err) } - defer stmt.Close() - - for i, t := range touches { - touchType := t.Get("_type").String() - id := t.Get("id").Int() - - var serverID int64 - switch touchType { - case "server": - // In BoltDB, server IDs are 1-based sequential numbers. - // After migration to SQLite, servers get new auto-increment IDs. - // We look up the server by type='server' and sort=(id-1) since - // sort is set to the array index (0-based) during migration. - if err := tx.QueryRow( - "SELECT id FROM servers WHERE type = 'server' AND sort = ?", - id-1, - ).Scan(&serverID); err != nil { - log.Warn("Could not find server with sort=%d (original id=%d): %v — skipping connection entry", id-1, id, err) - continue - } - case "subscriptionServer": - // In BoltDB, subscription server IDs are 1-based within each subscription. - // The "sub" field in the touch data is the BoltDB subscription index (0-based). - // We need to map it to the SQLite subscription ID using subIDMap. - boltSubIdx := t.Get("sub").Int() - sqlSubID, ok := subIDMap[boltSubIdx] - if !ok { - log.Warn("Could not find SQLite subscription ID for BoltDB subscription index %d — skipping connection entry", boltSubIdx) - continue - } - if err := tx.QueryRow( - "SELECT id FROM servers WHERE type = 'subscription_server' AND sub_id = ? AND sort = ?", - sqlSubID, id-1, - ).Scan(&serverID); err != nil { - log.Warn("Could not find subscription server for sub_id=%d (BoltDB sub=%d), sort=%d: %v — skipping connection entry", sqlSubID, boltSubIdx, id-1, err) - continue - } - default: - continue - } - - if _, err := stmt.Exec(outboundName, serverID, i); err != nil { - // Log the error and skip this connection entry rather than failing the entire migration. - // This provides resilience against edge cases where server references may be stale. - log.Warn("Failed to insert outbound connection for %s (server_id=%d, sort=%d): %v — skipping", outboundName, serverID, i, err) - continue - } - } - return nil } @@ -524,7 +503,9 @@ func verifyMigration(boltDB *bbolt.DB, sqldb *sql.DB) error { }) var sqlCount int - sqldb.QueryRow("SELECT COUNT(*) FROM system_config").Scan(&sqlCount) + if err := sqldb.QueryRow("SELECT COUNT(*) FROM system_config WHERE key LIKE 'system:%'").Scan(&sqlCount); err != nil { + return fmt.Errorf("failed to count migrated system config: %w", err) + } if boltCount != sqlCount { return fmt.Errorf("system config count mismatch: BoltDB=%d, SQLite=%d", boltCount, sqlCount) @@ -559,7 +540,29 @@ func verifyMigration(boltDB *bbolt.DB, sqldb *sql.DB) error { } } - return nil + return btx.ForEach(func(name []byte, bucket *bbolt.Bucket) error { + if bucket == nil || !strings.HasPrefix(string(name), "outbound.") { + return nil + } + for _, key := range []string{"setting", "connectedServers"} { + value := bucket.Get([]byte(key)) + if value == nil { + continue + } + fullKey := makeKey(string(name), key) + var migrated string + if err := sqldb.QueryRow("SELECT value FROM system_config WHERE key = ?", fullKey).Scan(&migrated); err != nil { + return fmt.Errorf("failed to read migrated %s: %w", fullKey, err) + } + if migrated != string(value) { + return fmt.Errorf("outbound config mismatch for %s", fullKey) + } + if err := verifyOutboundValue(key, []byte(migrated)); err != nil { + return fmt.Errorf("invalid migrated %s: %w", fullKey, err) + } + } + return nil + }) }) if err != nil { @@ -569,3 +572,28 @@ func verifyMigration(boltDB *bbolt.DB, sqldb *sql.DB) error { log.Warn("Migration verification passed!") return nil } + +func verifyOutboundValue(key string, value []byte) error { + switch key { + case "setting": + var setting struct { + ProbeURL string `json:"probeURL"` + ProbeInterval string `json:"probeInterval"` + Type string `json:"type"` + Selected string `json:"selected,omitempty"` + } + return json.Unmarshal(value, &setting) + case "connectedServers": + var whiches struct { + Touches []struct { + Type string `json:"_type"` + ID int `json:"id"` + Sub int `json:"sub"` + Outbound string `json:"outbound"` + } `json:"touches"` + } + return json.Unmarshal(value, &whiches) + default: + return nil + } +} diff --git a/service/db/migrate_test.go b/service/db/migrate_test.go new file mode 100644 index 00000000..ee8eba31 --- /dev/null +++ b/service/db/migrate_test.go @@ -0,0 +1,160 @@ +package db_test + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/v2rayA/v2rayA/conf" + "github.com/v2rayA/v2rayA/db" + "github.com/v2rayA/v2rayA/db/configure" + "go.etcd.io/bbolt" +) + +const ( + validConnections = `{"touches":[{"_type":"subscriptionServer","id":1,"sub":0,"outbound":"proxy"},{"_type":"subscriptionServer","id":2,"sub":0,"outbound":"proxy"}]}` + outboundSetting = `{"probeURL":"https://example.test/ping","probeInterval":"45s","type":"leastping"}` + subscriptions = `[{"address":"https://example.test/subscription","servers":[{},{}]}]` +) + +func TestBoltMigrationPublishesVerifiedDatabaseAndRestarts(t *testing.T) { + configDir := t.TempDir() + conf.GetEnvironmentConfig().Config = configDir + boltPath := filepath.Join(configDir, "bolt.db") + sqlitePath := filepath.Join(configDir, "v2raya.db") + + writeBoltFixture(t, boltPath, []byte(`{"touches":`)) + before, err := os.ReadFile(boltPath) + if err != nil { + t.Fatal(err) + } + + if err := db.MigrateFromBoltDB(); err == nil { + t.Fatal("migration with corrupt outbound connections succeeded") + } else if !strings.Contains(err.Error(), "migration verification failed") { + t.Fatalf("migration failed before verification: %v", err) + } + if _, err := os.Stat(sqlitePath); !os.IsNotExist(err) { + t.Fatalf("failed migration published %s: %v", sqlitePath, err) + } + after, err := os.ReadFile(boltPath) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(after, before) { + t.Fatal("failed migration changed bolt.db") + } + temps, err := filepath.Glob(filepath.Join(configDir, ".v2raya.db.migrate-*")) + if err != nil { + t.Fatal(err) + } + if len(temps) != 0 { + t.Fatalf("failed migration left temporary files: %v", temps) + } + + setBoltValue(t, boltPath, "outbound.proxy", "connectedServers", []byte(validConnections)) + if err := db.MigrateFromBoltDB(); err != nil { + t.Fatalf("migration after fixture repair: %v", err) + } + if err := db.Open(); err != nil { + t.Fatalf("open migrated database: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + connected := configure.GetConnectedServersByOutbound("proxy") + if connected == nil || connected.Len() != 2 { + t.Fatalf("connected servers: %+v", connected) + } + for i, which := range connected.Get() { + if which.TYPE != configure.SubscriptionServerType || which.Sub != 0 || which.ID != i+1 || which.Outbound != "proxy" { + t.Fatalf("connected server %d: %+v", i, which) + } + } + wantSetting := configure.OutboundSetting{ + ProbeURL: "https://example.test/ping", + ProbeInterval: "45s", + Type: configure.LeastPing, + } + if got := configure.GetOutboundSetting("proxy"); got != wantSetting { + t.Fatalf("outbound setting: got %+v, want %+v", got, wantSetting) + } + + backup, err := os.ReadFile(boltPath + ".bak") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(boltPath, backup, 0600); err != nil { + t.Fatal(err) + } + if err := runStartupSequence(); err != nil { + t.Fatalf("startup with both databases: %v", err) + } + if err := runStartupSequence(); err != nil { + t.Fatalf("second startup: %v", err) + } + if _, err := os.Stat(boltPath); !os.IsNotExist(err) { + t.Fatalf("startup left bolt.db in place: %v", err) + } +} + +func runStartupSequence() error { + err := db.Open() + if !errors.Is(err, db.ErrNeedMigration) { + return err + } + if err := db.MigrateFromBoltDB(); err != nil { + return err + } + return db.Open() +} + +func writeBoltFixture(t *testing.T, path string, connections []byte) { + t.Helper() + database, err := bbolt.Open(path, 0600, nil) + if err != nil { + t.Fatal(err) + } + if err := database.Update(func(tx *bbolt.Tx) error { + touch, err := tx.CreateBucket([]byte("touch")) + if err != nil { + return err + } + if err := touch.Put([]byte("subscriptions"), []byte(subscriptions)); err != nil { + return err + } + outbound, err := tx.CreateBucket([]byte("outbound.proxy")) + if err != nil { + return err + } + if err := outbound.Put([]byte("setting"), []byte(outboundSetting)); err != nil { + return err + } + return outbound.Put([]byte("connectedServers"), connections) + }); err != nil { + _ = database.Close() + t.Fatal(err) + } + if err := database.Close(); err != nil { + t.Fatal(err) + } +} + +func setBoltValue(t *testing.T, path, bucket, key string, value []byte) { + t.Helper() + database, err := bbolt.Open(path, 0600, nil) + if err != nil { + t.Fatal(err) + } + if err := database.Update(func(tx *bbolt.Tx) error { + return tx.Bucket([]byte(bucket)).Put([]byte(key), value) + }); err != nil { + _ = database.Close() + t.Fatal(err) + } + if err := database.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/service/db/plainOp.go b/service/db/plainOp.go index 69b8049b..777a54f4 100644 --- a/service/db/plainOp.go +++ b/service/db/plainOp.go @@ -114,13 +114,24 @@ func GetBucketKeys(bucket string) (keys []string, err error) { // Set inserts or replaces a value in the system_config table. func Set(bucket string, key string, val interface{}) (err error) { + return setValue(GetDB(), bucket, key, val) +} + +func SetTx(tx *sql.Tx, bucket string, key string, val interface{}) error { + return setValue(tx, bucket, key, val) +} + +type sqlExecer interface { + Exec(query string, args ...interface{}) (sql.Result, error) +} + +func setValue(execer sqlExecer, bucket string, key string, val interface{}) error { b, err := jsoniter.Marshal(val) if err != nil { return err } - db := GetDB() fullKey := makeKey(bucket, key) - _, err = db.Exec("INSERT OR REPLACE INTO system_config (key, value) VALUES (?, ?)", fullKey, string(b)) + _, err = execer.Exec("INSERT OR REPLACE INTO system_config (key, value) VALUES (?, ?)", fullKey, string(b)) return err } diff --git a/service/db/schema.go b/service/db/schema.go index 7d510200..d134ff1d 100644 --- a/service/db/schema.go +++ b/service/db/schema.go @@ -57,22 +57,6 @@ CREATE TABLE IF NOT EXISTS outbound_names ( name TEXT PRIMARY KEY, sort INTEGER NOT NULL DEFAULT 0 ); - -CREATE TABLE IF NOT EXISTS outbound_connections ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - outbound_name TEXT NOT NULL, - server_id INTEGER NOT NULL, - sort INTEGER NOT NULL DEFAULT 0, - FOREIGN KEY (outbound_name) REFERENCES outbound_names(name), - FOREIGN KEY (server_id) REFERENCES servers(id), - UNIQUE(outbound_name, server_id) -); - -CREATE TABLE IF NOT EXISTS outbound_settings ( - outbound_name TEXT PRIMARY KEY, - setting_json TEXT NOT NULL DEFAULT '{}', - FOREIGN KEY (outbound_name) REFERENCES outbound_names(name) -); ` // InitSchema creates all tables if they don't exist diff --git a/service/db/schema_test.go b/service/db/schema_test.go new file mode 100644 index 00000000..68b0fad2 --- /dev/null +++ b/service/db/schema_test.go @@ -0,0 +1,48 @@ +package db + +import ( + "database/sql" + "testing" +) + +func TestInitSchemaLeavesLegacyOutboundTablesUnmanaged(t *testing.T) { + database, err := sql.Open(sqliteDriverName, ":memory:") + if err != nil { + t.Fatal(err) + } + defer database.Close() + database.SetMaxOpenConns(1) + + if err := InitSchema(database); err != nil { + t.Fatal(err) + } + for _, name := range []string{"outbound_connections", "outbound_settings"} { + if tableExists(t, database, name) { + t.Fatalf("fresh schema created legacy table %s", name) + } + } + + if _, err := database.Exec(` + CREATE TABLE outbound_connections (id INTEGER PRIMARY KEY); + CREATE TABLE outbound_settings (outbound_name TEXT PRIMARY KEY); + `); err != nil { + t.Fatal(err) + } + if err := InitSchema(database); err != nil { + t.Fatal(err) + } + for _, name := range []string{"outbound_connections", "outbound_settings"} { + if !tableExists(t, database, name) { + t.Fatalf("schema initialization dropped legacy table %s", name) + } + } +} + +func tableExists(t *testing.T, database *sql.DB, name string) bool { + t.Helper() + var count int + if err := database.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?", name).Scan(&count); err != nil { + t.Fatal(err) + } + return count != 0 +} diff --git a/service/db/setOp.go b/service/db/setOp.go index fd05f059..faa5d061 100644 --- a/service/db/setOp.go +++ b/service/db/setOp.go @@ -41,10 +41,6 @@ func SetRemove(bucket string, key string, val interface{}) (err error) { if !ok { return fmt.Errorf("SetRemove: outbound name must be a string") } - // Delete related connections and settings first - _, _ = db.Exec("DELETE FROM outbound_connections WHERE outbound_name = ?", name) - _, _ = db.Exec("DELETE FROM outbound_settings WHERE outbound_name = ?", name) - // Delete the outbound name _, err = db.Exec("DELETE FROM outbound_names WHERE name = ?", name) return err diff --git a/service/pre_database.go b/service/pre_database.go index 73b64a58..3a0390fb 100644 --- a/service/pre_database.go +++ b/service/pre_database.go @@ -16,11 +16,9 @@ import ( "github.com/v2rayA/v2rayA/kernel/v2ray/asset" "github.com/v2rayA/v2rayA/kernel/v2ray/asset/dat" "github.com/v2rayA/v2rayA/kernel/v2ray/where" - "github.com/v2rayA/v2rayA/pkg/util/copyfile" "github.com/v2rayA/v2rayA/pkg/util/log" "github.com/v2rayA/v2rayA/server/service" - confv4 "github.com/v2rayA/v2rayA-lib4/conf" touchv4 "github.com/v2rayA/v2rayA-lib4/core/touch" configurev4 "github.com/v2rayA/v2rayA-lib4/db/configure" servicev4 "github.com/v2rayA/v2rayA-lib4/server/service" @@ -92,19 +90,9 @@ func initConfigure() { // to create that directory and a boltv4.db file there. // Skip the v4 migration check entirely on Windows. if runtime.GOOS != "windows" && !configurev4.IsConfigureNotExists() { - // There is different format in server and subscription. - // So we keep other content and reimport servers and subscriptions. + // The v4 readers use boltv4.db directly. A bolt.db copy would + // make the next Open return ErrNeedMigration after SQLite exists. log.Warn("Migrating from v4 to main") - if err := copyfile.CopyFileContent(filepath.Join( - confv4.GetEnvironmentConfig().Config, - "boltv4.db", - ), filepath.Join( - conf.GetEnvironmentConfig().Config, - "bolt.db", - )); err != nil { - log.Fatal("Failed to copy boltv4.db to bolt.db: %v", err) - } - // clear connects of outbounds for _, out := range configure.GetOutbounds() { _ = configure.ClearConnects(out) diff --git a/service/server/service/subscription.go b/service/server/service/subscription.go index e9469386..9bf0d5da 100644 --- a/service/server/service/subscription.go +++ b/service/server/service/subscription.go @@ -362,9 +362,6 @@ func UpdateSubscription(index int, disconnectIfNecessary bool) (err error) { } } } - if err := configure.OverwriteConnects(configure.NewWhiches(cssAfter)); err != nil { - return err - } subscription = configure.GetSubscription(index) if subscription == nil { return common.Coded("SUBSCRIPTION_NOT_FOUND", fmt.Errorf("subscription #%d no longer exists; reload the page", index+1), map[string]interface{}{"id": index + 1}) @@ -372,7 +369,7 @@ func UpdateSubscription(index int, disconnectIfNecessary bool) (err error) { subscription.Servers = infoServerRaws subscription.Status = string(touch.NewUpdateStatus()) subscription.Info = status - if err := configure.SetSubscription(index, subscription); err != nil { + if err := configure.SetSubscriptionAndConnects(index, subscription, configure.NewWhiches(cssAfter)); err != nil { return err } // A remapped connection may point at a server whose config differs from the old