diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 6d297f05..c880c0ee 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -12,6 +12,7 @@ - [Database setup](./setup/database.md) - [Homeserver configuration](./setup/homeserver.md) - [Configuring a reverse proxy](./setup/reverse-proxy.md) +- [Configuring .well-known](./setup/well-known.md) - [Running the service](./setup/running.md) - [Migrating an existing homeserver](./setup/migration.md) # Usage diff --git a/docs/setup/migration.md b/docs/setup/migration.md index ae9d5ea9..2439f791 100644 --- a/docs/setup/migration.md +++ b/docs/setup/migration.md @@ -12,4 +12,67 @@ Features that are provided to support this include: - Ability to import existing upstream IdP subject ID mappings - Provides a compatibility layer for legacy Matrix authentication +If There will be tools to help with the migration process itself. But these aren't quite ready yet. + +## Preparation for the migration + +The deployment is non-trivial so it is important to read through and understand the steps involved and make a plan before starting. + +### Run the migration advisor + +You can use the advisor mode of the `syn2mas` tool to identify extra configuration steps or issues with the configuration of the homeserver. + +```sh +syn2mas --command=advisor --synapseConfigFile=homeserver.yaml +``` + +This will output `WARN` entries for any identified actions and `ERROR` entries in the case of any issues that will prevent the migration from working. + +### Install and configure MAS alongside your existing homeserver + +Follow the instructions in the [installation guide](installation.md) to install MAS alongside your existing homeserver. + +### Map any upstream SSO providers + +If you are using an upstream SSO provider then you will need to provision the upstream provide in MAS manually. + +Each upstream provider will need to be given as an `--upstreamProviderMapping` command line option to the import tool. + +### Do a dry-run of the import to test + +```sh +syn2mas --command migrate --synapseConfigFile homeserver.yaml --masConfigFile config.yaml --dryRun +``` + +If no errors are reported then you can proceed to the next step. + +## Doing the migration + +### Backup your data + +As with any migration, it is important to backup your data before proceeding. + +### Shutdown the homeserver + +This is to ensure that no new sessions are created whilst the migration is in progress. + +### Configure the homeserver + +Follow the instructions in the [homeserver configuration guide](homeserver.md) to configure the homeserver to use MAS. + +### Do the import + +Run `syn2mas` in non-dry-run mode. + +```sh +syn2mas --command migrate --synapseConfigFile homeserver.yaml --masConfigFile config.yaml --dryRun false +``` + +### Start up the homeserver + +Start up the homeserver again with the new configuration. + +### Update or serve the .well-known + +The `.well-known/matrix/client` needs to be served as described [here](./well-known.md). diff --git a/docs/setup/well-known.md b/docs/setup/well-known.md new file mode 100644 index 00000000..7082a53c --- /dev/null +++ b/docs/setup/well-known.md @@ -0,0 +1,23 @@ +# .well-known configuration + +A `.well-known/matrix/client` file is required to be served to allow clients to discover the authentication service. + +If no `.well-known/matrix/client` file is served currently then this will need to be enabled. + +If the homeserver is Synapse and serving this file already then the correct values will already be included when the homeserver is [configured to use MAS](./homeserver.md). + +If the .well-known is hosted elsewhere then `org.matrix.msc2965.authentication` entries need to be included similar to the following: + +```json +{ + "m.homeserver": { + "base_url": "https://matrix.example.com" + }, + "org.matrix.msc2965.authentication": { + "issuer": "https://example.com/", + "account": "https://auth.example.com/account" + } +} +``` + +For more context on what the correct values are, see [here](./README.md). diff --git a/tools/syn2mas/src/advisor.mts b/tools/syn2mas/src/advisor.mts index 3778b29a..34d3beed 100644 --- a/tools/syn2mas/src/advisor.mts +++ b/tools/syn2mas/src/advisor.mts @@ -119,7 +119,7 @@ export async function advisor(argv?: string[]): Promise { if (synapseConfig.password_config?.enabled !== false && synapseConfig.password_config?.localdb_enabled === false) { warn("Synapse has a non-standard password auth enabled which won't work after migration and will need to be manually mapped to an upstream OpenID Provider during migration"); } else if (synapseConfig.password_config?.enabled !== false) { - warn("Migration of Synapse password auth is not yet supported"); + warn("Synapse has password auth enabled, but support for password auth in MAS is not feature complete"); } const externalIdAuthProviders = await synapse.select("auth_provider").count("* as Count").from("user_external_ids").groupBy("auth_provider") as { auth_provider: string; "Count": number }[]; diff --git a/tools/syn2mas/src/index.ts b/tools/syn2mas/src/index.ts index 74ced292..0acb727a 100644 --- a/tools/syn2mas/src/index.ts +++ b/tools/syn2mas/src/index.ts @@ -1,7 +1,7 @@ import { ArgumentConfig, parse } from "ts-command-line-args"; import log4js from "log4js"; -// import { migrate } from "./migrate.mjs"; +import { migrate } from "./migrate.mjs"; import { advisor } from "./advisor.mjs"; log4js.configure({ @@ -28,10 +28,10 @@ const mainArgOptions: ArgumentConfig = { export const mainArgs = parse(mainArgOptions, { stopAtFirstUnknown: true }); try { - // if (mainArgs.command === "migrate") { - // await migrate(); - // process.exit(0); - // } + if (mainArgs.command === "migrate") { + await migrate(); + process.exit(0); + } if (mainArgs.command === "advisor") { await advisor(); diff --git a/tools/syn2mas/src/migrate.mts b/tools/syn2mas/src/migrate.mts new file mode 100644 index 00000000..2fa2dcd8 --- /dev/null +++ b/tools/syn2mas/src/migrate.mts @@ -0,0 +1,280 @@ +import { readFile } from "node:fs/promises"; +import { parse } from "ts-command-line-args"; +import id128 from "id128"; +import log4js from "log4js"; +import yaml from "yaml"; + +import { SUser } from "./types/SUser"; +import { SUserThreePid } from "./types/SUserThreePid"; +import { MUserPassword } from "./types/MUserPassword"; +import { MUserEmail } from "./types/MUserEmail"; +import { SUserExternalId } from "./types/SUserExternalId"; +import { SAccessToken } from "./types/SAccessToken"; +import { SRefreshToken } from "./types/SRefreshToken"; +import { MCompatAccessToken } from "./types/MCompatAccessToken"; +import { MCompatRefreshToken } from "./types/MCompatRefreshToken"; +import { MCompatSession } from "./types/MCompatSession"; +import { MUpstreamOauthLink } from "./types/MUpstreamOauthLink"; +import { MUpstreamOauthProvider } from "./types/MUpstreamOauthProvider"; +import { UUID } from "./types"; +import { SynapseConfig } from "./types/SynapseConfig"; +import { MASConfig } from "./types/MASConfig"; +import { connectToSynapseDatabase, connectToMASDatabase } from "./db.mjs"; + +const log = log4js.getLogger("migrate"); + +interface MigrationOptions { + command: string; + synapseConfigFile: string; + masConfigFile: string; + upstreamProviderMapping: string[]; + dryRun?: boolean; + help?: boolean; +} + +export async function migrate(argv?: string[]): Promise { + const args = parse({ + command: { type: String, description: "Command to run", defaultOption: true, typeLabel: "migrate" }, + synapseConfigFile: { type: String, description: "Path to synapse homeserver.yaml config file" }, + masConfigFile: { type: String, description: "Path to MAS config.yaml" }, + upstreamProviderMapping: { type: String, defaultValue: [], multiple: true, description: "Mapping of upstream provider IDs to MAS provider IDs. Format: :" }, + dryRun: { type: Boolean, optional: true, defaultValue: false }, + help: { type: Boolean, optional: true, alias: "h", description: "Prints this usage guide" }, + }, + { + helpArg: "help", + argv, + }); + + const warnings: string[] = []; + function warn(message: string): void { + warnings.push(message); + } + + let fatals = 0; + function fatal(message: string): void { + log.fatal(message); + warnings.forEach((w) => log.warn(w)); + if (!args.dryRun) { + process.exit(1); + } + fatals += 1; + } + + function makeUuid(): UUID { + return id128.Uuid4.fromRaw(id128.UlidMonotonic.generate().toRaw()).toCanonical(); + } + + // load synapse config + const synapseConfig: SynapseConfig = yaml.parse(await readFile(args.synapseConfigFile, "utf8")); + + // connect to synapse databases + const synapse = connectToSynapseDatabase(synapseConfig); + + // load MAS config + const masConfig: MASConfig = yaml.parse(await readFile(args.masConfigFile, "utf8")); + + if (!masConfig.database?.uri) { + log.fatal("Missing database URI in MAS config"); + process.exit(1); + } + const mas = connectToMASDatabase(masConfig); + + const upstreamProviders = new Map(); + + for (const mapping of args.upstreamProviderMapping) { + const [providerId, masProviderId] = mapping.split(":"); + if (!id128.Uuid.isRaw(masProviderId) && !id128.Uuid.isCanonical(masProviderId)) { + throw new Error(`Upstream provider mapping UUID is not in correct format. It should be a UUID: ${masProviderId}`); + } + log.info(`Loading existing upstream provider ${masProviderId} from MAS database as ${providerId}`); + const existingProvider = await mas("upstream_oauth_providers").select("*").where({ upstream_oauth_provider_id: masProviderId }).first(); + if (!existingProvider) { + throw new Error(`Could not find upstream provider ${masProviderId} in MAS database`); + } + upstreamProviders.set(providerId, existingProvider); + } + + function stringifyAndRedact(input: any): string { + const x = JSON.stringify(input); + + return x.replace(/("(password_hash|hashed_password|access_token|token)":")[^"]*"/, "$1redacted\""); + } + + type Execution = (() => Promise); + + const existingMasUsers = await mas.count({ count: "*" }).from("users").first(); + + if (parseInt(`${existingMasUsers?.count ?? 0}`) > 0) { + fatal(`Found ${existingMasUsers?.count} existing users in MAS. Refusing to continue. Please clean MAS and try again.`); + } + + const synapseUsers = await synapse.select("*").from("users"); + log.info(`Found ${synapseUsers.length} users in Synapse`); + for (const user of synapseUsers) { + const localpart = user.name.split(":")[0].substring(1); + log.info(`Processing user ${user.name} as ${localpart}`); + + let warningsForUser = 0; + const executions: Execution[] = []; + + if (user.deactivated === 1) { + fatal(`Migration of deactivated users is not supported: ${user.name}`); + } + + if (user.is_guest === 1) { + fatal(`Migration of guest users is not supported: ${user.name}`); + } + + // users => users + const masUser = { + user_id: makeUuid(), + username: localpart, + created_at: new Date(parseInt(`${user.creation_ts}`) * 1000), + }; + executions.push(() => mas.insert(masUser!).into("users")); + log.debug(`${stringifyAndRedact(user)} => ${stringifyAndRedact(masUser)}`); + // users.password_hash => user_passwords + if (user.password_hash) { + const masUserPassword: MUserPassword = { + user_password_id: makeUuid(), + user_id: masUser.user_id, + hashed_password: user.password_hash, + created_at: masUser.created_at, // TODO: should we use now() instead of created_at? + version: 1, + }; + + log.debug(`Password ${user.password_hash.slice(-4)} => ${stringifyAndRedact(masUserPassword)}`); + executions.push(() => mas.insert(masUserPassword).into("user_passwords")); + } + + // user_threepids => user_emails + let primaryEmail: MUserEmail | undefined; + const synapseThreePids = await synapse.select("*").from("user_threepids").where({ user_id: user.name }); + for (const threePid of synapseThreePids) { + if (threePid.medium !== "email") { + warningsForUser += 1; + warn(`Skipping non-email 3pid ${threePid.medium} for user ${user.name}`); + continue; + } + const masUserEmail: MUserEmail = { + user_email_id: makeUuid(), + user_id: masUser.user_id, + email: threePid.address.toLowerCase(), + created_at: new Date(parseInt(`${threePid.added_at}`) * 1000), + confirmed_at: threePid.validated_at ? new Date(parseInt(`${threePid.validated_at}`) * 1000) : undefined, + }; + + log.debug(`${stringifyAndRedact(threePid)} => ${stringifyAndRedact(masUserEmail)}`); + if (!primaryEmail && threePid.validated_at) { + primaryEmail = masUserEmail; + } + executions.push(() => mas.insert(masUserEmail).into("user_emails")); + } + if (primaryEmail) { + log.debug(`Setting primary email for existing user ${masUser.username} to ${primaryEmail.email} as update`); + executions.push(() => mas("users").where({ user_id: masUser!.user_id }).update({ primary_user_email_id: primaryEmail!.user_email_id })); + } + + // user_external_ids => upstream_oauth_links + const synapseExternalIds = await synapse.select("*").from("user_external_ids").where({ user_id: user.name }); + for (const externalId of synapseExternalIds) { + try { + if (!upstreamProviders.has(externalId.auth_provider)) { + throw new Error(`Unknown upstream provider ${externalId.auth_provider}`); + } + const provider = upstreamProviders.get(externalId.auth_provider)!; + const masUpstreamOauthLink: MUpstreamOauthLink = { + upstream_oauth_link_id: makeUuid(), + user_id: masUser.user_id, + upstream_oauth_provider_id: provider.upstream_oauth_provider_id, + subject: externalId.external_id, + created_at: masUser.created_at, + }; + + log.debug(`${stringifyAndRedact(synapseExternalIds)} => ${stringifyAndRedact(masUpstreamOauthLink)}`); + + executions.push(() => mas.insert(masUpstreamOauthLink).into("upstream_oauth_links")); + } catch (e) { + fatal(`Failed to import external id ${externalId.external_id} with ${externalId.auth_provider} for user ${user.name}: ${e}`); + } + } + + // access_tokens,refresh_tokens => compat_sessions,compat_access_tokens + const synapseAccessTokens = await synapse.select("*").from("access_tokens").where({ user_id: user.name }); + for (const accessToken of synapseAccessTokens) { + if (!accessToken.device_id) { + warningsForUser += 1; + warn(`Skipping access token ${accessToken.token} for user ${user.name} with no device_id`); + continue; + } + + const masCompatSession: MCompatSession = { + compat_session_id: makeUuid(), + user_id: masUser.user_id, + device_id: accessToken.device_id, + created_at: accessToken.last_validated ? new Date(parseInt(`${accessToken.last_validated}`)) : masUser.created_at, + is_synapse_admin: user.admin === 1, + }; + log.debug(`${stringifyAndRedact(accessToken)} => ${stringifyAndRedact(masCompatSession)}`); + executions.push(() => mas.insert(masCompatSession).into("compat_sessions")); + + const masCompatAccessToken: MCompatAccessToken = { + compat_access_token_id: makeUuid(), + compat_session_id: masCompatSession.compat_session_id, + access_token: accessToken.token, + created_at: masCompatSession.created_at, + }; + log.debug(`Access token ${accessToken.id} => ${stringifyAndRedact(masCompatAccessToken)}`); + executions.push(() => mas.insert(masCompatAccessToken).into("compat_access_tokens")); + + if (accessToken.refresh_token_id) { + const synapseRefreshToken = await synapse.select("*").from("refresh_tokens").where({ id: accessToken.refresh_token_id }).first(); + if (synapseRefreshToken) { + const masCompatRefreshToken: MCompatRefreshToken = { + compat_refresh_token_id: makeUuid(), + compat_session_id: masCompatSession.compat_session_id, + compat_access_token_id: masCompatAccessToken.compat_access_token_id, + refresh_token: synapseRefreshToken.token, + created_at: masCompatSession.created_at, + }; + log.debug(`Refresh token ${synapseRefreshToken.id} => ${stringifyAndRedact(masCompatRefreshToken)}`); + executions.push(() => mas.insert(masCompatRefreshToken).into("compat_refresh_tokens")); + } else { + warningsForUser += 1; + warn(`Unable to locate refresh token ${accessToken.refresh_token_id} for user ${user.name}`); + } + } + } + + if (warningsForUser > 0) { + if (!args.dryRun) { + fatal(`User ${user.name} had ${warningsForUser} warnings`); + } else { + log.warn(`User ${user.name} had ${warningsForUser} warnings`); + } + } else if (!args.dryRun) { + log.info(`Running ${executions.length} updates for user ${user.name}`); + const tx = await mas.transaction(); + try { + for (const execution of executions) { + await execution(); + } + await tx.commit(); + log.info(`Migrated user ${user.name}`); + } catch (e) { + try { + await tx.rollback(); + } catch (e2) { + log.error(`Failed to rollback transaction: ${e2}`); + } + throw e; + } + } + } + log.info(`Completed migration ${args.dryRun ? "dry-run " : ""}of ${synapseUsers.length} users with ${fatals} fatals and ${warnings.length} warnings:`); + warnings.forEach((w) => log.warn(w)); + if (fatals > 0) { + throw new Error(`Migration failed with ${fatals} fatals`); + } +}