From 8a0ddc43abba6417db6993e4d30237ba21faa8ff Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 21 Aug 2019 14:08:46 -0600 Subject: [PATCH 1/6] Use the v2 (hashed) lookup for identity server queries Fixes https://github.com/vector-im/riot-web/issues/10556 Implements [MSC2134](https://github.com/matrix-org/matrix-doc/pull/2134) with assistance from [MSC2140](https://github.com/matrix-org/matrix-doc/pull/2140) for auth. Note: this also changes all identity server requests to use JSON as a request body. URL encoded forms were allowed in v1 but deprecated in favour of JSON. v2 APIs do not allow URL encoded forms. --- src/base-apis.js | 161 +++++++++++++++++++++++++++++++++++++++++------ src/http-api.js | 2 +- 2 files changed, 144 insertions(+), 19 deletions(-) diff --git a/src/base-apis.js b/src/base-apis.js index 3e9a9bc74..028e14795 100644 --- a/src/base-apis.js +++ b/src/base-apis.js @@ -1857,6 +1857,93 @@ MatrixBaseApis.prototype.submitMsisdnToken = async function( } }; +/** + * Gets the V2 hashing information from the identity server. Primarily useful for + * lookups. + * @param identityAccessToken The access token for the identity server. + * @returns {Promise} The hashing information for the identity server. + */ +MatrixBaseApis.prototype.getIdentityHashDetails = function(identityAccessToken) { + return this._http.idServerRequest( + undefined, "GET", "/hash_details", + null, httpApi.PREFIX_IDENTITY_V2, identityAccessToken, + ); +}; + +/** + * Performs a hashed lookup of addresses against the identity server. This is + * only supported on identity servers which have at least the version 2 API. + * @param addressPairs An array of 2 element arrays. The first element of each + * pair is the address, the second is the 3PID medium. Eg: ["email@example.org", + * "email"] + * @param identityAccessToken The access token for the identity server. + * @returns {Promise<{address, mxid}[]>} A collection of address mappings to + * found MXIDs. Results where no user could be found will not be listed. + */ +MatrixBaseApis.prototype.identityHashedLookup = async function( + addressPairs, // [["email@example.org", "email"], ["10005550000", "msisdn"]] + identityAccessToken, +) { + const params = { + // addresses: ["email@example.org", "10005550000"], + // algorithm: "sha256", + // pepper: "abc123" + }; + + // Get hash information first before trying to do a lookup + const hashes = await this.getIdentityHashDetails(); + if (!hashes || !hashes['lookup_pepper'] || !hashes['algorithms']) { + throw new Error("Unsupported identity server: bad response"); + } + + params['pepper'] = hashes['lookup_pepper']; + + const localMapping = { + // hashed identifier => plain text address + // For use in this function's return format + }; + + // When picking an algorithm, we pick the hashed over no hashes + if (hashes['algorithms'].includes('sha256')) { + // Abuse the olm hashing + const olmutil = new global.Olm.Utility(); + params["addresses"] = addressPairs.map(p => { + const hashed = olmutil.sha256(`${p[0]} ${p[1]} ${params['pepper']}`); + localMapping[hashed] = p[0]; + return hashed; + }); + params["algorithm"] = "sha256"; + } else if (hashes['algorithms'].includes('none')) { + params["addresses"] = addressPairs.map(p => { + const unhashed = `${p[0]} ${p[1]}`; + localMapping[unhashed] = p[0]; + return unhashed; + }); + params["algorithm"] = "none"; + } else { + throw new Error("Unsupported identity server: unknown hash algorithm"); + } + + const response = await this._http.idServerRequest( + undefined, "POST", "/lookup", + params, httpApi.PREFIX_IDENTITY_V2, identityAccessToken, + ); + + if (!response || !response['mappings']) return []; // no results + + const foundAddresses = [/* {address: "plain@example.org", mxid} */]; + for (const hashed of Object.keys(response['mappings'])) { + const mxid = response['mappings'][hashed]; + const plainAddress = localMapping[hashed]; + if (!plainAddress) { + throw new Error("Identity server returned more results than expected"); + } + + foundAddresses.push({address: plainAddress, mxid}); + } + return foundAddresses; +}; + /** * Looks up the public Matrix ID mapping for a given 3rd party * identifier from the Identity Server @@ -1878,31 +1965,51 @@ MatrixBaseApis.prototype.lookupThreePid = async function( callback, identityAccessToken, ) { - const params = { - medium: medium, - address: address, - }; - try { - const response = await this._http.idServerRequest( - undefined, "GET", "/lookup", - params, httpApi.PREFIX_IDENTITY_V2, identityAccessToken, + // Note: we're using the V2 API by calling this function, but our + // function contract requires a V1 response. We therefore have to + // convert it manually. + const response = await this.identityHashedLookup( + [[address, medium]], identityAccessToken, ); + const result = response.find(p => p.address === address); + if (!result) { + // TODO: Fold callback into above call once v1 path below is removed + if (callback) callback(null, {}); + return {}; + } + + const mapping = { + address, + medium, + mxid: result.mxid, + + // We can't reasonably fill these parameters: + // not_before + // not_after + // ts + // signatures + }; + // TODO: Fold callback into above call once v1 path below is removed - if (callback) callback(null, response); - return response; + if (callback) callback(null, mapping); + return mapping; } catch (err) { if (err.cors === "rejected" || err.httpStatus === 404) { // Fall back to deprecated v1 API for now // TODO: Remove this path once v2 is only supported version // See https://github.com/vector-im/riot-web/issues/10443 + const params = { + medium: medium, + address: address, + }; logger.warn("IS doesn't support v2, falling back to deprecated v1"); return await this._http.idServerRequest( callback, "GET", "/lookup", params, httpApi.PREFIX_IDENTITY_V1, ); } - if (callback) callback(err); + if (callback) callback(err, undefined); throw err; } }; @@ -1922,20 +2029,38 @@ MatrixBaseApis.prototype.bulkLookupThreePids = async function( query, identityAccessToken, ) { - const params = { - threepids: query, - }; - try { - return await this._http.idServerRequest( - undefined, "POST", "/bulk_lookup", JSON.stringify(params), - httpApi.PREFIX_IDENTITY_V2, identityAccessToken, + // Note: we're using the V2 API by calling this function, but our + // function contract requires a V1 response. We therefore have to + // convert it manually. + const response = await this.identityHashedLookup( + // We have to reverse the query order to get [address, medium] pairs + query.map(p => [p[1], p[0]]), identityAccessToken, ); + + const v1results = []; + for (const mapping of response) { + const originalQuery = query.find(p => p[1] === mapping.address); + if (!originalQuery) { + throw new Error("Identity sever returned unexpected results"); + } + + v1results.push([ + originalQuery[0], // medium + mapping.address, + mapping.mxid, + ]); + } + + return {threepids: v1results}; } catch (err) { if (err.cors === "rejected" || err.httpStatus === 404) { // Fall back to deprecated v1 API for now // TODO: Remove this path once v2 is only supported version // See https://github.com/vector-im/riot-web/issues/10443 + const params = { + threepids: query, + }; logger.warn("IS doesn't support v2, falling back to deprecated v1"); return await this._http.idServerRequest( undefined, "POST", "/bulk_lookup", JSON.stringify(params), diff --git a/src/http-api.js b/src/http-api.js index 2b083a3e5..76990c052 100644 --- a/src/http-api.js +++ b/src/http-api.js @@ -412,7 +412,7 @@ module.exports.MatrixHttpApi.prototype = { if (method == 'GET') { opts.qs = params; } else if (typeof params === "object") { - opts.form = params; + opts.json = params; } else if (typeof params === "string") { // Assume the caller has serialised the body to JSON opts.body = params; From 241811298f69ad9dbbe7f8f7b0b648ebbc994044 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 21 Aug 2019 14:17:55 -0600 Subject: [PATCH 2/6] Appease the js-doc --- src/base-apis.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/base-apis.js b/src/base-apis.js index 028e14795..52fbcd937 100644 --- a/src/base-apis.js +++ b/src/base-apis.js @@ -1860,7 +1860,7 @@ MatrixBaseApis.prototype.submitMsisdnToken = async function( /** * Gets the V2 hashing information from the identity server. Primarily useful for * lookups. - * @param identityAccessToken The access token for the identity server. + * @param {string} identityAccessToken The access token for the identity server. * @returns {Promise} The hashing information for the identity server. */ MatrixBaseApis.prototype.getIdentityHashDetails = function(identityAccessToken) { @@ -1873,10 +1873,10 @@ MatrixBaseApis.prototype.getIdentityHashDetails = function(identityAccessToken) /** * Performs a hashed lookup of addresses against the identity server. This is * only supported on identity servers which have at least the version 2 API. - * @param addressPairs An array of 2 element arrays. The first element of each - * pair is the address, the second is the 3PID medium. Eg: ["email@example.org", - * "email"] - * @param identityAccessToken The access token for the identity server. + * @param {Array>} addressPairs An array of 2 element arrays. + * The first element of each pair is the address, the second is the 3PID medium. + * Eg: ["email@example.org", "email"] + * @param {string} identityAccessToken The access token for the identity server. * @returns {Promise<{address, mxid}[]>} A collection of address mappings to * found MXIDs. Results where no user could be found will not be listed. */ From ba78d1a9ae80b9d8fed2783031c4a7a3d032ce31 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 21 Aug 2019 14:30:04 -0600 Subject: [PATCH 3/6] JSON is JSON --- src/http-api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/http-api.js b/src/http-api.js index 76990c052..0b2f68a25 100644 --- a/src/http-api.js +++ b/src/http-api.js @@ -431,7 +431,7 @@ module.exports.MatrixHttpApi.prototype = { // option as we do with the home server, but it does return JSON, so // parse it manually return defer.promise.then(function(response) { - return JSON.parse(response); + return typeof(response) === 'string' ? JSON.parse(response) : response; }); }, From 3d5a79be3b707b9618af44466cf87b44e8bbb167 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 21 Aug 2019 14:30:16 -0600 Subject: [PATCH 4/6] Hashes need tokens too --- src/base-apis.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/base-apis.js b/src/base-apis.js index 52fbcd937..f1930d061 100644 --- a/src/base-apis.js +++ b/src/base-apis.js @@ -1891,7 +1891,7 @@ MatrixBaseApis.prototype.identityHashedLookup = async function( }; // Get hash information first before trying to do a lookup - const hashes = await this.getIdentityHashDetails(); + const hashes = await this.getIdentityHashDetails(identityAccessToken); if (!hashes || !hashes['lookup_pepper'] || !hashes['algorithms']) { throw new Error("Unsupported identity server: bad response"); } From b306df726a7b6ea4dc40765620be5c1905d1f6b6 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 21 Aug 2019 14:30:24 -0600 Subject: [PATCH 5/6] Lookups are URL safe --- src/base-apis.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/base-apis.js b/src/base-apis.js index f1930d061..670993638 100644 --- a/src/base-apis.js +++ b/src/base-apis.js @@ -1908,7 +1908,8 @@ MatrixBaseApis.prototype.identityHashedLookup = async function( // Abuse the olm hashing const olmutil = new global.Olm.Utility(); params["addresses"] = addressPairs.map(p => { - const hashed = olmutil.sha256(`${p[0]} ${p[1]} ${params['pepper']}`); + const hashed = olmutil.sha256(`${p[0]} ${p[1]} ${params['pepper']}`) + .replace(/\+/g, '-').replace(/\//g, '_'); // URL-safe base64 localMapping[hashed] = p[0]; return hashed; }); From 3980b62df2df96a549682f2ec3904439cf3d00a3 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 21 Aug 2019 14:32:30 -0600 Subject: [PATCH 6/6] js-doc syntax is unknown to our js-doc --- src/base-apis.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/base-apis.js b/src/base-apis.js index 670993638..8d3d2248b 100644 --- a/src/base-apis.js +++ b/src/base-apis.js @@ -1877,7 +1877,7 @@ MatrixBaseApis.prototype.getIdentityHashDetails = function(identityAccessToken) * The first element of each pair is the address, the second is the 3PID medium. * Eg: ["email@example.org", "email"] * @param {string} identityAccessToken The access token for the identity server. - * @returns {Promise<{address, mxid}[]>} A collection of address mappings to + * @returns {Promise>} A collection of address mappings to * found MXIDs. Results where no user could be found will not be listed. */ MatrixBaseApis.prototype.identityHashedLookup = async function(