diff --git a/spec/test-utils/webrtcReports.ts b/spec/test-utils/webrtcReports.ts new file mode 100644 index 000000000..c34a28f40 --- /dev/null +++ b/spec/test-utils/webrtcReports.ts @@ -0,0 +1,1114 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +/** + * Here you can find two example Webrtc reports and use them for testing. + * The reports were generated with Chrome/Mac and `RTCPeerConnection: getStats()` method. + * Webrtc stats change over time and people are often interested in time-dependent statements. + * That's why there are two reports (`prevChromeReport`, `currentChromeReport`) here that were recorded 10 seconds apart. + * + * Notice: + * Because RTCStatsReport is more than just a data object, I left out the cast of the reports to RTCStatsReport. + * With other words, I didn't want cover the behavior of RTCStatsReport here. + */ +export const prevChromeReport = [ + { + id: "AP", + timestamp: 1685442202456.655, + type: "media-playout", + kind: "audio", + synthesizedSamplesDuration: 0, + synthesizedSamplesEvents: 0, + totalPlayoutDelay: 11192.89248, + totalSamplesCount: 477600, + totalSamplesDuration: 9.95, + }, + { + id: "CF8D:03:B2:A8:7F:FE:52:60:98:42:3F:F1:A3:61:89:CD:B2:39:0E:17:F7:AE:57:79:5F:96:6F:41:E8:DA:CB:2D", + timestamp: 1685442202456.655, + type: "certificate", + base64Certificate: + "MIIBFjCBvKADAgECAggGFX25uzXk/jAKBggqhkjOPQQDAjARMQ8wDQYDVQQDDAZXZWJSVEMwHhcNMjMwNTI5MTAyMzEyWhcNMjMwNjI5MTAyMzEyWjARMQ8wDQYDVQQDDAZXZWJSVEMwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATICxbNzVNS8Z+SEntEKIYTryX5Ib0Fglm9Z8+nqsgiLOPOKPz+tc/dzE/55de4VQXJIJrO+wJ/c+JCOfp3sqlSMAoGCCqGSM49BAMCA0kAMEYCIQCZ95K1ot41YP/3Q4cURKDjHcYkBcAVkHBupmnWxVY+LQIhAKQwE43fZLYiEBG+AvFjj2sicilsEZ6r71E61YYZmYqz", + fingerprint: "8D:03:B2:A8:7F:FE:52:60:98:42:3F:F1:A3:61:89:CD:B2:39:0E:17:F7:AE:57:79:5F:96:6F:41:E8:DA:CB:2D", + fingerprintAlgorithm: "sha-256", + }, + { + id: "CFCE:A3:2C:36:33:68:11:31:AB:DC:67:BE:3C:7D:03:00:F5:73:BC:09:23:72:F1:5D:21:F8:54:58:2C:FC:3E:74", + timestamp: 1685442202456.655, + type: "certificate", + base64Certificate: + "MIIBFjCBvKADAgECAghUitPBs0RuVDAKBggqhkjOPQQDAjARMQ8wDQYDVQQDDAZXZWJSVEMwHhcNMjMwNTI5MTAyMzEyWhcNMjMwNjI5MTAyMzEyWjARMQ8wDQYDVQQDDAZXZWJSVEMwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAASY7R/n7WzMx3v6bhwLpuv+XhtRAAkFOTnruyGc3oeh04OqoriE9FQ4sHm5rqD8vGah8eqHfN6g1XDgA1yEC3rnMAoGCCqGSM49BAMCA0kAMEYCIQCfgRuep47juMTV+9yUKbuqTklkbDfZ0vLw4+ySKbaWGAIhAPqYvacUo7jmH1DV7sl9mLNeTpQqpqbqsF+lbuQ83qcM", + fingerprint: "CE:A3:2C:36:33:68:11:31:AB:DC:67:BE:3C:7D:03:00:F5:73:BC:09:23:72:F1:5D:21:F8:54:58:2C:FC:3E:74", + fingerprintAlgorithm: "sha-256", + }, + { + id: "CIT01_111_minptime=10;usedtx=1;useinbandfec=1", + timestamp: 1685442202456.655, + type: "codec", + channels: 2, + clockRate: 48000, + mimeType: "audio/opus", + payloadType: 111, + sdpFmtpLine: "minptime=10;usedtx=1;useinbandfec=1", + transportId: "T01", + }, + { + id: "CIT01_96", + timestamp: 1685442202456.655, + type: "codec", + clockRate: 90000, + mimeType: "video/VP8", + payloadType: 96, + transportId: "T01", + }, + { + id: "COT01_111_minptime=10;usedtx=1;useinbandfec=1", + timestamp: 1685442202456.655, + type: "codec", + channels: 2, + clockRate: 48000, + mimeType: "audio/opus", + payloadType: 111, + sdpFmtpLine: "minptime=10;usedtx=1;useinbandfec=1", + transportId: "T01", + }, + { + id: "COT01_96", + timestamp: 1685442202456.655, + type: "codec", + clockRate: 90000, + mimeType: "video/VP8", + payloadType: 96, + transportId: "T01", + }, + { + id: "CPE/y2OM79_6JRzkx+P", + timestamp: 1685442202456.655, + type: "candidate-pair", + bytesDiscardedOnSend: 0, + bytesReceived: 11236, + bytesSent: 26145, + consentRequestsSent: 2, + currentRoundTripTime: 0, + lastPacketReceivedTimestamp: 1685442193001, + lastPacketSentTimestamp: 1685442193019, + localCandidateId: "IE/y2OM79", + nominated: true, + packetsDiscardedOnSend: 0, + packetsReceived: 40, + packetsSent: 55, + priority: 9114756780654345000, + remoteCandidateId: "I6JRzkx+P", + requestsReceived: 3, + requestsSent: 3, + responsesReceived: 3, + responsesSent: 3, + state: "succeeded", + totalRoundTripTime: 0, + transportId: "T01", + writable: true, + }, + { + id: "CPE/y2OM79_fG99OI7Z", + timestamp: 1685442202456.655, + type: "candidate-pair", + bytesDiscardedOnSend: 0, + bytesReceived: 0, + bytesSent: 0, + consentRequestsSent: 0, + localCandidateId: "IE/y2OM79", + nominated: false, + packetsDiscardedOnSend: 0, + packetsReceived: 0, + packetsSent: 0, + priority: 9114475305677635000, + remoteCandidateId: "IfG99OI7Z", + requestsReceived: 1, + requestsSent: 0, + responsesReceived: 0, + responsesSent: 1, + state: "waiting", + totalRoundTripTime: 0, + transportId: "T01", + writable: false, + }, + { + id: "CPhz/Z/Pd/_6JRzkx+P", + timestamp: 1685442202456.655, + type: "candidate-pair", + bytesDiscardedOnSend: 0, + bytesReceived: 0, + bytesSent: 0, + consentRequestsSent: 0, + currentRoundTripTime: 0.001, + localCandidateId: "Ihz/Z/Pd/", + nominated: false, + packetsDiscardedOnSend: 0, + packetsReceived: 0, + packetsSent: 0, + priority: 9114475305677635000, + remoteCandidateId: "I6JRzkx+P", + requestsReceived: 0, + requestsSent: 1, + responsesReceived: 1, + responsesSent: 0, + state: "succeeded", + totalRoundTripTime: 0.001, + transportId: "T01", + writable: true, + }, + { + id: "CPhz/Z/Pd/_fG99OI7Z", + timestamp: 1685442202456.655, + type: "candidate-pair", + bytesDiscardedOnSend: 0, + bytesReceived: 0, + bytesSent: 0, + consentRequestsSent: 0, + localCandidateId: "Ihz/Z/Pd/", + nominated: false, + packetsDiscardedOnSend: 0, + packetsReceived: 0, + packetsSent: 0, + priority: 9114475305677503000, + remoteCandidateId: "IfG99OI7Z", + requestsReceived: 0, + requestsSent: 0, + responsesReceived: 0, + responsesSent: 0, + state: "waiting", + totalRoundTripTime: 0, + transportId: "T01", + writable: false, + }, + { + id: "CPryM2UC4o_8H+/O2yO", + timestamp: 1685442202456.655, + type: "candidate-pair", + availableOutgoingBitrate: 5332923, + bytesDiscardedOnSend: 0, + bytesReceived: 835094, + bytesSent: 1901967, + consentRequestsSent: 6, + currentRoundTripTime: 0.001, + lastPacketReceivedTimestamp: 1685442202448, + lastPacketSentTimestamp: 1685442202431, + localCandidateId: "IryM2UC4o", + nominated: true, + packetsDiscardedOnSend: 0, + packetsReceived: 1296, + packetsSent: 2238, + priority: 9115049250747470000, + remoteCandidateId: "I8H+/O2yO", + requestsReceived: 7, + requestsSent: 7, + responsesReceived: 7, + responsesSent: 7, + state: "succeeded", + totalRoundTripTime: 0.004, + transportId: "T01", + writable: true, + }, + { + id: "D12", + timestamp: 1685442202456.655, + type: "data-channel", + bytesReceived: 0, + bytesSent: 0, + dataChannelIdentifier: 1, + label: "datachannel", + messagesReceived: 0, + messagesSent: 0, + protocol: "", + state: "open", + }, + { + id: "I6JRzkx+P", + timestamp: 1685442202456.655, + type: "remote-candidate", + address: "192.168.64.1", + candidateType: "host", + foundation: "2620169396", + ip: "192.168.64.1", + isRemote: true, + port: 60959, + priority: 2122194687, + protocol: "udp", + transportId: "T01", + usernameFragment: "AUZN", + }, + { + id: "I8H+/O2yO", + timestamp: 1685442202456.655, + type: "remote-candidate", + address: "aaaa:1111:aaaa:111:aaaa:1111:1111:aaa", + candidateType: "host", + foundation: "3525121815", + ip: "aaaa:1111:aaaa:111:aaaa:1111:1111:aaa", + isRemote: true, + port: 50629, + priority: 2122262783, + protocol: "udp", + transportId: "T01", + usernameFragment: "AUZN", + }, + { + id: "IE/y2OM79", + timestamp: 1685442202456.655, + type: "local-candidate", + address: "192.168.64.1", + candidateType: "host", + foundation: "2979504628", + ip: "192.168.64.1", + isRemote: false, + networkType: "ethernet", + port: 62639, + priority: 2122194687, + protocol: "udp", + transportId: "T01", + usernameFragment: "zBQ8", + }, + { + id: "IT01A195186619", + timestamp: 1685442202456.655, + type: "inbound-rtp", + codecId: "CIT01_111_minptime=10;usedtx=1;useinbandfec=1", + kind: "audio", + mediaType: "audio", + ssrc: 195186619, + transportId: "T01", + jitter: 0, + packetsLost: 0, + packetsReceived: 272, + audioLevel: 0.0012207403790398877, + bytesReceived: 19670, + concealedSamples: 15360, + concealmentEvents: 1, + estimatedPlayoutTimestamp: 3894431002370, + fecPacketsDiscarded: 0, + fecPacketsReceived: 0, + headerBytesReceived: 7616, + insertedSamplesForDeceleration: 1309, + jitterBufferDelay: 7660.8, + jitterBufferEmittedCount: 261120, + jitterBufferMinimumDelay: 5702.4, + jitterBufferTargetDelay: 5856, + lastPacketReceivedTimestamp: 1685442202124, + mid: "0", + packetsDiscarded: 0, + playoutId: "AP", + remoteId: "ROA195186619", + removedSamplesForAcceleration: 0, + silentConcealedSamples: 13800, + totalAudioEnergy: 0.0001488620375037452, + totalSamplesDuration: 9.949999999999832, + totalSamplesReceived: 477600, + trackIdentifier: "f0e99682-e578-4ba4-a1f0-41a4b7ae7d01", + }, + { + id: "IT01V447914256", + timestamp: 1685442202456.655, + type: "inbound-rtp", + codecId: "CIT01_96", + kind: "video", + mediaType: "video", + ssrc: 447914256, + transportId: "T01", + jitter: 0.001, + packetsLost: 0, + packetsReceived: 832, + bytesReceived: 751998, + decoderImplementation: "libvpx", + estimatedPlayoutTimestamp: 3894431002387, + firCount: 0, + frameHeight: 270, + frameWidth: 480, + framesAssembledFromMultiplePackets: 277, + framesDecoded: 287, + framesDropped: 0, + framesPerSecond: 30, + framesReceived: 287, + freezeCount: 0, + googTimingFrameInfo: + "1014689708,12908278,12908292,12908294,12908294,12908294,12908278,12908278,12908294,12908294,12908298,12908299,12908309,0,1", + headerBytesReceived: 20804, + jitterBufferDelay: 6.355, + jitterBufferEmittedCount: 286, + keyFramesDecoded: 2, + lastPacketReceivedTimestamp: 1685442202412, + mid: "1", + nackCount: 0, + pauseCount: 0, + pliCount: 0, + powerEfficientDecoder: false, + qpSum: 3799, + totalAssemblyTime: 0.038318, + totalDecodeTime: 0.15875999999999998, + totalFreezesDuration: 0, + totalInterFrameDelay: 9.526, + totalPausesDuration: 0, + totalProcessingDelay: 2.0045129999999998, + totalSquaredInterFrameDelay: 0.325962, + trackIdentifier: "4c93cfe8-e3bc-4c27-8614-b8c895a49118", + }, + { + id: "IfG99OI7Z", + timestamp: 1685442202456.655, + type: "remote-candidate", + address: "192.168.178.71", + candidateType: "host", + foundation: "2532229949", + ip: "192.168.178.71", + isRemote: true, + port: 54661, + priority: 2122129151, + protocol: "udp", + transportId: "T01", + usernameFragment: "AUZN", + }, + { + id: "Ihz/Z/Pd/", + timestamp: 1685442202456.655, + type: "local-candidate", + address: "192.168.178.71", + candidateType: "host", + foundation: "3142975101", + ip: "192.168.178.71", + isRemote: false, + networkType: "ethernet", + port: 59368, + priority: 2122129151, + protocol: "udp", + transportId: "T01", + usernameFragment: "zBQ8", + }, + { + id: "IryM2UC4o", + timestamp: 1685442202456.655, + type: "local-candidate", + address: "aaaa:1111:aaaa:111:aaaa:1111:1111:aaa", + candidateType: "host", + foundation: "4289079895", + ip: "aaaa:1111:aaaa:111:aaaa:1111:1111:aaa", + isRemote: false, + networkType: "ethernet", + port: 65502, + priority: 2122262783, + protocol: "udp", + transportId: "T01", + usernameFragment: "zBQ8", + }, + { + id: "OT01A958266064", + timestamp: 1685442202456.655, + type: "outbound-rtp", + codecId: "COT01_111_minptime=10;usedtx=1;useinbandfec=1", + kind: "audio", + mediaType: "audio", + ssrc: 958266064, + transportId: "T01", + bytesSent: 17065, + packetsSent: 272, + active: true, + headerBytesSent: 7616, + mediaSourceId: "SA23", + mid: "0", + nackCount: 0, + remoteId: "RIA958266064", + retransmittedBytesSent: 0, + retransmittedPacketsSent: 0, + targetBitrate: 32000, + totalPacketSendDelay: 0, + }, + { + id: "OT01V474891051", + timestamp: 1685442202456.655, + type: "outbound-rtp", + codecId: "COT01_96", + kind: "video", + mediaType: "video", + ssrc: 474891051, + transportId: "T01", + bytesSent: 1820653, + packetsSent: 1821, + active: true, + encoderImplementation: "libvpx", + firCount: 0, + frameHeight: 360, + frameWidth: 640, + framesEncoded: 288, + framesPerSecond: 30, + framesSent: 288, + headerBytesSent: 50536, + hugeFramesSent: 0, + keyFramesEncoded: 1, + mediaSourceId: "SV24", + mid: "1", + nackCount: 0, + pliCount: 2, + powerEfficientEncoder: false, + qpSum: 3895, + qualityLimitationDurations: { + bandwidth: 0, + cpu: 0, + none: 9.956, + other: 0, + }, + qualityLimitationReason: "none", + qualityLimitationResolutionChanges: 0, + remoteId: "RIV474891051", + retransmittedBytesSent: 0, + retransmittedPacketsSent: 0, + scalabilityMode: "L1T1", + targetBitrate: 1700000, + totalEncodeTime: 0.786, + totalEncodedBytesTarget: 0, + totalPacketSendDelay: 0.08921899999999999, + }, + { + id: "P", + timestamp: 1685442202456.655, + type: "peer-connection", + dataChannelsClosed: 0, + dataChannelsOpened: 1, + }, + { + id: "RIA958266064", + timestamp: 1685442198537, + type: "remote-inbound-rtp", + codecId: "COT01_111_minptime=10;usedtx=1;useinbandfec=1", + kind: "audio", + ssrc: 958266064, + transportId: "T01", + jitter: 0.000125, + packetsLost: 0, + fractionLost: 0, + localId: "OT01A958266064", + roundTripTime: 0.001, + roundTripTimeMeasurements: 2, + totalRoundTripTime: 0.002, + }, + { + id: "RIV474891051", + timestamp: 1685442201974, + type: "remote-inbound-rtp", + codecId: "COT01_96", + kind: "video", + ssrc: 474891051, + transportId: "T01", + jitter: 0.0013444444444444445, + packetsLost: 0, + fractionLost: 0, + localId: "OT01V474891051", + roundTripTime: 0.001, + roundTripTimeMeasurements: 10, + totalRoundTripTime: 0.013, + }, + { + id: "ROA195186619", + timestamp: 1685442202064, + type: "remote-outbound-rtp", + codecId: "CIT01_111_minptime=10;usedtx=1;useinbandfec=1", + kind: "audio", + ssrc: 195186619, + transportId: "T01", + bytesSent: 19604, + packetsSent: 270, + localId: "IT01A195186619", + remoteTimestamp: 1685442202064, + reportsSent: 3, + roundTripTimeMeasurements: 0, + totalRoundTripTime: 0, + }, + { + id: "SA23", + timestamp: 1685442202456.655, + type: "media-source", + kind: "audio", + trackIdentifier: "c9d6dc16-8f14-4e8f-865b-c25091bec7e4", + audioLevel: 0.0013733329264198737, + echoReturnLoss: -30, + echoReturnLossEnhancement: 0.17551203072071075, + totalAudioEnergy: 0.00023577541256478306, + totalSamplesDuration: 9.949999999999832, + }, + { + id: "SV24", + timestamp: 1685442202456.655, + type: "media-source", + kind: "video", + trackIdentifier: "9ae8f8bc-f13e-4cfa-9c63-8d17fa132992", + frames: 299, + framesPerSecond: 31, + height: 360, + width: 640, + }, + { + id: "T01", + timestamp: 1685442202456.655, + type: "transport", + bytesReceived: 846330, + bytesSent: 1928112, + dtlsCipher: "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + dtlsRole: "client", + dtlsState: "connected", + iceLocalUsernameFragment: "zBQ8", + iceRole: "controlled", + iceState: "connected", + localCertificateId: + "CF8D:03:B2:A8:7F:FE:52:60:98:42:3F:F1:A3:61:89:CD:B2:39:0E:17:F7:AE:57:79:5F:96:6F:41:E8:DA:CB:2D", + packetsReceived: 1336, + packetsSent: 2293, + remoteCertificateId: + "CFCE:A3:2C:36:33:68:11:31:AB:DC:67:BE:3C:7D:03:00:F5:73:BC:09:23:72:F1:5D:21:F8:54:58:2C:FC:3E:74", + selectedCandidatePairChanges: 4, + selectedCandidatePairId: "CPryM2UC4o_8H+/O2yO", + srtpCipher: "AES_CM_128_HMAC_SHA1_80", + tlsVersion: "FEFD", + }, +]; + +export const currentChromeReport = [ + { + id: "AP", + timestamp: 1685442212456.4521, + type: "media-playout", + kind: "audio", + synthesizedSamplesDuration: 0, + synthesizedSamplesEvents: 0, + totalPlayoutDelay: 22455.74544, + totalSamplesCount: 957600, + totalSamplesDuration: 19.95, + }, + { + id: "CF8D:03:B2:A8:7F:FE:52:60:98:42:3F:F1:A3:61:89:CD:B2:39:0E:17:F7:AE:57:79:5F:96:6F:41:E8:DA:CB:2D", + timestamp: 1685442212456.4521, + type: "certificate", + base64Certificate: + "MIIBFjCBvKADAgECAggGFX25uzXk/jAKBggqhkjOPQQDAjARMQ8wDQYDVQQDDAZXZWJSVEMwHhcNMjMwNTI5MTAyMzEyWhcNMjMwNjI5MTAyMzEyWjARMQ8wDQYDVQQDDAZXZWJSVEMwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATICxbNzVNS8Z+SEntEKIYTryX5Ib0Fglm9Z8+nqsgiLOPOKPz+tc/dzE/55de4VQXJIJrO+wJ/c+JCOfp3sqlSMAoGCCqGSM49BAMCA0kAMEYCIQCZ95K1ot41YP/3Q4cURKDjHcYkBcAVkHBupmnWxVY+LQIhAKQwE43fZLYiEBG+AvFjj2sicilsEZ6r71E61YYZmYqz", + fingerprint: "8D:03:B2:A8:7F:FE:52:60:98:42:3F:F1:A3:61:89:CD:B2:39:0E:17:F7:AE:57:79:5F:96:6F:41:E8:DA:CB:2D", + fingerprintAlgorithm: "sha-256", + }, + { + id: "CFCE:A3:2C:36:33:68:11:31:AB:DC:67:BE:3C:7D:03:00:F5:73:BC:09:23:72:F1:5D:21:F8:54:58:2C:FC:3E:74", + timestamp: 1685442212456.4521, + type: "certificate", + base64Certificate: + "MIIBFjCBvKADAgECAghUitPBs0RuVDAKBggqhkjOPQQDAjARMQ8wDQYDVQQDDAZXZWJSVEMwHhcNMjMwNTI5MTAyMzEyWhcNMjMwNjI5MTAyMzEyWjARMQ8wDQYDVQQDDAZXZWJSVEMwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAASY7R/n7WzMx3v6bhwLpuv+XhtRAAkFOTnruyGc3oeh04OqoriE9FQ4sHm5rqD8vGah8eqHfN6g1XDgA1yEC3rnMAoGCCqGSM49BAMCA0kAMEYCIQCfgRuep47juMTV+9yUKbuqTklkbDfZ0vLw4+ySKbaWGAIhAPqYvacUo7jmH1DV7sl9mLNeTpQqpqbqsF+lbuQ83qcM", + fingerprint: "CE:A3:2C:36:33:68:11:31:AB:DC:67:BE:3C:7D:03:00:F5:73:BC:09:23:72:F1:5D:21:F8:54:58:2C:FC:3E:74", + fingerprintAlgorithm: "sha-256", + }, + { + id: "CIT01_111_minptime=10;usedtx=1;useinbandfec=1", + timestamp: 1685442212456.4521, + type: "codec", + channels: 2, + clockRate: 48000, + mimeType: "audio/opus", + payloadType: 111, + sdpFmtpLine: "minptime=10;usedtx=1;useinbandfec=1", + transportId: "T01", + }, + { + id: "CIT01_96", + timestamp: 1685442212456.4521, + type: "codec", + clockRate: 90000, + mimeType: "video/VP8", + payloadType: 96, + transportId: "T01", + }, + { + id: "COT01_111_minptime=10;usedtx=1;useinbandfec=1", + timestamp: 1685442212456.4521, + type: "codec", + channels: 2, + clockRate: 48000, + mimeType: "audio/opus", + payloadType: 111, + sdpFmtpLine: "minptime=10;usedtx=1;useinbandfec=1", + transportId: "T01", + }, + { + id: "COT01_96", + timestamp: 1685442212456.4521, + type: "codec", + clockRate: 90000, + mimeType: "video/VP8", + payloadType: 96, + transportId: "T01", + }, + { + id: "CPE/y2OM79_6JRzkx+P", + timestamp: 1685442212456.4521, + type: "candidate-pair", + bytesDiscardedOnSend: 0, + bytesReceived: 11236, + bytesSent: 26145, + consentRequestsSent: 2, + currentRoundTripTime: 0, + lastPacketReceivedTimestamp: 1685442193001, + lastPacketSentTimestamp: 1685442193019, + localCandidateId: "IE/y2OM79", + nominated: true, + packetsDiscardedOnSend: 0, + packetsReceived: 40, + packetsSent: 55, + priority: 9114756780654345000, + remoteCandidateId: "I6JRzkx+P", + requestsReceived: 3, + requestsSent: 3, + responsesReceived: 3, + responsesSent: 3, + state: "succeeded", + totalRoundTripTime: 0, + transportId: "T01", + writable: true, + }, + { + id: "CPE/y2OM79_fG99OI7Z", + timestamp: 1685442212456.4521, + type: "candidate-pair", + bytesDiscardedOnSend: 0, + bytesReceived: 0, + bytesSent: 0, + consentRequestsSent: 0, + localCandidateId: "IE/y2OM79", + nominated: false, + packetsDiscardedOnSend: 0, + packetsReceived: 0, + packetsSent: 0, + priority: 9114475305677635000, + remoteCandidateId: "IfG99OI7Z", + requestsReceived: 1, + requestsSent: 0, + responsesReceived: 0, + responsesSent: 1, + state: "waiting", + totalRoundTripTime: 0, + transportId: "T01", + writable: false, + }, + { + id: "CPhz/Z/Pd/_6JRzkx+P", + timestamp: 1685442212456.4521, + type: "candidate-pair", + bytesDiscardedOnSend: 0, + bytesReceived: 0, + bytesSent: 0, + consentRequestsSent: 0, + currentRoundTripTime: 0.001, + localCandidateId: "Ihz/Z/Pd/", + nominated: false, + packetsDiscardedOnSend: 0, + packetsReceived: 0, + packetsSent: 0, + priority: 9114475305677635000, + remoteCandidateId: "I6JRzkx+P", + requestsReceived: 0, + requestsSent: 1, + responsesReceived: 1, + responsesSent: 0, + state: "succeeded", + totalRoundTripTime: 0.001, + transportId: "T01", + writable: true, + }, + { + id: "CPryM2UC4o_8H+/O2yO", + timestamp: 1685442212456.4521, + type: "candidate-pair", + availableOutgoingBitrate: 5332923, + bytesDiscardedOnSend: 0, + bytesReceived: 2963285, + bytesSent: 4140357, + consentRequestsSent: 10, + currentRoundTripTime: 0, + lastPacketReceivedTimestamp: 1685442212451, + lastPacketSentTimestamp: 1685442212450, + localCandidateId: "IryM2UC4o", + nominated: true, + packetsDiscardedOnSend: 0, + packetsReceived: 3708, + packetsSent: 4734, + priority: 9115049250747470000, + remoteCandidateId: "I8H+/O2yO", + requestsReceived: 11, + requestsSent: 11, + responsesReceived: 11, + responsesSent: 11, + state: "succeeded", + totalRoundTripTime: 0.005, + transportId: "T01", + writable: true, + }, + { + id: "D12", + timestamp: 1685442212456.4521, + type: "data-channel", + bytesReceived: 0, + bytesSent: 0, + dataChannelIdentifier: 1, + label: "datachannel", + messagesReceived: 0, + messagesSent: 0, + protocol: "", + state: "open", + }, + { + id: "I6JRzkx+P", + timestamp: 1685442212456.4521, + type: "remote-candidate", + address: "192.168.64.1", + candidateType: "host", + foundation: "2620169396", + ip: "192.168.64.1", + isRemote: true, + port: 60959, + priority: 2122194687, + protocol: "udp", + transportId: "T01", + usernameFragment: "AUZN", + }, + { + id: "I8H+/O2yO", + timestamp: 1685442212456.4521, + type: "remote-candidate", + address: "aaaa:1111:aaaa:111:aaaa:1111:1111:aaa", + candidateType: "host", + foundation: "3525121815", + ip: "aaaa:1111:aaaa:111:aaaa:1111:1111:aaa", + isRemote: true, + port: 50629, + priority: 2122262783, + protocol: "udp", + transportId: "T01", + usernameFragment: "AUZN", + }, + { + id: "IE/y2OM79", + timestamp: 1685442212456.4521, + type: "local-candidate", + address: "192.168.64.1", + candidateType: "host", + foundation: "2979504628", + ip: "192.168.64.1", + isRemote: false, + networkType: "ethernet", + port: 62639, + priority: 2122194687, + protocol: "udp", + transportId: "T01", + usernameFragment: "zBQ8", + }, + { + id: "IT01A195186619", + timestamp: 1685442212456.4521, + type: "inbound-rtp", + codecId: "CIT01_111_minptime=10;usedtx=1;useinbandfec=1", + kind: "audio", + mediaType: "audio", + ssrc: 195186619, + transportId: "T01", + jitter: 0, + packetsLost: 0, + packetsReceived: 499, + audioLevel: 0.0009765923032319102, + bytesReceived: 33941, + concealedSamples: 15360, + concealmentEvents: 1, + estimatedPlayoutTimestamp: 3894431012370, + fecPacketsDiscarded: 0, + fecPacketsReceived: 0, + headerBytesReceived: 13972, + insertedSamplesForDeceleration: 1309, + jitterBufferDelay: 14198.4, + jitterBufferEmittedCount: 479040, + jitterBufferMinimumDelay: 10060.8, + jitterBufferTargetDelay: 10214.4, + lastPacketReceivedTimestamp: 1685442212144, + mid: "0", + packetsDiscarded: 0, + playoutId: "AP", + remoteId: "ROA195186619", + removedSamplesForAcceleration: 0, + silentConcealedSamples: 13800, + totalAudioEnergy: 0.007086603036643953, + totalSamplesDuration: 19.95000000000032, + totalSamplesReceived: 957600, + trackIdentifier: "f0e99682-e578-4ba4-a1f0-41a4b7ae7d01", + }, + { + id: "IT01V447914256", + timestamp: 1685442212456.4521, + type: "inbound-rtp", + codecId: "CIT01_96", + kind: "video", + mediaType: "video", + ssrc: 447914256, + transportId: "T01", + jitter: 0.001, + packetsLost: 0, + packetsReceived: 2812, + bytesReceived: 2777554, + decoderImplementation: "libvpx", + estimatedPlayoutTimestamp: 3894431012385, + firCount: 0, + frameHeight: 360, + frameWidth: 640, + framesAssembledFromMultiplePackets: 574, + framesDecoded: 586, + framesDropped: 0, + framesPerSecond: 30, + framesReceived: 587, + freezeCount: 0, + googTimingFrameInfo: + "1015557128,12917915,12917921,12917923,12917923,12917924,12917915,12917915,12917924,12917924,12917939,12917940,12917951,0,1", + headerBytesReceived: 69144, + jitterBufferDelay: 13.732, + jitterBufferEmittedCount: 585, + keyFramesDecoded: 3, + lastPacketReceivedTimestamp: 1685442212416, + mid: "1", + nackCount: 0, + pauseCount: 0, + pliCount: 0, + powerEfficientDecoder: false, + qpSum: 6594, + totalAssemblyTime: 0.064493, + totalDecodeTime: 0.534748, + totalFreezesDuration: 0, + totalInterFrameDelay: 19.496, + totalPausesDuration: 0, + totalProcessingDelay: 4.843700999999999, + totalSquaredInterFrameDelay: 0.6692820000000006, + trackIdentifier: "4c93cfe8-e3bc-4c27-8614-b8c895a49118", + }, + { + id: "IfG99OI7Z", + timestamp: 1685442212456.4521, + type: "remote-candidate", + address: "192.168.178.71", + candidateType: "host", + foundation: "2532229949", + ip: "192.168.178.71", + isRemote: true, + port: 54661, + priority: 2122129151, + protocol: "udp", + transportId: "T01", + usernameFragment: "AUZN", + }, + { + id: "Ihz/Z/Pd/", + timestamp: 1685442212456.4521, + type: "local-candidate", + address: "192.168.178.71", + candidateType: "host", + foundation: "3142975101", + ip: "192.168.178.71", + isRemote: false, + networkType: "ethernet", + port: 59368, + priority: 2122129151, + protocol: "udp", + transportId: "T01", + usernameFragment: "zBQ8", + }, + { + id: "IryM2UC4o", + timestamp: 1685442212456.4521, + type: "local-candidate", + address: "aaaa:1111:aaaa:111:aaaa:1111:1111:aaa", + candidateType: "host", + foundation: "4289079895", + ip: "aaaa:1111:aaaa:111:aaaa:1111:1111:aaa", + isRemote: false, + networkType: "ethernet", + port: 65502, + priority: 2122262783, + protocol: "udp", + transportId: "T01", + usernameFragment: "zBQ8", + }, + { + id: "OT01A958266064", + timestamp: 1685442212456.4521, + type: "outbound-rtp", + codecId: "COT01_111_minptime=10;usedtx=1;useinbandfec=1", + kind: "audio", + mediaType: "audio", + ssrc: 958266064, + transportId: "T01", + bytesSent: 31336, + packetsSent: 499, + active: true, + headerBytesSent: 13972, + mediaSourceId: "SA23", + mid: "0", + nackCount: 0, + remoteId: "RIA958266064", + retransmittedBytesSent: 0, + retransmittedPacketsSent: 0, + targetBitrate: 32000, + totalPacketSendDelay: 0, + }, + { + id: "OT01V474891051", + timestamp: 1685442212456.4521, + type: "outbound-rtp", + codecId: "COT01_96", + kind: "video", + mediaType: "video", + ssrc: 474891051, + transportId: "T01", + bytesSent: 3953856, + packetsSent: 3889, + active: true, + encoderImplementation: "libvpx", + firCount: 0, + frameHeight: 360, + frameWidth: 640, + framesEncoded: 588, + framesPerSecond: 30, + framesSent: 588, + headerBytesSent: 100920, + hugeFramesSent: 0, + keyFramesEncoded: 1, + mediaSourceId: "SV24", + mid: "1", + nackCount: 0, + pliCount: 2, + powerEfficientEncoder: false, + qpSum: 6657, + qualityLimitationDurations: { + bandwidth: 0, + cpu: 0, + none: 19.956, + other: 0, + }, + qualityLimitationReason: "none", + qualityLimitationResolutionChanges: 0, + remoteId: "RIV474891051", + retransmittedBytesSent: 0, + retransmittedPacketsSent: 0, + scalabilityMode: "L1T1", + targetBitrate: 1700000, + totalEncodeTime: 1.557, + totalEncodedBytesTarget: 0, + totalPacketSendDelay: 0.090407, + }, + { + id: "P", + timestamp: 1685442212456.4521, + type: "peer-connection", + dataChannelsClosed: 0, + dataChannelsOpened: 1, + }, + { + id: "RIA958266064", + timestamp: 1685442205897, + type: "remote-inbound-rtp", + codecId: "COT01_111_minptime=10;usedtx=1;useinbandfec=1", + kind: "audio", + ssrc: 958266064, + transportId: "T01", + jitter: 0, + packetsLost: 0, + fractionLost: 0, + localId: "OT01A958266064", + roundTripTime: 0.001, + roundTripTimeMeasurements: 3, + totalRoundTripTime: 0.003, + }, + { + id: "RIV474891051", + timestamp: 1685442211724, + type: "remote-inbound-rtp", + codecId: "COT01_96", + kind: "video", + ssrc: 474891051, + transportId: "T01", + jitter: 0.0013, + packetsLost: 0, + fractionLost: 0, + localId: "OT01V474891051", + roundTripTime: 0.001, + roundTripTimeMeasurements: 20, + totalRoundTripTime: 0.023, + }, + { + id: "ROA195186619", + timestamp: 1685442210064, + type: "remote-outbound-rtp", + codecId: "CIT01_111_minptime=10;usedtx=1;useinbandfec=1", + kind: "audio", + ssrc: 195186619, + transportId: "T01", + bytesSent: 31570, + packetsSent: 460, + localId: "IT01A195186619", + remoteTimestamp: 1685442210064, + reportsSent: 5, + roundTripTimeMeasurements: 0, + totalRoundTripTime: 0, + }, + { + id: "SA23", + timestamp: 1685442212456.4521, + type: "media-source", + kind: "audio", + trackIdentifier: "c9d6dc16-8f14-4e8f-865b-c25091bec7e4", + audioLevel: 0.0017090365306558428, + echoReturnLoss: -30, + echoReturnLossEnhancement: 0.17551203072071075, + totalAudioEnergy: 0.011151931473826194, + totalSamplesDuration: 19.95000000000032, + }, + { + id: "SV24", + timestamp: 1685442212456.4521, + type: "media-source", + kind: "video", + trackIdentifier: "9ae8f8bc-f13e-4cfa-9c63-8d17fa132992", + frames: 599, + framesPerSecond: 31, + height: 360, + width: 640, + }, + { + id: "T01", + timestamp: 1685442212456.4521, + type: "transport", + bytesReceived: 2974521, + bytesSent: 4166502, + dtlsCipher: "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + dtlsRole: "client", + dtlsState: "connected", + iceLocalUsernameFragment: "zBQ8", + iceRole: "controlled", + iceState: "connected", + localCertificateId: + "CF8D:03:B2:A8:7F:FE:52:60:98:42:3F:F1:A3:61:89:CD:B2:39:0E:17:F7:AE:57:79:5F:96:6F:41:E8:DA:CB:2D", + packetsReceived: 3748, + packetsSent: 4789, + remoteCertificateId: + "CFCE:A3:2C:36:33:68:11:31:AB:DC:67:BE:3C:7D:03:00:F5:73:BC:09:23:72:F1:5D:21:F8:54:58:2C:FC:3E:74", + selectedCandidatePairChanges: 4, + selectedCandidatePairId: "CPryM2UC4o_8H+/O2yO", + srtpCipher: "AES_CM_128_HMAC_SHA1_80", + tlsVersion: "FEFD", + }, +]; diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 876b658e2..876a447d2 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -18,25 +18,25 @@ import { mocked } from "jest-mock"; import { EventType, GroupCallIntent, GroupCallType, MatrixCall, MatrixEvent, Room, RoomMember } from "../../../src"; import { RoomStateEvent } from "../../../src/models/room-state"; -import { GroupCall, GroupCallEvent, GroupCallState } from "../../../src/webrtc/groupCall"; +import { GroupCall, GroupCallEvent, GroupCallState, GroupCallStatsReportEvent } from "../../../src/webrtc/groupCall"; import { IMyDevice, MatrixClient } from "../../../src/client"; import { + FAKE_CONF_ID, + FAKE_DEVICE_ID_1, + FAKE_DEVICE_ID_2, + FAKE_ROOM_ID, + FAKE_SESSION_ID_1, + FAKE_SESSION_ID_2, + FAKE_USER_ID_1, + FAKE_USER_ID_2, + FAKE_USER_ID_3, installWebRTCMocks, MockCallFeed, MockCallMatrixClient, + MockMatrixCall, MockMediaStream, MockMediaStreamTrack, MockRTCPeerConnection, - MockMatrixCall, - FAKE_ROOM_ID, - FAKE_USER_ID_1, - FAKE_CONF_ID, - FAKE_DEVICE_ID_2, - FAKE_SESSION_ID_2, - FAKE_USER_ID_2, - FAKE_DEVICE_ID_1, - FAKE_SESSION_ID_1, - FAKE_USER_ID_3, } from "../../test-utils/webrtc"; import { SDPStreamMetadataKey, SDPStreamMetadataPurpose } from "../../../src/webrtc/callEventTypes"; import { sleep } from "../../../src/utils"; @@ -44,6 +44,9 @@ import { CallEventHandlerEvent } from "../../../src/webrtc/callEventHandler"; import { CallFeed } from "../../../src/webrtc/callFeed"; import { CallEvent, CallState } from "../../../src/webrtc/call"; import { flushPromises } from "../../test-utils/flushPromises"; +import { CallFeedReport } from "../../../src/webrtc/stats/statsReport"; +import { CallFeedStatsReporter } from "../../../src/webrtc/stats/callFeedStatsReporter"; +import { StatsReportEmitter } from "../../../src/webrtc/stats/statsReportEmitter"; const FAKE_STATE_EVENTS = [ { @@ -1726,4 +1729,89 @@ describe("Group Call", function () { expect(start).toHaveBeenCalled(); }); }); + + describe("as stats event listener and a CallFeedReport was triggered", () => { + let groupCall: GroupCall; + let reportEmitter: StatsReportEmitter; + const report: CallFeedReport = {} as CallFeedReport; + beforeEach(async () => { + CallFeedStatsReporter.expandCallFeedReport = jest.fn().mockReturnValue(report); + const typedMockClient = new MockCallMatrixClient(FAKE_USER_ID_1, FAKE_DEVICE_ID_1, FAKE_SESSION_ID_1); + const mockClient = typedMockClient.typed(); + const room = new Room(FAKE_ROOM_ID, mockClient, FAKE_USER_ID_1); + room.currentState.members[FAKE_USER_ID_1] = { + userId: FAKE_USER_ID_1, + membership: "join", + } as unknown as RoomMember; + room.currentState.members[FAKE_USER_ID_2] = { + userId: FAKE_USER_ID_2, + membership: "join", + } as unknown as RoomMember; + room.currentState.getStateEvents = jest.fn().mockImplementation(mockGetStateEvents()); + groupCall = await createAndEnterGroupCall(mockClient, room); + reportEmitter = groupCall.getGroupCallStats().reports; + }); + + it("should not extends with feed stats if no call exists", async () => { + const testPromise = new Promise((done) => { + groupCall.on(GroupCallStatsReportEvent.CallFeedStats, () => { + expect(CallFeedStatsReporter.expandCallFeedReport).toHaveBeenCalledWith({}, [], "from-call-feed"); + done(); + }); + }); + const report: CallFeedReport = {} as CallFeedReport; + reportEmitter.emitCallFeedReport(report); + await testPromise; + }); + + it("and a CallFeedReport was triggered then it should extends with local feed", async () => { + const localCallFeed = {} as CallFeed; + groupCall.localCallFeed = localCallFeed; + + const testPromise = new Promise((done) => { + groupCall.on(GroupCallStatsReportEvent.CallFeedStats, () => { + expect(CallFeedStatsReporter.expandCallFeedReport).toHaveBeenCalledWith( + report, + [localCallFeed], + "from-local-feed", + ); + expect(CallFeedStatsReporter.expandCallFeedReport).toHaveBeenCalledWith( + report, + [], + "from-call-feed", + ); + done(); + }); + }); + const report: CallFeedReport = {} as CallFeedReport; + reportEmitter.emitCallFeedReport(report); + await testPromise; + }); + + it("and a CallFeedReport was triggered then it should extends with remote feed", async () => { + const localCallFeed = {} as CallFeed; + groupCall.localCallFeed = localCallFeed; + // @ts-ignore Suppress error because access to private property + const call = groupCall.calls.get(FAKE_USER_ID_2)!.get(FAKE_DEVICE_ID_2)!; + report.callId = call.callId; + const feeds = call.getFeeds(); + const testPromise = new Promise((done) => { + groupCall.on(GroupCallStatsReportEvent.CallFeedStats, () => { + expect(CallFeedStatsReporter.expandCallFeedReport).toHaveBeenCalledWith( + report, + [localCallFeed], + "from-local-feed", + ); + expect(CallFeedStatsReporter.expandCallFeedReport).toHaveBeenCalledWith( + report, + feeds, + "from-call-feed", + ); + done(); + }); + }); + reportEmitter.emitCallFeedReport(report); + await testPromise; + }); + }); }); diff --git a/spec/unit/webrtc/stats/__snapshots__/callFeedStatsReporter.spec.ts.snap b/spec/unit/webrtc/stats/__snapshots__/callFeedStatsReporter.spec.ts.snap new file mode 100644 index 000000000..54af0b494 --- /dev/null +++ b/spec/unit/webrtc/stats/__snapshots__/callFeedStatsReporter.spec.ts.snap @@ -0,0 +1,110 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`CallFeedStatsReporter should builds CallFeedReport 1`] = ` +{ + "callFeeds": [], + "callId": "CALL_ID", + "opponentMemberId": "USER_ID", + "transceiver": [ + { + "currentDirection": "sendonly", + "direction": "sendrecv", + "mid": "0", + "receiver": { + "constrainDeviceId": "constrainDeviceId-receiver_audio_0", + "enabled": true, + "id": "receiver_audio_0", + "kind": "audio", + "label": "receiver", + "muted": false, + "readyState": "live", + "settingDeviceId": "settingDeviceId-receiver_audio_0", + }, + "sender": { + "constrainDeviceId": "constrainDeviceId-sender_audio_0", + "enabled": true, + "id": "sender_audio_0", + "kind": "audio", + "label": "sender", + "muted": false, + "readyState": "live", + "settingDeviceId": "settingDeviceId-sender_audio_0", + }, + }, + { + "currentDirection": "sendrecv", + "direction": "recvonly", + "mid": "1", + "receiver": { + "constrainDeviceId": "constrainDeviceId-receiver_video_1", + "enabled": true, + "id": "receiver_video_1", + "kind": "video", + "label": "receiver", + "muted": false, + "readyState": "live", + "settingDeviceId": "settingDeviceId-receiver_video_1", + }, + "sender": { + "constrainDeviceId": "constrainDeviceId-sender_video_1", + "enabled": true, + "id": "sender_video_1", + "kind": "video", + "label": "sender", + "muted": false, + "readyState": "live", + "settingDeviceId": "settingDeviceId-sender_video_1", + }, + }, + { + "currentDirection": "recvonly", + "direction": "recvonly", + "mid": "2", + "receiver": { + "constrainDeviceId": "constrainDeviceId-receiver_video_2", + "enabled": true, + "id": "receiver_video_2", + "kind": "video", + "label": "receiver", + "muted": false, + "readyState": "live", + "settingDeviceId": "settingDeviceId-receiver_video_2", + }, + "sender": null, + }, + ], +} +`; + +exports[`CallFeedStatsReporter should extends CallFeedReport with call feeds 1`] = ` +[ + { + "audio": { + "constrainDeviceId": "constrainDeviceId-video-1", + "enabled": true, + "id": "video-1", + "kind": "video", + "label": "--", + "muted": false, + "readyState": "live", + "settingDeviceId": "settingDeviceId-video-1", + }, + "isAudioMuted": true, + "isVideoMuted": false, + "prefix": "unknown", + "purpose": undefined, + "stream": "stream-1", + "type": "local", + "video": { + "constrainDeviceId": "constrainDeviceId-audio-1", + "enabled": true, + "id": "audio-1", + "kind": "audio", + "label": "--", + "muted": false, + "readyState": "live", + "settingDeviceId": "settingDeviceId-audio-1", + }, + }, +] +`; diff --git a/spec/unit/webrtc/stats/callFeedStatsReporter.spec.ts b/spec/unit/webrtc/stats/callFeedStatsReporter.spec.ts new file mode 100644 index 000000000..32e43ec3e --- /dev/null +++ b/spec/unit/webrtc/stats/callFeedStatsReporter.spec.ts @@ -0,0 +1,117 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { CallFeedStatsReporter } from "../../../../src/webrtc/stats/callFeedStatsReporter"; +import { CallFeedReport } from "../../../../src/webrtc/stats/statsReport"; +import { CallFeed } from "../../../../src/webrtc/callFeed"; + +const CALL_ID = "CALL_ID"; +const USER_ID = "USER_ID"; +describe("CallFeedStatsReporter", () => { + let rtcSpy: RTCPeerConnection; + beforeEach(() => { + rtcSpy = {} as RTCPeerConnection; + rtcSpy.getTransceivers = jest.fn().mockReturnValue(buildTransceiverMocks()); + }); + + describe("should", () => { + it("builds CallFeedReport", async () => { + expect(CallFeedStatsReporter.buildCallFeedReport(CALL_ID, USER_ID, rtcSpy)).toMatchSnapshot(); + }); + + it("extends CallFeedReport with call feeds", async () => { + const feed = buildCallFeedMock("1"); + const callFeedList: CallFeed[] = [feed]; + const report = { + callId: "callId", + opponentMemberId: "opponentMemberId", + transceiver: [], + callFeeds: [], + } as CallFeedReport; + + expect(CallFeedStatsReporter.expandCallFeedReport(report, callFeedList).callFeeds).toMatchSnapshot(); + }); + }); + + const buildTransceiverMocks = (): RTCRtpTransceiver[] => { + const trans1 = { + mid: "0", + direction: "sendrecv", + currentDirection: "sendonly", + sender: buildSenderMock("sender_audio_0", "audio"), + receiver: buildReceiverMock("receiver_audio_0", "audio"), + } as RTCRtpTransceiver; + const trans2 = { + mid: "1", + direction: "recvonly", + currentDirection: "sendrecv", + sender: buildSenderMock("sender_video_1", "video"), + receiver: buildReceiverMock("receiver_video_1", "video"), + } as RTCRtpTransceiver; + const trans3 = { + mid: "2", + direction: "recvonly", + currentDirection: "recvonly", + sender: { track: null } as RTCRtpSender, + receiver: buildReceiverMock("receiver_video_2", "video"), + } as RTCRtpTransceiver; + return [trans1, trans2, trans3]; + }; + + const buildSenderMock = (id: string, kind: "audio" | "video"): RTCRtpSender => { + const track = buildTrackMock(id, kind); + return { + track, + } as RTCRtpSender; + }; + + const buildReceiverMock = (id: string, kind: "audio" | "video"): RTCRtpReceiver => { + const track = buildTrackMock(id, kind); + return { + track, + } as RTCRtpReceiver; + }; + + const buildTrackMock = (id: string, kind: "audio" | "video"): MediaStreamTrack => { + return { + id, + kind, + enabled: true, + label: "--", + muted: false, + readyState: "live", + getSettings: () => ({ deviceId: `settingDeviceId-${id}` }), + getConstraints: () => ({ deviceId: `constrainDeviceId-${id}` }), + } as MediaStreamTrack; + }; + + const buildCallFeedMock = (id: string, isLocal = true): CallFeed => { + const stream = { + id: `stream-${id}`, + getAudioTracks(): MediaStreamTrack[] { + return [buildTrackMock(`video-${id}`, "video")]; + }, + getVideoTracks(): MediaStreamTrack[] { + return [buildTrackMock(`audio-${id}`, "audio")]; + }, + } as MediaStream; + return { + stream, + isLocal: () => isLocal, + isVideoMuted: () => false, + isAudioMuted: () => true, + } as CallFeed; + }; +}); diff --git a/spec/unit/webrtc/stats/callStatsReportGatherer.spec.ts b/spec/unit/webrtc/stats/callStatsReportGatherer.spec.ts index e6a364d6a..0c9c943cb 100644 --- a/spec/unit/webrtc/stats/callStatsReportGatherer.spec.ts +++ b/spec/unit/webrtc/stats/callStatsReportGatherer.spec.ts @@ -17,6 +17,8 @@ limitations under the License. import { CallStatsReportGatherer } from "../../../../src/webrtc/stats/callStatsReportGatherer"; import { StatsReportEmitter } from "../../../../src/webrtc/stats/statsReportEmitter"; import { MediaSsrcHandler } from "../../../../src/webrtc/stats/media/mediaSsrcHandler"; +import { currentChromeReport, prevChromeReport } from "../../../test-utils/webrtcReports"; +import { MockRTCPeerConnection } from "../../../test-utils/webrtc"; const CALL_ID = "CALL_ID"; const USER_ID = "USER_ID"; @@ -28,6 +30,7 @@ describe("CallStatsReportGatherer", () => { beforeEach(() => { rtcSpy = { getStats: () => new Promise(() => null) } as RTCPeerConnection; rtcSpy.addEventListener = jest.fn(); + rtcSpy.getTransceivers = jest.fn().mockReturnValue([]); emitter = new StatsReportEmitter(); collector = new CallStatsReportGatherer(CALL_ID, USER_ID, rtcSpy, emitter); }); @@ -145,6 +148,70 @@ describe("CallStatsReportGatherer", () => { }); expect(collector.getActive()).toBeTruthy(); }); + + describe("should not only produce a call summary stat but also", () => { + const wantedSummaryReport = { + isFirstCollection: false, + audioTrackSummary: { + concealedAudio: 0, + count: 0, + maxJitter: 0, + maxPacketLoss: 0, + muted: 0, + totalAudio: 0, + }, + receivedAudioMedia: 0, + receivedMedia: 0, + receivedVideoMedia: 0, + videoTrackSummary: { + concealedAudio: 0, + count: 0, + maxJitter: 0, + maxPacketLoss: 0, + muted: 0, + totalAudio: 0, + }, + }; + beforeEach(() => { + rtcSpy = new MockRTCPeerConnection() as unknown as RTCPeerConnection; + collector = new CallStatsReportGatherer(CALL_ID, USER_ID, rtcSpy, emitter); + const getStats = jest.spyOn(rtcSpy, "getStats"); + + const previous = prevChromeReport as unknown as RTCStatsReport; + previous.get = (id: string) => { + return prevChromeReport.find((data) => data.id === id); + }; + // @ts-ignore + collector.previousStatsReport = previous; + + const current = currentChromeReport as unknown as RTCStatsReport; + current.get = (id: string) => { + return currentChromeReport.find((data) => data.id === id); + }; + + // @ts-ignore + getStats.mockResolvedValue(current); + }); + + it("emit byteSentStatsReport", async () => { + const emitByteSendReport = jest.spyOn(emitter, "emitByteSendReport"); + const actual = await collector.processStats("GROUP_CALL_ID", "LOCAL_USER_ID"); + expect(actual).toEqual(wantedSummaryReport); + expect(emitByteSendReport).toHaveBeenCalled(); + }); + it("emit emitConnectionStatsReport", async () => { + const emitConnectionStatsReport = jest.spyOn(emitter, "emitConnectionStatsReport"); + const actual = await collector.processStats("GROUP_CALL_ID", "LOCAL_USER_ID"); + expect(actual).toEqual(wantedSummaryReport); + expect(emitConnectionStatsReport).toHaveBeenCalled(); + }); + it("emit callFeedStatsReport", async () => { + const emitCallFeedReport = jest.spyOn(emitter, "emitCallFeedReport"); + const actual = await collector.processStats("GROUP_CALL_ID", "LOCAL_USER_ID"); + expect(actual).toEqual(wantedSummaryReport); + expect(emitCallFeedReport).toHaveBeenCalled(); + }); + }); }); describe("on signal state change event", () => { diff --git a/spec/unit/webrtc/stats/statsReportEmitter.spec.ts b/spec/unit/webrtc/stats/statsReportEmitter.spec.ts index d5d5df84c..c5237fd19 100644 --- a/spec/unit/webrtc/stats/statsReportEmitter.spec.ts +++ b/spec/unit/webrtc/stats/statsReportEmitter.spec.ts @@ -16,6 +16,7 @@ limitations under the License. import { StatsReportEmitter } from "../../../../src/webrtc/stats/statsReportEmitter"; import { ByteSentStatsReport, + CallFeedReport, ConnectionStatsReport, StatsReport, SummaryStatsReport, @@ -62,4 +63,16 @@ describe("StatsReportEmitter", () => { emitter.emitSummaryStatsReport(report); }); }); + + it("should emit and receive CallFeedReports", async () => { + const report = {} as CallFeedReport; + return new Promise((resolve, _) => { + emitter.on(StatsReport.CALL_FEED_REPORT, (r) => { + expect(r).toBe(report); + resolve(null); + return; + }); + emitter.emitCallFeedReport(report); + }); + }); }); diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index d6f7b4f1e..4c3224435 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -25,8 +25,15 @@ import { GroupCallEventHandlerEvent } from "./groupCallEventHandler"; import { IScreensharingOpts } from "./mediaHandler"; import { mapsEqual } from "../utils"; import { GroupCallStats } from "./stats/groupCallStats"; -import { ByteSentStatsReport, ConnectionStatsReport, StatsReport, SummaryStatsReport } from "./stats/statsReport"; +import { + ByteSentStatsReport, + CallFeedReport, + ConnectionStatsReport, + StatsReport, + SummaryStatsReport, +} from "./stats/statsReport"; import { SummaryStatsReportGatherer } from "./stats/summaryStatsReportGatherer"; +import { CallFeedStatsReporter } from "./stats/callFeedStatsReporter"; export enum GroupCallIntent { Ring = "m.ring", @@ -97,6 +104,7 @@ export enum GroupCallStatsReportEvent { ConnectionStats = "GroupCall.connection_stats", ByteSentStats = "GroupCall.byte_sent_stats", SummaryStats = "GroupCall.summary_stats", + CallFeedStats = "GroupCall.call_feed_stats", } /** @@ -106,6 +114,7 @@ export type GroupCallStatsReportEventHandlerMap = { [GroupCallStatsReportEvent.ConnectionStats]: (report: GroupCallStatsReport) => void; [GroupCallStatsReportEvent.ByteSentStats]: (report: GroupCallStatsReport) => void; [GroupCallStatsReportEvent.SummaryStats]: (report: GroupCallStatsReport) => void; + [GroupCallStatsReportEvent.CallFeedStats]: (report: GroupCallStatsReport) => void; }; export enum GroupCallErrorCode { @@ -114,7 +123,9 @@ export enum GroupCallErrorCode { PlaceCallFailed = "place_call_failed", } -export interface GroupCallStatsReport { +export interface GroupCallStatsReport< + T extends ConnectionStatsReport | ByteSentStatsReport | SummaryStatsReport | CallFeedReport, +> { report: T; } @@ -288,6 +299,22 @@ export class GroupCall extends TypedEventEmitter< this.emit(GroupCallStatsReportEvent.SummaryStats, { report }); }; + private onCallFeedReport = (report: CallFeedReport): void => { + if (this.localCallFeed) { + report = CallFeedStatsReporter.expandCallFeedReport(report, [this.localCallFeed], "from-local-feed"); + } + + const callFeeds: CallFeed[] = []; + this.forEachCall((call) => { + if (call.callId === report.callId) { + call.getFeeds().forEach((f) => callFeeds.push(f)); + } + }); + + report = CallFeedStatsReporter.expandCallFeedReport(report, callFeeds, "from-call-feed"); + this.emit(GroupCallStatsReportEvent.CallFeedStats, { report }); + }; + public async create(): Promise { this.creationTs = Date.now(); this.client.groupCallEventHandler!.groupCalls.set(this.room.roomId, this); @@ -1642,6 +1669,7 @@ export class GroupCall extends TypedEventEmitter< this.stats.reports.on(StatsReport.CONNECTION_STATS, this.onConnectionStats); this.stats.reports.on(StatsReport.BYTE_SENT_STATS, this.onByteSentStats); this.stats.reports.on(StatsReport.SUMMARY_STATS, this.onSummaryStats); + this.stats.reports.on(StatsReport.CALL_FEED_REPORT, this.onCallFeedReport); } return this.stats; } diff --git a/src/webrtc/stats/callFeedStatsReporter.ts b/src/webrtc/stats/callFeedStatsReporter.ts new file mode 100644 index 000000000..2b2c092d7 --- /dev/null +++ b/src/webrtc/stats/callFeedStatsReporter.ts @@ -0,0 +1,94 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { CallFeedReport, CallFeedStats, TrackStats, TransceiverStats } from "./statsReport"; +import { CallFeed } from "../callFeed"; + +export class CallFeedStatsReporter { + public static buildCallFeedReport(callId: string, opponentMemberId: string, pc: RTCPeerConnection): CallFeedReport { + const rtpTransceivers = pc.getTransceivers(); + const transceiver: TransceiverStats[] = []; + const callFeeds: CallFeedStats[] = []; + + rtpTransceivers.forEach((t) => { + const sender = t.sender?.track ? CallFeedStatsReporter.buildTrackStats(t.sender.track, "sender") : null; + const receiver = CallFeedStatsReporter.buildTrackStats(t.receiver.track, "receiver"); + transceiver.push({ + mid: t.mid == null ? "null" : t.mid, + direction: t.direction, + currentDirection: t.currentDirection == null ? "null" : t.currentDirection, + sender, + receiver, + }); + }); + + return { + callId, + opponentMemberId, + transceiver, + callFeeds, + }; + } + + private static buildTrackStats(track: MediaStreamTrack, label = "--"): TrackStats { + const settingDeviceId = track.getSettings()?.deviceId; + const constrainDeviceId = track.getConstraints()?.deviceId; + + return { + id: track.id, + kind: track.kind, + settingDeviceId: settingDeviceId ? settingDeviceId : "unknown", + constrainDeviceId: constrainDeviceId ? constrainDeviceId : "unknown", + muted: track.muted, + enabled: track.enabled, + readyState: track.readyState, + label, + } as TrackStats; + } + + public static expandCallFeedReport( + report: CallFeedReport, + callFeeds: CallFeed[], + prefix = "unknown", + ): CallFeedReport { + if (!report.callFeeds) { + report.callFeeds = []; + } + callFeeds.forEach((feed) => { + const audioTracks = feed.stream.getAudioTracks(); + const videoTracks = feed.stream.getVideoTracks(); + const audio = + audioTracks.length > 0 + ? CallFeedStatsReporter.buildTrackStats(feed.stream.getAudioTracks()[0], feed.purpose) + : null; + const video = + videoTracks.length > 0 + ? CallFeedStatsReporter.buildTrackStats(feed.stream.getVideoTracks()[0], feed.purpose) + : null; + const feedStats = { + stream: feed.stream.id, + type: feed.isLocal() ? "local" : "remote", + audio, + video, + purpose: feed.purpose, + prefix, + isVideoMuted: feed.isVideoMuted(), + isAudioMuted: feed.isAudioMuted(), + } as CallFeedStats; + report.callFeeds.push(feedStats); + }); + return report; + } +} diff --git a/src/webrtc/stats/callStatsReportGatherer.ts b/src/webrtc/stats/callStatsReportGatherer.ts index fb3e50e69..cc52156f0 100644 --- a/src/webrtc/stats/callStatsReportGatherer.ts +++ b/src/webrtc/stats/callStatsReportGatherer.ts @@ -26,6 +26,8 @@ import { TrackStatsBuilder } from "./trackStatsBuilder"; import { ConnectionStatsReportBuilder } from "./connectionStatsReportBuilder"; import { ValueFormatter } from "./valueFormatter"; import { CallStatsReportSummary } from "./callStatsReportSummary"; +import { logger } from "../../logger"; +import { CallFeedStatsReporter } from "./callFeedStatsReporter"; export class CallStatsReportGatherer { private isActive = true; @@ -62,10 +64,11 @@ export class CallStatsReportGatherer { .then((report) => { // @ts-ignore this.currentStatsReport = typeof report?.result === "function" ? report.result() : report; + try { this.processStatsReport(groupCallId, localUserId); } catch (error) { - this.isActive = false; + this.handleError(error); return summary; } @@ -161,6 +164,9 @@ export class CallStatsReportGatherer { }); this.emitter.emitByteSendReport(byteSentStatsReport); + this.emitter.emitCallFeedReport( + CallFeedStatsReporter.buildCallFeedReport(this.callId, this.opponentMemberId, this.pc), + ); this.processAndEmitConnectionStatsReport(); } @@ -172,8 +178,9 @@ export class CallStatsReportGatherer { return this.isActive; } - private handleError(_: any): void { + private handleError(error: any): void { this.isActive = false; + logger.warn(`CallStatsReportGatherer ${this.callId} processStatsReport fails and set to inactive ${error}`); } private processAndEmitConnectionStatsReport(): void { diff --git a/src/webrtc/stats/statsReport.ts b/src/webrtc/stats/statsReport.ts index 737e8db11..9f24267e1 100644 --- a/src/webrtc/stats/statsReport.ts +++ b/src/webrtc/stats/statsReport.ts @@ -20,19 +20,22 @@ import { Resolution } from "./media/mediaTrackStats"; export enum StatsReport { CONNECTION_STATS = "StatsReport.connection_stats", + CALL_FEED_REPORT = "StatsReport.call_feed_report", BYTE_SENT_STATS = "StatsReport.byte_sent_stats", SUMMARY_STATS = "StatsReport.summary_stats", } -export type TrackID = string; -export type ByteSend = number; - +/// ByteSentStatsReport ################################################################################################ export interface ByteSentStatsReport extends Map { callId?: string; opponentMemberId?: string; // is a map: `local trackID` => byte send } +export type TrackID = string; +export type ByteSend = number; + +/// ConnectionStatsReport ############################################################################################## export interface ConnectionStatsReport { callId?: string; opponentMemberId?: string; @@ -68,6 +71,7 @@ export interface CodecMap { remote: Map; } +/// SummaryStatsReport ################################################################################################# export interface SummaryStatsReport { /** * Aggregated the information for percentage of received media @@ -89,3 +93,41 @@ export interface SummaryStatsReport { ratioPeerConnectionToDevices?: number; // Todo: Decide if we want an index (or a timestamp) of this report in relation to the group call, to help differenciate when issues occur and ignore/track initial connection delays. } + +/// CallFeedReport ##################################################################################################### +export interface CallFeedReport { + callId: string; + opponentMemberId: string; + transceiver: TransceiverStats[]; + callFeeds: CallFeedStats[]; +} + +export interface CallFeedStats { + stream: string; + type: "remote" | "local"; + audio: TrackStats | null; + video: TrackStats | null; + purpose: string; + prefix: string; + isVideoMuted: boolean; + isAudioMuted: boolean; +} + +export interface TransceiverStats { + readonly mid: string; + readonly sender: TrackStats | null; + readonly receiver: TrackStats | null; + readonly direction: string; + readonly currentDirection: string; +} + +export interface TrackStats { + readonly id: string; + readonly kind: "audio" | "video"; + readonly settingDeviceId: string; + readonly constrainDeviceId: string; + readonly muted: boolean; + readonly enabled: boolean; + readonly readyState: "ended" | "live"; + readonly label: string; +} diff --git a/src/webrtc/stats/statsReportEmitter.ts b/src/webrtc/stats/statsReportEmitter.ts index 944bca736..1b8834326 100644 --- a/src/webrtc/stats/statsReportEmitter.ts +++ b/src/webrtc/stats/statsReportEmitter.ts @@ -15,11 +15,18 @@ limitations under the License. */ import { TypedEventEmitter } from "../../models/typed-event-emitter"; -import { ByteSentStatsReport, ConnectionStatsReport, StatsReport, SummaryStatsReport } from "./statsReport"; +import { + ByteSentStatsReport, + CallFeedReport, + ConnectionStatsReport, + StatsReport, + SummaryStatsReport, +} from "./statsReport"; export type StatsReportHandlerMap = { [StatsReport.BYTE_SENT_STATS]: (report: ByteSentStatsReport) => void; [StatsReport.CONNECTION_STATS]: (report: ConnectionStatsReport) => void; + [StatsReport.CALL_FEED_REPORT]: (report: CallFeedReport) => void; [StatsReport.SUMMARY_STATS]: (report: SummaryStatsReport) => void; }; @@ -32,6 +39,10 @@ export class StatsReportEmitter extends TypedEventEmitter