From 96abc8160d99596cd3cbc3e3039256a23f14ce28 Mon Sep 17 00:00:00 2001 From: "Sharad Jaiswal (Eng)" Date: Wed, 16 Nov 2022 13:02:27 -0800 Subject: [PATCH] Codec changes to support ACK_RECEIVE_TIMESTAMPS Summary: Create a new ACK_RECEIVE_TIMESTAMPS frame, as outlined in https://www.ietf.org/archive/id/draft-smith-quic-receive-ts-00.html#name-ack_receive_timestamps-fram Reviewed By: mjoras Differential Revision: D37799050 fbshipit-source-id: 0157c7fa7c4e489bb310f7c9cd6c0c1877e4967f --- quic/QuicConstants.h | 3 + quic/api/QuicPacketScheduler.cpp | 37 +- quic/api/test/QuicPacketSchedulerTest.cpp | 6 +- quic/client/QuicClientTransport.cpp | 34 +- quic/client/state/ClientStateMachine.cpp | 31 +- quic/codec/Decode.cpp | 149 +++- quic/codec/Decode.h | 32 +- quic/codec/QuicPacketRebuilder.cpp | 33 +- quic/codec/QuicWriteCodec.cpp | 265 +++++- quic/codec/QuicWriteCodec.h | 94 ++- quic/codec/Types.cpp | 2 + quic/codec/Types.h | 18 +- quic/codec/test/QuicPacketRebuilderTest.cpp | 14 +- quic/codec/test/QuicWriteCodecTest.cpp | 793 ++++++++++++++++-- quic/common/test/TestPacketBuilders.cpp | 11 +- .../client/test/QuicClientTransportTest.cpp | 7 +- quic/handshake/TransportParameters.h | 3 + quic/logging/QLoggerConstants.cpp | 2 + quic/server/state/ServerStateMachine.cpp | 58 +- quic/server/test/QuicServerTransportTest.cpp | 6 +- quic/state/AckStates.h | 29 +- quic/state/StateData.h | 7 + quic/state/TransportSettings.h | 19 + 23 files changed, 1468 insertions(+), 185 deletions(-) diff --git a/quic/QuicConstants.h b/quic/QuicConstants.h index 704adf349..70d79fd7e 100644 --- a/quic/QuicConstants.h +++ b/quic/QuicConstants.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -251,6 +252,7 @@ enum class FrameType : uint64_t { GROUP_STREAM_OFF_FIN = 0x37, GROUP_STREAM_OFF_LEN = 0x38, GROUP_STREAM_OFF_LEN_FIN = 0x39, + ACK_RECEIVE_TIMESTAMPS = 0xB0 }; inline constexpr uint16_t toFrameError(FrameType frame) { @@ -723,5 +725,6 @@ constexpr uint16_t kStreamGroupsEnabledCustomParamId = 0xFF99; // Maximum packet receive timestamps stored. constexpr uint8_t kMaxReceivedPktsTimestampsStored = 5; +constexpr uint8_t kDefaultReceiveTimestampsExponent = 3; } // namespace quic diff --git a/quic/api/QuicPacketScheduler.cpp b/quic/api/QuicPacketScheduler.cpp index e3f802ba4..84597991e 100644 --- a/quic/api/QuicPacketScheduler.cpp +++ b/quic/api/QuicPacketScheduler.cpp @@ -5,8 +5,10 @@ * LICENSE file in the root directory of this source tree. */ +#include #include #include +#include namespace { using namespace quic; @@ -589,8 +591,39 @@ folly::Optional AckScheduler::writeNextAcks( ? std::chrono::duration_cast( ackingTime - receivedTime) : 0us); - AckFrameMetaData meta(ackState_.acks, ackDelay, ackDelayExponentToUse); - auto ackWriteResult = writeAckFrame(meta, builder); + + AckFrameMetaData meta = { + ackState_, /* ackState*/ + ackDelay, /* ackDelay */ + static_cast(ackDelayExponentToUse), /* ackDelayExponent */ + conn_.connectionTime, /* connect timestamp */ + folly::none, /* recvTimestampsConfig */ + folly::none /* maxAckReceiveTimestampsToSend */}; + + folly::Optional ackWriteResult; + + bool isAckReceiveTimestampsSupported = + conn_.transportSettings.maybeAckReceiveTimestampsConfigSentToPeer && + conn_.maybePeerAckReceiveTimestampsConfig; + + uint64_t peerRequestedTimestampsCount = + conn_.maybePeerAckReceiveTimestampsConfig.has_value() + ? conn_.maybePeerAckReceiveTimestampsConfig.value() + .maxReceiveTimestampsPerAck + : 0; + + // If ack_receive_timestamps are not enabled on *either* end-points OR + // the peer requests 0 timestamps, we fall-back to using FrameType::ACK + if (!isAckReceiveTimestampsSupported || !peerRequestedTimestampsCount) { + ackWriteResult = writeAckFrame(meta, builder, FrameType::ACK); + } else { + meta.recvTimestampsConfig = + conn_.transportSettings.maybeAckReceiveTimestampsConfigSentToPeer + .value(); + meta.maxAckReceiveTimestampsToSend = peerRequestedTimestampsCount; + ackWriteResult = writeAckFrameWithReceivedTimestamps( + meta, builder, FrameType::ACK_RECEIVE_TIMESTAMPS); + } if (!ackWriteResult) { return folly::none; } diff --git a/quic/api/test/QuicPacketSchedulerTest.cpp b/quic/api/test/QuicPacketSchedulerTest.cpp index cc69c2fe0..da0bc5a78 100644 --- a/quic/api/test/QuicPacketSchedulerTest.cpp +++ b/quic/api/test/QuicPacketSchedulerTest.cpp @@ -646,7 +646,11 @@ TEST_F(QuicPacketSchedulerTest, WriteOnlyOutstandingPacketsTest) { AckBlocks ackBlocks; ackBlocks.insert(10, 100); ackBlocks.insert(200, 1000); - AckFrameMetaData ackMeta(ackBlocks, 0us, kDefaultAckDelayExponent); + WriteAckState writeAckState = {.acks = ackBlocks}; + AckFrameMetaData ackMeta = { + .ackState = writeAckState, + .ackDelay = 0us, + .ackDelayExponent = static_cast(kDefaultAckDelayExponent)}; // Write those framses with a regular builder writeFrame(connCloseFrame, regularBuilder); diff --git a/quic/client/QuicClientTransport.cpp b/quic/client/QuicClientTransport.cpp index 8f69eaabc..0b883abb1 100644 --- a/quic/client/QuicClientTransport.cpp +++ b/quic/client/QuicClientTransport.cpp @@ -90,7 +90,9 @@ QuicClientTransport::QuicClientTransport( } conn_->readCodec->setCodecParameters(CodecParameters( - conn_->peerAckDelayExponent, conn_->originalVersion.value())); + conn_->peerAckDelayExponent, + conn_->originalVersion.value(), + conn_->transportSettings.maybeAckReceiveTimestampsConfigSentToPeer)); VLOG(10) << "client created " << *conn_; } @@ -1016,7 +1018,6 @@ void QuicClientTransport::startCryptoHandshake() { setD6DProbeTimeoutTransportParameter(); setSupportedExtensionTransportParameters(); maybeEnableStreamGroups(); - auto paramsExtension = std::make_shared( conn_->originalVersion.value(), conn_->transportSettings.advertisedInitialConnectionWindowSize, @@ -1735,6 +1736,35 @@ void QuicClientTransport::setSupportedExtensionTransportParameters() { conn_->datagramState.maxReadFrameSize); customTransportParameters_.push_back(maxDatagramFrameSize->encode()); } + + auto ackReceiveTimestampsEnabled = + std::make_unique( + static_cast( + TransportParameterId::ack_receive_timestamps_enabled), + conn_->transportSettings.maybeAckReceiveTimestampsConfigSentToPeer + .has_value() + ? 1 + : 0); + customTransportParameters_.push_back(ackReceiveTimestampsEnabled->encode()); + if (conn_->transportSettings.maybeAckReceiveTimestampsConfigSentToPeer + .has_value()) { + auto maxReceiveTimestampsPerAck = + std::make_unique( + static_cast( + TransportParameterId::max_receive_timestamps_per_ack), + conn_->transportSettings.maybeAckReceiveTimestampsConfigSentToPeer + .value() + .max_receive_timestamps_per_ack); + customTransportParameters_.push_back(maxReceiveTimestampsPerAck->encode()); + auto receiveTimestampsExponent = + std::make_unique( + static_cast( + TransportParameterId::receive_timestamps_exponent), + conn_->transportSettings.maybeAckReceiveTimestampsConfigSentToPeer + .value() + .receive_timestamps_exponent); + customTransportParameters_.push_back(receiveTimestampsExponent->encode()); + } } void QuicClientTransport::adjustGROBuffers() { diff --git a/quic/client/state/ClientStateMachine.cpp b/quic/client/state/ClientStateMachine.cpp index 7e5676e9e..699989421 100644 --- a/quic/client/state/ClientStateMachine.cpp +++ b/quic/client/state/ClientStateMachine.cpp @@ -10,7 +10,9 @@ #include #include +#include #include +#include #include #include #include @@ -54,7 +56,9 @@ std::unique_ptr undoAllClientStateForRetry( newConn->readCodec = std::make_unique(QuicNodeType::Client); newConn->readCodec->setClientConnectionId(*conn->clientConnectionId); newConn->readCodec->setCodecParameters(CodecParameters( - conn->peerAckDelayExponent, conn->originalVersion.value())); + conn->peerAckDelayExponent, + conn->originalVersion.value(), + conn->transportSettings.maybeAckReceiveTimestampsConfigSentToPeer)); newConn->earlyDataAppParamsValidator = std::move(conn->earlyDataAppParamsValidator); newConn->earlyDataAppParamsGetter = std::move(conn->earlyDataAppParamsGetter); @@ -134,6 +138,15 @@ void processServerInitialParams( auto minAckDelay = getIntegerParameter( TransportParameterId::min_ack_delay, serverParams.parameters); + auto isAckReceiveTimestampsEnabled = getIntegerParameter( + TransportParameterId::ack_receive_timestamps_enabled, + serverParams.parameters); + auto maxReceiveTimestampsPerAck = getIntegerParameter( + TransportParameterId::max_receive_timestamps_per_ack, + serverParams.parameters); + auto receiveTimestampsExponent = getIntegerParameter( + TransportParameterId::receive_timestamps_exponent, + serverParams.parameters); if (conn.version == QuicVersion::QUIC_DRAFT || conn.version == QuicVersion::QUIC_V1 || conn.version == QuicVersion::QUIC_V1_ALIAS) { @@ -224,7 +237,7 @@ void processServerInitialParams( : conn.transportSettings.advertisedInitialBidiRemoteStreamWindowSize; handleStreamWindowUpdate(s, windowSize, packetNum); }); - if (maxDatagramFrameSize.hasValue()) { + if (maxDatagramFrameSize.has_value()) { if (maxDatagramFrameSize.value() > 0 && maxDatagramFrameSize.value() <= kMaxDatagramPacketOverhead) { throw QuicTransportException( @@ -237,6 +250,20 @@ void processServerInitialParams( if (peerMaxStreamGroupsAdvertized) { conn.peerMaxStreamGroupsAdvertized = *peerMaxStreamGroupsAdvertized; } + + if (isAckReceiveTimestampsEnabled.has_value() && + isAckReceiveTimestampsEnabled.value() == 1) { + if (maxReceiveTimestampsPerAck.has_value() && + receiveTimestampsExponent.has_value()) { + conn.maybePeerAckReceiveTimestampsConfig.assign( + {std::min( + static_cast(maxReceiveTimestampsPerAck.value()), + kMaxReceivedPktsTimestampsStored), + std::max( + static_cast(receiveTimestampsExponent.value()), + static_cast(0))}); + } + } } void cacheServerInitialParams( diff --git a/quic/codec/Decode.cpp b/quic/codec/Decode.cpp index cb36f6ad2..f724cebe8 100644 --- a/quic/codec/Decode.cpp +++ b/quic/codec/Decode.cpp @@ -8,9 +8,13 @@ #include #include +#include #include #include #include +#include +#include +#include namespace { @@ -140,11 +144,41 @@ ImmediateAckFrame decodeImmediateAckFrame(folly::io::Cursor&) { return ImmediateAckFrame(); } +uint64_t computeAdjustedDelay( + FrameType frameType, + uint8_t exponentToUse, + folly::Optional> delay) { + // ackDelayExponentToUse is guaranteed to be less than the size of uint64_t + uint64_t delayOverflowMask = 0xFFFFFFFFFFFFFFFF; + uint8_t leftShift = (sizeof(delay->first) * 8 - exponentToUse); + DCHECK_LT(leftShift, sizeof(delayOverflowMask) * 8); + delayOverflowMask = delayOverflowMask << leftShift; + if ((delay->first & delayOverflowMask) != 0) { + throw QuicTransportException( + "Decoded delay overflows", + quic::TransportErrorCode::FRAME_ENCODING_ERROR, + frameType); + } + uint64_t adjustedDelay = delay->first << exponentToUse; + if (adjustedDelay > + static_cast( + std::numeric_limits::max())) { + throw QuicTransportException( + "Bad delay", quic::TransportErrorCode::FRAME_ENCODING_ERROR, frameType); + } else if (UNLIKELY(adjustedDelay > 1000 * 1000 * 1000 /* 1000s */)) { + LOG(ERROR) << "Quic recvd long ack delay=" << adjustedDelay; + adjustedDelay = 0; + } + return adjustedDelay; +} + ReadAckFrame decodeAckFrame( folly::io::Cursor& cursor, const PacketHeader& header, - const CodecParameters& params) { + const CodecParameters& params, + FrameType frameType) { ReadAckFrame frame; + frame.frameType = frameType; auto largestAckedInt = decodeQuicInteger(cursor); if (!largestAckedInt) { throw QuicTransportException( @@ -182,33 +216,12 @@ ReadAckFrame decodeAckFrame( ? kDefaultAckDelayExponent : params.peerAckDelayExponent; DCHECK_LT(ackDelayExponentToUse, sizeof(ackDelay->first) * 8); - // ackDelayExponentToUse is guaranteed to be less than the size of uint64_t - uint64_t delayOverflowMask = 0xFFFFFFFFFFFFFFFF; - uint8_t leftShift = (sizeof(ackDelay->first) * 8 - ackDelayExponentToUse); - DCHECK_LT(leftShift, sizeof(delayOverflowMask) * 8); - delayOverflowMask = delayOverflowMask << leftShift; - if ((ackDelay->first & delayOverflowMask) != 0) { - throw QuicTransportException( - "Decoded ack delay overflows", - quic::TransportErrorCode::FRAME_ENCODING_ERROR, - quic::FrameType::ACK); - } - uint64_t adjustedAckDelay = ackDelay->first << ackDelayExponentToUse; - if (adjustedAckDelay > - static_cast( - std::numeric_limits::max())) { - throw QuicTransportException( - "Bad ack delay", - quic::TransportErrorCode::FRAME_ENCODING_ERROR, - quic::FrameType::ACK); - } else if (UNLIKELY(adjustedAckDelay > 1000 * 1000 * 1000 /* 1000s */)) { - LOG(ERROR) << "Quic recvd long ack delay=" << adjustedAckDelay; - adjustedAckDelay = 0; - } + PacketNum currentPacketNum = nextAckedPacketLen(largestAcked, firstAckBlockLen->first); frame.largestAcked = largestAcked; - frame.ackDelay = std::chrono::microseconds(adjustedAckDelay); + frame.ackDelay = std::chrono::microseconds( + computeAdjustedDelay(frameType, ackDelayExponentToUse, ackDelay)); frame.ackBlocks.emplace_back(currentPacketNum, largestAcked); for (uint64_t numBlocks = 0; numBlocks < additionalAckBlocks->first; ++numBlocks) { @@ -233,6 +246,84 @@ ReadAckFrame decodeAckFrame( // already would have processed it in the previous iteration. frame.ackBlocks.emplace_back(currentPacketNum, nextEndPacket); } + + return frame; +} + +ReadAckFrame decodeAckFrameWithReceivedTimestamps( + folly::io::Cursor& cursor, + const PacketHeader& header, + const CodecParameters& params, + FrameType frameType) { + ReadAckFrame frame; + + frame = decodeAckFrame(cursor, header, params, frameType); + + auto latestRecvdPacketNum = decodeQuicInteger(cursor); + if (!latestRecvdPacketNum) { + throw QuicTransportException( + "Bad latest received packet number", + quic::TransportErrorCode::FRAME_ENCODING_ERROR, + quic::FrameType::ACK_RECEIVE_TIMESTAMPS); + } + frame.maybeLatestRecvdPacketNum = latestRecvdPacketNum->first; + + auto latestRecvdPacketTimeDelta = decodeQuicInteger(cursor); + if (!latestRecvdPacketTimeDelta) { + throw QuicTransportException( + "Bad receive packet timestamp delta", + quic::TransportErrorCode::FRAME_ENCODING_ERROR, + quic::FrameType::ACK_RECEIVE_TIMESTAMPS); + } + frame.maybeLatestRecvdPacketTime = + std::chrono::microseconds(latestRecvdPacketTimeDelta->first); + + auto timeStampRangeCount = decodeQuicInteger(cursor); + if (!timeStampRangeCount) { + throw QuicTransportException( + "Bad receive timestamps range count", + quic::TransportErrorCode::FRAME_ENCODING_ERROR, + quic::FrameType::ACK_RECEIVE_TIMESTAMPS); + } + for (uint64_t numRanges = 0; numRanges < timeStampRangeCount->first; + numRanges++) { + RecvdPacketsTimestampsRange timeStampRange; + auto receiveTimeStampsGap = decodeQuicInteger(cursor); + if (!receiveTimeStampsGap) { + throw QuicTransportException( + "Bad receive timestamps gap", + quic::TransportErrorCode::FRAME_ENCODING_ERROR, + quic::FrameType::ACK_RECEIVE_TIMESTAMPS); + } + timeStampRange.gap = receiveTimeStampsGap->first; + auto receiveTimeStampsLen = decodeQuicInteger(cursor); + if (!receiveTimeStampsLen) { + throw QuicTransportException( + "Bad receive timestamps block length", + quic::TransportErrorCode::FRAME_ENCODING_ERROR, + quic::FrameType::ACK_RECEIVE_TIMESTAMPS); + } + timeStampRange.timestamp_delta_count = receiveTimeStampsLen->first; + uint8_t receiveTimestampsExponentToUse = + (params.maybeAckReceiveTimestampsConfig) + ? params.maybeAckReceiveTimestampsConfig.value() + .receive_timestamps_exponent + : kDefaultReceiveTimestampsExponent; + for (uint64_t i = 0; i < receiveTimeStampsLen->first; i++) { + auto delta = decodeQuicInteger(cursor); + if (!delta) { + throw QuicTransportException( + "Bad receive timestamps delta", + quic::TransportErrorCode::FRAME_ENCODING_ERROR, + quic::FrameType::ACK_RECEIVE_TIMESTAMPS); + } + DCHECK_LT(receiveTimestampsExponentToUse, sizeof(delta->first) * 8); + auto adjustedDelta = computeAdjustedDelay( + frameType, receiveTimestampsExponentToUse, delta); + timeStampRange.deltas.push_back(adjustedDelta); + } + frame.recvdPacketsTimestampRanges.emplace_back(timeStampRange); + } return frame; } @@ -754,7 +845,9 @@ QuicFrame parseFrame( }; cursor.reset(queue.front()); FrameType frameType = static_cast(frameTypeInt->first); - try { + try + + { switch (frameType) { case FrameType::PADDING: return QuicFrame(decodePaddingFrame(cursor)); @@ -842,6 +935,10 @@ QuicFrame parseFrame( return QuicFrame(decodeAckFrequencyFrame(cursor)); case FrameType::IMMEDIATE_ACK: return QuicFrame(decodeImmediateAckFrame(cursor)); + case FrameType::ACK_RECEIVE_TIMESTAMPS: + auto frame = QuicFrame(decodeAckFrameWithReceivedTimestamps( + cursor, header, params, FrameType::ACK_RECEIVE_TIMESTAMPS)); + return frame; } } catch (const std::exception& e) { error = true; diff --git a/quic/codec/Decode.h b/quic/codec/Decode.h index 670d25a17..809654376 100644 --- a/quic/codec/Decode.h +++ b/quic/codec/Decode.h @@ -8,8 +8,10 @@ #pragma once #include +#include #include #include +#include namespace quic { @@ -21,13 +23,23 @@ struct CodecParameters { // This must not be set to zero. uint8_t peerAckDelayExponent{kDefaultAckDelayExponent}; QuicVersion version{QuicVersion::MVFST}; + folly::Optional maybeAckReceiveTimestampsConfig = + folly::none; CodecParameters() = default; + CodecParameters( + uint8_t peerAckDelayExponentIn, + QuicVersion versionIn, + folly::Optional + maybeAckReceiveTimestampsConfigIn) + : peerAckDelayExponent(peerAckDelayExponentIn), + version(versionIn), + maybeAckReceiveTimestampsConfig(maybeAckReceiveTimestampsConfigIn) {} + CodecParameters(uint8_t peerAckDelayExponentIn, QuicVersion versionIn) : peerAckDelayExponent(peerAckDelayExponentIn), version(versionIn) {} }; - struct ParsedLongHeaderInvariant { uint8_t initialByte; LongHeaderInvariant invariant; @@ -117,7 +129,14 @@ PathResponseFrame decodePathResponseFrame(folly::io::Cursor& cursor); ReadAckFrame decodeAckFrame( folly::io::Cursor& cursor, const PacketHeader& header, - const CodecParameters& params); + const CodecParameters& params, + FrameType frameType = FrameType::ACK); + +ReadAckFrame decodeAckFrameWithReceivedTimestamps( + folly::io::Cursor& cursor, + const PacketHeader& header, + const CodecParameters& params, + FrameType frameType); ReadAckFrame decodeAckFrameWithECN( folly::io::Cursor& cursor, @@ -143,8 +162,8 @@ DatagramFrame decodeDatagramFrame(BufQueue& queue, bool hasLen); /** * Parse the Invariant fields in Long Header. * - * cursor: points to the byte just past initialByte. After parsing, cursor will - * be moved to the byte right after Source Connection ID. + * cursor: points to the byte just past initialByte. After parsing, cursor + * will be moved to the byte right after Source Connection ID. */ folly::Expected parseLongHeaderInvariant(uint8_t initalByte, folly::io::Cursor& cursor); @@ -213,4 +232,9 @@ folly::Expected parseShortHeader( uint8_t initialByte, folly::io::Cursor& cursor, size_t dstConnIdSize = kDefaultConnectionIdSize); + +uint64_t computeAdjustedDelay( + FrameType frameType, + uint8_t exponentToUse, + folly::Optional> delay); } // namespace quic diff --git a/quic/codec/QuicPacketRebuilder.cpp b/quic/codec/QuicPacketRebuilder.cpp index d6787fb00..9f6b51372 100644 --- a/quic/codec/QuicPacketRebuilder.cpp +++ b/quic/codec/QuicPacketRebuilder.cpp @@ -228,9 +228,38 @@ folly::Optional PacketRebuilder::rebuildFromPacket( ? std::chrono::duration_cast( ackingTime - receivedTime) : 0us); - AckFrameMetaData meta(ackState_.acks, ackDelay, ackDelayExponent); + + AckFrameMetaData meta = { + ackState_, /* ackState*/ + ackDelay, /* ackDelay */ + static_cast(ackDelayExponent), /* ackDelayExponent */ + conn_.connectionTime, /* connect timestamp */ + folly::none, /* recvTimestampsConfig */ + folly::none /* maxAckReceiveTimestampsToSend */}; + + folly::Optional ackWriteResult; + + uint64_t peerRequestedTimestampsCount = + conn_.maybePeerAckReceiveTimestampsConfig.has_value() + ? conn_.maybePeerAckReceiveTimestampsConfig.value() + .maxReceiveTimestampsPerAck + : 0; + // Write the AckFrame ignoring the result. This is best-effort. - writeAckFrame(meta, builder_); + bool isAckReceiveTimestampsSupported = + conn_.transportSettings.maybeAckReceiveTimestampsConfigSentToPeer && + conn_.maybePeerAckReceiveTimestampsConfig; + + if (!isAckReceiveTimestampsSupported || !peerRequestedTimestampsCount) { + writeAckFrame(meta, builder_, FrameType::ACK); + } else { + meta.recvTimestampsConfig = + conn_.transportSettings.maybeAckReceiveTimestampsConfigSentToPeer + .value(); + meta.maxAckReceiveTimestampsToSend = peerRequestedTimestampsCount; + writeAckFrameWithReceivedTimestamps( + meta, builder_, FrameType::ACK_RECEIVE_TIMESTAMPS); + } } // We shouldn't clone if: // (1) we only end up cloning only acks, ping, or paddings. diff --git a/quic/codec/QuicWriteCodec.cpp b/quic/codec/QuicWriteCodec.cpp index 193e394c0..6304da4fb 100644 --- a/quic/codec/QuicWriteCodec.cpp +++ b/quic/codec/QuicWriteCodec.cpp @@ -12,6 +12,8 @@ #include #include #include +#include +#include namespace { @@ -265,22 +267,131 @@ static size_t fillFrameWithAckBlocks( return numAdditionalAckBlocks; } -folly::Optional writeAckFrame( +size_t computeSizeUsedByRecvdTimestamps(WriteAckFrame& ackFrame) { + size_t usedSize = 0; + for (auto& recvdPacketsTimestampRanges : + ackFrame.recvdPacketsTimestampRanges) { + usedSize += getQuicIntegerSizeThrows(recvdPacketsTimestampRanges.gap); + usedSize += getQuicIntegerSizeThrows( + recvdPacketsTimestampRanges.timestamp_delta_count); + for (auto& timestampDelta : recvdPacketsTimestampRanges.deltas) { + usedSize += getQuicIntegerSizeThrows(timestampDelta); + } + } + return usedSize; +} + +static size_t fillPacketReceiveTimestamps( const quic::AckFrameMetaData& ackFrameMetaData, - PacketBuilderInterface& builder) { - if (ackFrameMetaData.ackBlocks.empty()) { + WriteAckFrame& ackFrame, + uint64_t largestAckedPacketNum, + uint64_t spaceLeft, + uint64_t receiveTimestampsExponent) { + if (ackFrameMetaData.ackState.recvdPacketInfos.size() == 0) { + return 0; + } + auto recvdPacketInfos = ackFrameMetaData.ackState.recvdPacketInfos; + // Insert all received packet timestamps into an interval set, to identify + // continguous ranges + + DCHECK(ackFrameMetaData.maxAckReceiveTimestampsToSend.has_value()); + auto maxRecvTimestampsToSend = + ackFrameMetaData.maxAckReceiveTimestampsToSend.value(); + uint64_t pktsAdded = 0; + IntervalSet receivedPktNumsIntervalSet; + for (auto& recvdPkt : recvdPacketInfos) { + // Add up to the peer requested max ack receive timestamps; + if (pktsAdded == maxRecvTimestampsToSend) { + break; + } + receivedPktNumsIntervalSet.insert(recvdPkt.pktNum); + pktsAdded++; + } + auto prevPktNum = largestAckedPacketNum; + auto timestampIt = recvdPacketInfos.crbegin(); + size_t cumUsedSpace = 0; + // We start from the latest timestamp intervals + bool outOfSpace = false; + for (auto timestampIntervalsIt = receivedPktNumsIntervalSet.crbegin(); + timestampIntervalsIt != receivedPktNumsIntervalSet.crend(); + timestampIntervalsIt++) { + RecvdPacketsTimestampsRange nextTimestampRange; + size_t nextTimestampRangeUsedSpace = 0; + // Compute pktNum gap for each time-stamp range + if (ackFrame.recvdPacketsTimestampRanges.empty()) { + nextTimestampRange.gap = prevPktNum - timestampIntervalsIt->end; + } else { + nextTimestampRange.gap = prevPktNum - 2 - timestampIntervalsIt->end; + } + // Intialize spaced used by the next candidate time-stamp range + nextTimestampRangeUsedSpace += + getQuicIntegerSizeThrows(nextTimestampRange.gap); + + while (timestampIt != recvdPacketInfos.crend() && + timestampIt->pktNum >= timestampIntervalsIt->start && + timestampIt->pktNum <= timestampIntervalsIt->end) { + std::chrono::microseconds deltaDuration; + if (timestampIt == recvdPacketInfos.crbegin()) { + deltaDuration = (timestampIt->timeStamp > ackFrameMetaData.connTime) + ? std::chrono::duration_cast( + timestampIt->timeStamp - ackFrameMetaData.connTime) + : 0us; + } else { + deltaDuration = std::chrono::duration_cast( + (timestampIt - 1)->timeStamp - timestampIt->timeStamp); + } + auto delta = deltaDuration.count() >> receiveTimestampsExponent; + // Check if adding a new time-stamp delta from the current time-stamp + // interval Will allow us to run out of space. Since adding a new delta + // impacts cumulative counts of deltas these are not already incorporated + // into nextTimestampRangeUsedSpace. + if (spaceLeft < + (cumUsedSpace + nextTimestampRangeUsedSpace + + getQuicIntegerSizeThrows(delta) + + getQuicIntegerSizeThrows(nextTimestampRange.deltas.size() + 1) + + getQuicIntegerSizeThrows( + ackFrame.recvdPacketsTimestampRanges.size() + 1))) { + outOfSpace = true; + break; + } + nextTimestampRange.deltas.push_back(delta); + nextTimestampRangeUsedSpace += getQuicIntegerSizeThrows(delta); + timestampIt++; + } + if (nextTimestampRange.deltas.size() > 0) { + nextTimestampRange.timestamp_delta_count = + nextTimestampRange.deltas.size(); + cumUsedSpace += nextTimestampRangeUsedSpace + + getQuicIntegerSizeThrows(nextTimestampRange.deltas.size()); + ackFrame.recvdPacketsTimestampRanges.push_back(nextTimestampRange); + prevPktNum = timestampIntervalsIt->start; + DCHECK(cumUsedSpace <= spaceLeft); + } + if (outOfSpace) { + break; + } + } + DCHECK(cumUsedSpace == computeSizeUsedByRecvdTimestamps(ackFrame)); + return ackFrame.recvdPacketsTimestampRanges.size(); +} + +folly::Optional writeAckFrameToPacketBuilder( + const quic::AckFrameMetaData& ackFrameMetaData, + PacketBuilderInterface& builder, + FrameType frameType) { + if (ackFrameMetaData.ackState.acks.empty()) { return folly::none; } + const WriteAckState& ackState = ackFrameMetaData.ackState; // The last block must be the largest block. - auto largestAckedPacket = ackFrameMetaData.ackBlocks.back().end; + auto largestAckedPacket = ackState.acks.back().end; // ackBlocks are already an interval set so each value is naturally // non-overlapping. - auto firstAckBlockLength = - largestAckedPacket - ackFrameMetaData.ackBlocks.back().start; + auto firstAckBlockLength = largestAckedPacket - ackState.acks.back().start; WriteAckFrame ackFrame; + ackFrame.frameType = frameType; uint64_t spaceLeft = builder.remainingSpaceInPkt(); - uint64_t beginningSpace = spaceLeft; ackFrame.ackBlocks.reserve(spaceLeft / 4); // We could technically split the range if the size of the representation of @@ -297,18 +408,42 @@ folly::Optional writeAckFrame( // Required fields are Type, LargestAcked, AckDelay, AckBlockCount, // firstAckBlockLength - QuicInteger encodedintFrameType(static_cast(FrameType::ACK)); + QuicInteger encodedintFrameType(static_cast(frameType)); auto headerSize = encodedintFrameType.getSize() + largestAckedPacketInt.getSize() + ackDelayInt.getSize() + minAdditionalAckBlockCount.getSize() + firstAckBlockLengthInt.getSize(); - if (spaceLeft < headerSize) { + + size_t minAdditionalAckReceiveTimestampsFieldsSize = 0; + if (frameType == FrameType::ACK_RECEIVE_TIMESTAMPS) { + // Compute minimum size requirements for 3 fields that must be sent + // in every ACK_RECEIVE_TIMESTAMPS frame + uint64_t countTimestampRanges = 0; + uint64_t maybeLastPktNum = 0; + std::chrono::microseconds maybeLastPktTsDelta = 0us; + if (ackState.lastRecvdPacketInfo) + maybeLastPktNum = ackState.lastRecvdPacketInfo.value().pktNum; + + maybeLastPktTsDelta = + (ackState.lastRecvdPacketInfo.value().timeStamp > + ackFrameMetaData.connTime + ? std::chrono::duration_cast( + ackState.lastRecvdPacketInfo.value().timeStamp - + ackFrameMetaData.connTime) + : 0us); + + minAdditionalAckReceiveTimestampsFieldsSize = + getQuicIntegerSize(countTimestampRanges).value_or(0) + + getQuicIntegerSize(maybeLastPktNum).value_or(0) + + getQuicIntegerSize(maybeLastPktTsDelta.count()).value_or(0); + } + if (spaceLeft < (headerSize + minAdditionalAckReceiveTimestampsFieldsSize)) { return folly::none; } - spaceLeft -= headerSize; + spaceLeft -= (headerSize + minAdditionalAckReceiveTimestampsFieldsSize); - ackFrame.ackBlocks.push_back(ackFrameMetaData.ackBlocks.back()); + ackFrame.ackBlocks.push_back(ackState.acks.back()); auto numAdditionalAckBlocks = - fillFrameWithAckBlocks(ackFrameMetaData.ackBlocks, ackFrame, spaceLeft); + fillFrameWithAckBlocks(ackState.acks, ackFrame, spaceLeft); QuicInteger numAdditionalAckBlocksInt(numAdditionalAckBlocks); builder.write(encodedintFrameType); @@ -317,7 +452,7 @@ folly::Optional writeAckFrame( builder.write(numAdditionalAckBlocksInt); builder.write(firstAckBlockLengthInt); - PacketNum currentSeqNum = ackFrameMetaData.ackBlocks.back().start; + PacketNum currentSeqNum = ackState.acks.back().start; for (auto it = ackFrame.ackBlocks.cbegin() + 1; it != ackFrame.ackBlocks.cend(); ++it) { @@ -331,10 +466,108 @@ folly::Optional writeAckFrame( currentSeqNum = it->start; } ackFrame.ackDelay = ackFrameMetaData.ackDelay; - builder.appendFrame(std::move(ackFrame)); - return AckFrameWriteResult( + return ackFrame; +} + +folly::Optional writeAckFrame( + const quic::AckFrameMetaData& ackFrameMetaData, + PacketBuilderInterface& builder, + FrameType frameType) { + uint64_t beginningSpace = builder.remainingSpaceInPkt(); + auto maybeWriteAckFrame = + writeAckFrameToPacketBuilder(ackFrameMetaData, builder, frameType); + + if (maybeWriteAckFrame.has_value()) { + builder.appendFrame(std::move(maybeWriteAckFrame.value())); + return AckFrameWriteResult( + beginningSpace - builder.remainingSpaceInPkt(), + maybeWriteAckFrame.value(), + maybeWriteAckFrame.value().ackBlocks.size()); + } else { + return folly::none; + } +} + +folly::Optional writeAckFrameWithReceivedTimestamps( + const quic::AckFrameMetaData& ackFrameMetaData, + PacketBuilderInterface& builder, + FrameType frameType) { + auto beginningSpace = builder.remainingSpaceInPkt(); + auto maybeAckFrame = + writeAckFrameToPacketBuilder(ackFrameMetaData, builder, frameType); + if (!maybeAckFrame.has_value()) { + return folly::none; + } + auto ackFrame = maybeAckFrame.value(); + const WriteAckState& ackState = ackFrameMetaData.ackState; + uint64_t spaceLeft = builder.remainingSpaceInPkt(); + uint64_t lastPktNum = 0; + std::chrono::microseconds lastPktTsDelta = 0us; + if (ackState.lastRecvdPacketInfo) { + lastPktNum = ackState.lastRecvdPacketInfo.value().pktNum; + lastPktTsDelta = + (ackState.lastRecvdPacketInfo.value().timeStamp > + ackFrameMetaData.connTime + ? std::chrono::duration_cast( + ackState.lastRecvdPacketInfo.value().timeStamp - + ackFrameMetaData.connTime) + : 0us); + } + QuicInteger lastRecvdPacketNumInt(lastPktNum); + builder.write(lastRecvdPacketNumInt); + ackFrame.maybeLatestRecvdPacketNum = lastRecvdPacketNumInt.getValue(); + QuicInteger lastRecvdPacketTimeInt(lastPktTsDelta.count()); + builder.write(lastRecvdPacketTimeInt); + ackFrame.maybeLatestRecvdPacketTime = + std::chrono::microseconds(lastRecvdPacketTimeInt.getValue()); + + size_t countTimestampRanges = 0; + size_t countTimestamps = 0; + spaceLeft = builder.remainingSpaceInPkt(); + if (spaceLeft > 0) { + auto largestAckedPacket = ackState.acks.back().end; + uint8_t receiveTimestampsExponentToUse = + ackFrameMetaData.recvTimestampsConfig.has_value() + ? ackFrameMetaData.recvTimestampsConfig.value() + .receive_timestamps_exponent + : kDefaultReceiveTimestampsExponent; + countTimestampRanges = fillPacketReceiveTimestamps( + ackFrameMetaData, + ackFrame, + largestAckedPacket, + spaceLeft, + receiveTimestampsExponentToUse); + if (countTimestampRanges > 0) { + QuicInteger timeStampRangeCountInt( + ackFrame.recvdPacketsTimestampRanges.size()); + builder.write(timeStampRangeCountInt); + for (auto& recvdPacketsTimestampRanges : + ackFrame.recvdPacketsTimestampRanges) { + QuicInteger gapInt(recvdPacketsTimestampRanges.gap); + QuicInteger timestampDeltaCountInt( + recvdPacketsTimestampRanges.timestamp_delta_count); + builder.write(gapInt); + builder.write(timestampDeltaCountInt); + for (auto& timestamp : recvdPacketsTimestampRanges.deltas) { + QuicInteger deltaInt(timestamp); + builder.write(deltaInt); + countTimestamps++; + } + } + } else { + QuicInteger timeStampRangeCountInt(0); + builder.write(timeStampRangeCountInt); + } + } + auto ackFrameWriteResult = AckFrameWriteResult( beginningSpace - builder.remainingSpaceInPkt(), - 1 + numAdditionalAckBlocks); + ackFrame, + ackFrame.ackBlocks.size(), + countTimestampRanges, + countTimestamps); + + builder.appendFrame(std::move(ackFrame)); + return ackFrameWriteResult; } size_t writeSimpleFrame( diff --git a/quic/codec/QuicWriteCodec.h b/quic/codec/QuicWriteCodec.h index 9e805a443..125cf17a9 100644 --- a/quic/codec/QuicWriteCodec.h +++ b/quic/codec/QuicWriteCodec.h @@ -7,37 +7,85 @@ #pragma once +#include #include #include +#include #include +#include +#include #include +#include namespace quic { +// Ack and PacketNumber states. This is per-packet number space. +struct WriteAckState { + AckBlocks acks; + + // Receive timestamp and packet number for the largest received packet. + // + // Updated whenever we receive a packet with a larger packet number + // than all previously received packets in the packet number space + // tracked by this AckState. + folly::Optional largestRecvdPacketInfo; + // Receive timestamp and packet number for the last received packet. + // + // Will be different from the value stored in largestRecvdPacketInfo + // if the last packet was received out of order and thus had a packet + // number less than that of a previously received packet in the packet + // number space tracked by this AckState. + folly::Optional lastRecvdPacketInfo; + + // Packet number and timestamp of recently received packets. + // + // The maximum number of packets stored in pktsReceivedTimestamps is + // controlled by kMaxReceivedPktsTimestampsStored. + // + // The packet number of entries in the deque is guarenteed to increase + // monotonically because an entry is only added for a received packet + // if the packet number is greater than the packet number of the last + // element in the deque (e.g., entries are not added for packets that + // arrive out of order relative to previously received packets). + CircularDeque recvdPacketInfos; +}; + struct AckFrameMetaData { - // Ack blocks. There must be at least 1 ACK block to send. - const AckBlocks& ackBlocks; + // ACK state. + const WriteAckState& ackState; + // Delay in sending ack from time that packet was received. std::chrono::microseconds ackDelay; // The ack delay exponent to use. uint8_t ackDelayExponent; - AckFrameMetaData( - const AckBlocks& acksIn, - std::chrono::microseconds ackDelayIn, - uint8_t ackDelayExponentIn) - : ackBlocks(acksIn), - ackDelay(ackDelayIn), - ackDelayExponent(ackDelayExponentIn) {} + // Receive timestamps basis + TimePoint connTime; + + folly::Optional recvTimestampsConfig = + folly::none; + + folly::Optional maxAckReceiveTimestampsToSend = folly::none; }; struct AckFrameWriteResult { uint64_t bytesWritten; + WriteAckFrame writeAckFrame; // This includes the first ack block size_t ackBlocksWritten; - - AckFrameWriteResult(uint64_t bytesWrittenIn, size_t ackBlocksWrittenIn) - : bytesWritten(bytesWrittenIn), ackBlocksWritten(ackBlocksWrittenIn) {} + size_t timestampRangesWritten; + size_t timestampsWritten; + AckFrameWriteResult( + uint64_t bytesWrittenIn, + WriteAckFrame writeAckFrameIn, + size_t ackBlocksWrittenIn, + size_t timestampRangesWrittenIn = 0, + size_t timestampsWrittenIn = 0) + : bytesWritten(bytesWrittenIn), + writeAckFrame(writeAckFrameIn), + ackBlocksWritten(ackBlocksWrittenIn), + timestampRangesWritten(timestampRangesWrittenIn), + timestampsWritten(timestampsWrittenIn) {} }; /** @@ -136,5 +184,25 @@ folly::Optional writeCryptoFrame( */ folly::Optional writeAckFrame( const AckFrameMetaData& ackFrameMetaData, - PacketBuilderInterface& builder); + PacketBuilderInterface& builder, + FrameType frameType = FrameType::ACK); + +/** + * Helper functions to write the fields for ACK_RECEIVE_TIMESTAMPS frame + */ +size_t computeSizeUsedByRecvdTimestamps(quic::WriteAckFrame& writeAckFrame); + +folly::Optional writeAckFrameWithReceivedTimestamps( + const AckFrameMetaData& ackFrameMetaData, + PacketBuilderInterface& builder, + FrameType frameType = FrameType::ACK_RECEIVE_TIMESTAMPS); + +folly::Optional writeAckFrameToPacketBuilder( + const quic::AckFrameMetaData& ackFrameMetaData, + quic::PacketBuilderInterface& builder, + quic::FrameType frameType); + } // namespace quic +// namespace quic +// namespace quic +// namespace quic diff --git a/quic/codec/Types.cpp b/quic/codec/Types.cpp index 96448b099..fb7a2feb2 100644 --- a/quic/codec/Types.cpp +++ b/quic/codec/Types.cpp @@ -446,6 +446,8 @@ std::string toString(FrameType frame) { case FrameType::GROUP_STREAM_OFF_LEN: case FrameType::GROUP_STREAM_OFF_LEN_FIN: return "GROUP_STREAM"; + case FrameType::ACK_RECEIVE_TIMESTAMPS: + return "ACK_RECEIVE_TIMESTAMPS"; } LOG(WARNING) << "toString has unhandled frame type"; return "UNKNOWN"; diff --git a/quic/codec/Types.h b/quic/codec/Types.h index 69585e296..c1f1cb791 100644 --- a/quic/codec/Types.h +++ b/quic/codec/Types.h @@ -153,6 +153,14 @@ struct AckBlock { : startPacket(start), endPacket(end) {} }; +struct RecvdPacketsTimestampsRange { + uint64_t gap; + uint64_t timestamp_delta_count; + std::vector deltas; +}; + +using RecvdPacketsTimestampsRangeVec = std::vector; + /** 0 1 2 3 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 @@ -182,7 +190,10 @@ struct ReadAckFrame { // These are ordered in descending order by start packet. using Vec = SmallVec; Vec ackBlocks; - + FrameType frameType = FrameType::ACK; + folly::Optional maybeLatestRecvdPacketTime; + folly::Optional maybeLatestRecvdPacketNum; + RecvdPacketsTimestampsRangeVec recvdPacketsTimestampRanges; bool operator==(const ReadAckFrame& /*rhs*/) const { // Can't compare ackBlocks, function is just here to appease compiler. return false; @@ -197,7 +208,10 @@ struct WriteAckFrame { AckBlockVec ackBlocks; // Delay in sending ack from time that packet was received. std::chrono::microseconds ackDelay{0us}; - + FrameType frameType = FrameType::ACK; + folly::Optional maybeLatestRecvdPacketTime; + folly::Optional maybeLatestRecvdPacketNum; + RecvdPacketsTimestampsRangeVec recvdPacketsTimestampRanges; bool operator==(const WriteAckFrame& /*rhs*/) const { // Can't compare ackBlocks, function is just here to appease compiler. return false; diff --git a/quic/codec/test/QuicPacketRebuilderTest.cpp b/quic/codec/test/QuicPacketRebuilderTest.cpp index d8481dc98..afe119b5f 100644 --- a/quic/codec/test/QuicPacketRebuilderTest.cpp +++ b/quic/codec/test/QuicPacketRebuilderTest.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -74,7 +75,12 @@ TEST_F(QuicPacketRebuilderTest, RebuildPacket) { AckBlocks ackBlocks; ackBlocks.insert(10, 100); ackBlocks.insert(200, 1000); - AckFrameMetaData ackMeta(ackBlocks, 0us, kDefaultAckDelayExponent); + WriteAckState writeAckState = {.acks = ackBlocks}; + // AckFrameMetaData ackMeta(ackBlocks, 0us, kDefaultAckDelayExponent); + AckFrameMetaData ackMeta = { + .ackState = writeAckState, + .ackDelay = 0us, + .ackDelayExponent = static_cast(kDefaultAckDelayExponent)}; QuicServerConnectionState conn( FizzServerQuicHandshakeContext::Builder().build()); conn.streamManager->setMaxLocalBidirectionalStreams(10); @@ -401,7 +407,11 @@ TEST_F(QuicPacketRebuilderTest, CannotRebuild) { AckBlocks ackBlocks; ackBlocks.insert(10, 100); ackBlocks.insert(200, 1000); - AckFrameMetaData ackMeta(ackBlocks, 0us, kDefaultAckDelayExponent); + WriteAckState writeAckState = {.acks = ackBlocks}; + AckFrameMetaData ackMeta = { + .ackState = writeAckState, + .ackDelay = 0us, + .ackDelayExponent = static_cast(kDefaultAckDelayExponent)}; QuicServerConnectionState conn( FizzServerQuicHandshakeContext::Builder().build()); conn.streamManager->setMaxLocalBidirectionalStreams(10); diff --git a/quic/codec/test/QuicWriteCodecTest.cpp b/quic/codec/test/QuicWriteCodecTest.cpp index 7b8d7828e..0d1c55062 100644 --- a/quic/codec/test/QuicWriteCodecTest.cpp +++ b/quic/codec/test/QuicWriteCodecTest.cpp @@ -11,12 +11,19 @@ #include #include #include +#include #include #include +#include #include #include #include +#include #include +#include +#include +#include +#include using namespace quic; using namespace quic::test; @@ -26,11 +33,24 @@ ShortHeader buildTestShortHeader() { return ShortHeader(ProtectionType::KeyPhaseZero, getTestConnectionId(), 0x01); } -QuicFrame parseQuicFrame(BufQueue& queue) { +QuicFrame parseQuicFrame( + BufQueue& queue, + bool isAckReceiveTimestampsSupported = false) { + folly::Optional receiveTimeStampsConfig = + folly::none; + if (isAckReceiveTimestampsSupported) { + receiveTimeStampsConfig.assign( + {.max_receive_timestamps_per_ack = 5, + .receive_timestamps_exponent = 3}); + } + return quic::parseFrame( queue, buildTestShortHeader(), - CodecParameters(kDefaultAckDelayExponent, QuicVersion::MVFST)); + CodecParameters( + kDefaultAckDelayExponent, + QuicVersion::MVFST, + receiveTimeStampsConfig)); } namespace quic { @@ -117,8 +137,150 @@ void setupCommonExpects(MockQuicPacketBuilder& pktBuilder) { pktBuilder.remaining_ -= quicInteger.getSize(); })); } +using PacketsReceivedTimestampsDeque = CircularDeque; +const auto kDefaultTimestampsDelta = 10us; +const AckReceiveTimestampsConfig defaultAckReceiveTimestmpsConfig = { + .receive_timestamps_exponent = kDefaultReceiveTimestampsExponent}; +PacketsReceivedTimestampsDeque populateReceiveTimestamps( + const AckBlocks& ackBlocks, + TimePoint connTime, + uint64_t maxTimeStamps = kMaxReceivedPktsTimestampsStored) { + PacketsReceivedTimestampsDeque pktsReceivedTimestamps; -class QuicWriteCodecTest : public Test {}; + uint64_t countTimestamps = 0; + for (auto it = ackBlocks.crbegin(); it != ackBlocks.crend(); it++) { + countTimestamps += (it->end - it->start + 1); + } + auto lastPacketDelta = (countTimestamps * kDefaultTimestampsDelta); + + for (auto it = ackBlocks.crbegin(); it != ackBlocks.crend(); it++) { + for (auto i = it->end; i >= it->start; i--) { + if (pktsReceivedTimestamps.size() < maxTimeStamps) { + RecvdPacketInfo rpi; + rpi.pktNum = i; + auto diff = std::chrono::microseconds( + lastPacketDelta -= kDefaultTimestampsDelta); + if ((connTime + diff) > connTime) { + rpi.timeStamp = connTime + diff; + } else { + rpi.timeStamp = connTime; + } + pktsReceivedTimestamps.emplace_front(rpi); + } else { + break; + } + } + } + return pktsReceivedTimestamps; +} + +size_t computeBytesForAckReceivedTimestamps( + AckFrameMetaData ackFrameMetadata, + AckFrameWriteResult ackFrameWriteResult, + FrameType frameType) { + if (frameType != FrameType::ACK_RECEIVE_TIMESTAMPS) { + return 0; + } + + size_t numRanges = ackFrameWriteResult.timestampRangesWritten; + TimePoint connTime = ackFrameMetadata.connTime; + auto lastPktNum = + ackFrameMetadata.ackState.lastRecvdPacketInfo.value().pktNum; + std::chrono::duration lastTimeStampDelta = + std::chrono::duration_cast( + ackFrameMetadata.ackState.lastRecvdPacketInfo.value().timeStamp - + connTime); + + // When FrameType == ACK_RECEIVE_TIMESTAMPS, the minimum additional + // information that is sent to the peer is: + // 1. last received packet's timestamp delta, + // 2. last received packet's number, + // 3. count of timestamp ranges + size_t sizeConsumed = + 1 + // Additional space for ACK_RECEIVE_TIMESTAMPS packet type + getQuicIntegerSizeThrows( + lastTimeStampDelta + .count()) + // latest received packet timestamp delta + getQuicIntegerSizeThrows(lastPktNum) + // latest received packet number + getQuicIntegerSizeThrows( + numRanges); // count of ack_receive_timestamp ranges + + if (numRanges > 0) { + sizeConsumed += + computeSizeUsedByRecvdTimestamps(ackFrameWriteResult.writeAckFrame); + } + return sizeConsumed; +} + +WriteAckState createTestWriteAckState( + FrameType frameType, + const TimePoint& connTime, + AckBlocks& ackBlocks, + uint64_t countTimestampsToStore = kMaxReceivedPktsTimestampsStored) { + WriteAckState ackState = {.acks = ackBlocks}; + ackState.acks = ackBlocks; + if (frameType == FrameType::ACK_RECEIVE_TIMESTAMPS) { + ackState.recvdPacketInfos = + populateReceiveTimestamps(ackBlocks, connTime, countTimestampsToStore); + ackState.lastRecvdPacketInfo.assign({ + ackState.recvdPacketInfos.back().pktNum, + ackState.recvdPacketInfos.back().timeStamp, + }); + } + return ackState; +} + +void assertsOnDecodedReceiveTimestamps( + const AckFrameMetaData& ackFrameMetaData, + const WriteAckFrame& writeAckFrame, + const ReadAckFrame& readAckFrame, + uint64_t expectedTimestampRangesCount, + uint64_t expectedTimestampsCount) { + EXPECT_TRUE(readAckFrame.maybeLatestRecvdPacketNum.has_value()); + EXPECT_TRUE(readAckFrame.maybeLatestRecvdPacketTime.has_value()); + EXPECT_EQ( + readAckFrame.maybeLatestRecvdPacketNum.value(), + ackFrameMetaData.ackState.lastRecvdPacketInfo.value().pktNum); + EXPECT_EQ( + readAckFrame.maybeLatestRecvdPacketTime.value(), + std::chrono::duration_cast( + ackFrameMetaData.ackState.lastRecvdPacketInfo.value().timeStamp - + ackFrameMetaData.connTime)); + EXPECT_EQ( + readAckFrame.recvdPacketsTimestampRanges.size(), + expectedTimestampRangesCount); + auto timeStamps = 0; + for (auto range : readAckFrame.recvdPacketsTimestampRanges) { + timeStamps += range.timestamp_delta_count; + } + EXPECT_EQ(timeStamps, expectedTimestampsCount); + EXPECT_EQ( + readAckFrame.recvdPacketsTimestampRanges.size(), + writeAckFrame.recvdPacketsTimestampRanges.size()); + // (XXX: sj77, clean this up) + for (uint64_t i = 0; i < readAckFrame.recvdPacketsTimestampRanges.size(); + ++i) { + EXPECT_EQ( + readAckFrame.recvdPacketsTimestampRanges[i].gap, + writeAckFrame.recvdPacketsTimestampRanges[i].gap); + EXPECT_EQ( + readAckFrame.recvdPacketsTimestampRanges[i].timestamp_delta_count, + writeAckFrame.recvdPacketsTimestampRanges[i].timestamp_delta_count); + for (uint64_t j = 0; + j < readAckFrame.recvdPacketsTimestampRanges[i].timestamp_delta_count; + j++) { + EXPECT_EQ( + readAckFrame.recvdPacketsTimestampRanges[i].deltas[j], + writeAckFrame.recvdPacketsTimestampRanges[i].deltas[j] * + pow(2, + ackFrameMetaData.recvTimestampsConfig.value() + .receive_timestamps_exponent)); + } + } +} + +// class QuicWriteCodecTest : public Test {}; +class QuicWriteCodecTest : public TestWithParam {}; TEST_F(QuicWriteCodecTest, WriteStreamFrameToEmptyPacket) { MockQuicPacketBuilder pktBuilder; @@ -715,7 +877,7 @@ TEST_F(QuicWriteCodecTest, WriteStreamFrameHeadeLengthHintFalse) { EXPECT_LT(*dataLen, 1200); } -TEST_F(QuicWriteCodecTest, AckFrameGapExceedsRepresentation) { +TEST_P(QuicWriteCodecTest, AckFrameGapExceedsRepresentation) { MockQuicPacketBuilder pktBuilder; setupCommonExpects(pktBuilder); @@ -723,14 +885,26 @@ TEST_F(QuicWriteCodecTest, AckFrameGapExceedsRepresentation) { // Can't use max directly, because it will exceed interval set's // representation. AckBlocks ackBlocks = {{max - 10, max - 10}, {1, 1}}; + auto frameType = GetParam(); + TimePoint connTime = Clock::now(); + WriteAckState ackState = + createTestWriteAckState(frameType, connTime, ackBlocks); + AckFrameMetaData ackFrameMetaData = { + .ackState = ackState, + .ackDelay = 0us, + .ackDelayExponent = static_cast(kDefaultAckDelayExponent), + .connTime = connTime, + .recvTimestampsConfig = defaultAckReceiveTimestmpsConfig}; + EXPECT_THROW( - writeAckFrame( - AckFrameMetaData(ackBlocks, 0us, kDefaultAckDelayExponent), - pktBuilder), + ((frameType == FrameType::ACK) + ? writeAckFrame(ackFrameMetaData, pktBuilder, frameType) + : writeAckFrameWithReceivedTimestamps( + ackFrameMetaData, pktBuilder, frameType)), QuicTransportException); } -TEST_F(QuicWriteCodecTest, AckFrameVeryLargeAckRange) { +TEST_P(QuicWriteCodecTest, AckFrameVeryLargeAckRange) { MockQuicPacketBuilder pktBuilder; setupCommonExpects(pktBuilder); @@ -741,12 +915,57 @@ TEST_F(QuicWriteCodecTest, AckFrameVeryLargeAckRange) { // total 11 bytes PacketNum largest = (uint64_t)1 << 55; AckBlocks ackBlocks = {{1, largest}}; - AckFrameMetaData ackMetadata(ackBlocks, 0us, kDefaultAckDelayExponent); + auto frameType = GetParam(); - auto ackFrameWriteResult = *writeAckFrame(ackMetadata, pktBuilder); + TimePoint connTime = Clock::now(); + + WriteAckState ackState = {.acks = ackBlocks}; + ackState.acks = ackBlocks; + if (frameType == FrameType::ACK_RECEIVE_TIMESTAMPS) { + auto lastPacketDelta = + (kMaxReceivedPktsTimestampsStored * kDefaultTimestampsDelta); + PacketsReceivedTimestampsDeque pktsReceivedTimestamps; + + for (auto it = ackBlocks.crbegin(); it != ackBlocks.crend(); it++) { + for (auto i = it->end; i >= it->start; i--) { + if (pktsReceivedTimestamps.size() < kMaxReceivedPktsTimestampsStored) { + RecvdPacketInfo rpi; + rpi.pktNum = i; + auto diff = std::chrono::microseconds( + lastPacketDelta -= kDefaultTimestampsDelta); + rpi.timeStamp = connTime + diff; + pktsReceivedTimestamps.emplace_front(rpi); + } else { + break; + } + } + } + ackState.recvdPacketInfos = pktsReceivedTimestamps; + ackState.lastRecvdPacketInfo.assign({ + ackState.recvdPacketInfos.back().pktNum, + ackState.recvdPacketInfos.back().timeStamp, + }); + } + + AckFrameMetaData ackFrameMetaData = { + .ackState = ackState, + .ackDelay = 0us, + .ackDelayExponent = static_cast(kDefaultAckDelayExponent), + .connTime = connTime, + .recvTimestampsConfig = defaultAckReceiveTimestmpsConfig, + .maxAckReceiveTimestampsToSend = kMaxReceivedPktsTimestampsStored}; + auto ackFrameWriteResult = (frameType == FrameType::ACK) + ? *writeAckFrame(ackFrameMetaData, pktBuilder, frameType) + : *writeAckFrameWithReceivedTimestamps( + ackFrameMetaData, pktBuilder, frameType); + auto addlBytesConsumed = computeBytesForAckReceivedTimestamps( + ackFrameMetaData, ackFrameWriteResult, frameType); + + EXPECT_EQ(19 + addlBytesConsumed, ackFrameWriteResult.bytesWritten); + EXPECT_EQ( + kDefaultUDPSendPacketLen - 19 - addlBytesConsumed, + pktBuilder.remainingSpaceInPkt()); - EXPECT_EQ(19, ackFrameWriteResult.bytesWritten); - EXPECT_EQ(kDefaultUDPSendPacketLen - 19, pktBuilder.remainingSpaceInPkt()); auto builtOut = std::move(pktBuilder).buildTestPacket(); auto regularPacket = builtOut.first; EXPECT_EQ(regularPacket.frames.size(), 1); @@ -755,9 +974,26 @@ TEST_F(QuicWriteCodecTest, AckFrameVeryLargeAckRange) { EXPECT_EQ(ackFrame.ackBlocks.back().start, 1); EXPECT_EQ(largest, ackFrame.ackBlocks.back().end); + + if (frameType == FrameType::ACK_RECEIVE_TIMESTAMPS) { + auto wireBuf = std::move(builtOut.second); + BufQueue queue; + queue.append(wireBuf->clone()); + QuicFrame decodedFrame = + parseQuicFrame(queue, frameType == FrameType::ACK_RECEIVE_TIMESTAMPS); + auto& decodedAckFrame = *decodedFrame.asReadAckFrame(); + // Single contingious ack blocks, default received timestamps stored. + assertsOnDecodedReceiveTimestamps( + ackFrameMetaData, + ackFrameWriteResult.writeAckFrame, + + decodedAckFrame, + 1 /* timestamp ranges count */, + kMaxReceivedPktsTimestampsStored /* timestamps count */); + } } -TEST_F(QuicWriteCodecTest, AckFrameNotEnoughForAnything) { +TEST_P(QuicWriteCodecTest, AckFrameNotEnoughForAnything) { MockQuicPacketBuilder pktBuilder; pktBuilder.remaining_ = 4; setupCommonExpects(pktBuilder); @@ -769,32 +1005,58 @@ TEST_F(QuicWriteCodecTest, AckFrameNotEnoughForAnything) { // 1 byte for first ack block length, then 2 bytes for each pair => 5 bytes // total 15 bytes AckBlocks ackBlocks = {{1000, 1000}, {500, 700}, {100, 200}}; - // 4 btyes are just not enough for anything - AckFrameMetaData ackMetadata(ackBlocks, 555us, kDefaultAckDelayExponent); + auto frameType = GetParam(); + TimePoint connTime = Clock::now(); + WriteAckState ackState = + createTestWriteAckState(frameType, connTime, ackBlocks); + AckFrameMetaData ackFrameMetaData = { + .ackState = ackState, + .ackDelay = 0us, + .ackDelayExponent = static_cast(kDefaultAckDelayExponent), + .connTime = connTime, + .recvTimestampsConfig = defaultAckReceiveTimestmpsConfig}; - auto result = writeAckFrame(ackMetadata, pktBuilder); + auto result = (frameType == FrameType::ACK) + ? writeAckFrame(ackFrameMetaData, pktBuilder, frameType) + : writeAckFrameWithReceivedTimestamps( + ackFrameMetaData, pktBuilder, frameType); EXPECT_FALSE(result.has_value()); EXPECT_EQ(pktBuilder.remainingSpaceInPkt(), 4); } -TEST_F(QuicWriteCodecTest, WriteSimpleAckFrame) { +TEST_P(QuicWriteCodecTest, WriteSimpleAckFrame) { MockQuicPacketBuilder pktBuilder; setupCommonExpects(pktBuilder); auto ackDelay = 111us; AckBlocks ackBlocks = {{501, 1000}, {101, 400}}; - AckFrameMetaData meta(ackBlocks, ackDelay, kDefaultAckDelayExponent); - + auto frameType = GetParam(); + TimePoint connTime = Clock::now(); + WriteAckState ackState = + createTestWriteAckState(frameType, connTime, ackBlocks); + AckFrameMetaData ackFrameMetaData = { + .ackState = ackState, + .ackDelay = ackDelay, + .ackDelayExponent = static_cast(kDefaultAckDelayExponent), + .connTime = connTime, + .recvTimestampsConfig = defaultAckReceiveTimestmpsConfig, + .maxAckReceiveTimestampsToSend = kMaxReceivedPktsTimestampsStored}; // 1 type byte, // 2 bytes for largest acked, 1 bytes for ack delay => 3 bytes // 1 byte for ack block count // There is 1 gap => each represented by 2 bytes => 2 bytes // 2 byte for first ack block length, then 2 bytes for the next len => 4 bytes // total 11 bytes + auto ackFrameWriteResult = (frameType == FrameType::ACK) + ? *writeAckFrame(ackFrameMetaData, pktBuilder, frameType) + : *writeAckFrameWithReceivedTimestamps( + ackFrameMetaData, pktBuilder, frameType); + auto addlBytesConsumed = computeBytesForAckReceivedTimestamps( + ackFrameMetaData, ackFrameWriteResult, frameType); + EXPECT_EQ(11 + addlBytesConsumed, ackFrameWriteResult.bytesWritten); + EXPECT_EQ( + kDefaultUDPSendPacketLen - 11 - addlBytesConsumed, + pktBuilder.remainingSpaceInPkt()); - auto result = *writeAckFrame(meta, pktBuilder); - - EXPECT_EQ(11, result.bytesWritten); - EXPECT_EQ(kDefaultUDPSendPacketLen - 11, pktBuilder.remainingSpaceInPkt()); auto builtOut = std::move(pktBuilder).buildTestPacket(); auto regularPacket = builtOut.first; WriteAckFrame& ackFrame = *regularPacket.frames.back().asWriteAckFrame(); @@ -805,11 +1067,11 @@ TEST_F(QuicWriteCodecTest, WriteSimpleAckFrame) { iter++; EXPECT_EQ(iter->start, 501); EXPECT_EQ(iter->end, 1000); - auto wireBuf = std::move(builtOut.second); BufQueue queue; queue.append(wireBuf->clone()); - QuicFrame decodedFrame = parseQuicFrame(queue); + QuicFrame decodedFrame = + parseQuicFrame(queue, frameType == FrameType::ACK_RECEIVE_TIMESTAMPS); auto& decodedAckFrame = *decodedFrame.asReadAckFrame(); EXPECT_EQ(decodedAckFrame.largestAcked, 1000); EXPECT_EQ( @@ -820,29 +1082,55 @@ TEST_F(QuicWriteCodecTest, WriteSimpleAckFrame) { EXPECT_EQ(decodedAckFrame.ackBlocks[0].endPacket, 1000); EXPECT_EQ(decodedAckFrame.ackBlocks[1].startPacket, 101); EXPECT_EQ(decodedAckFrame.ackBlocks[1].endPacket, 400); + + if (frameType == FrameType::ACK_RECEIVE_TIMESTAMPS) { + // Multiple ack blocks, however received timestamps storage limit limit + // achieved by the within the latest ack block + assertsOnDecodedReceiveTimestamps( + ackFrameMetaData, + ackFrameWriteResult.writeAckFrame, + + decodedAckFrame, + 1 /* timestamp ranges count */, + kMaxReceivedPktsTimestampsStored /* timestamps count */); + } } -TEST_F(QuicWriteCodecTest, WriteAckFrameWillSaveAckDelay) { +TEST_P(QuicWriteCodecTest, WriteAckFrameWillSaveAckDelay) { MockQuicPacketBuilder pktBuilder; setupCommonExpects(pktBuilder); auto ackDelay = 111us; AckBlocks ackBlocks = {{501, 1000}, {101, 400}}; - AckFrameMetaData meta(ackBlocks, ackDelay, kDefaultAckDelayExponent); + auto frameType = GetParam(); + TimePoint connTime = Clock::now(); + WriteAckState ackState = + createTestWriteAckState(frameType, connTime, ackBlocks); + AckFrameMetaData ackFrameMetaData = { + .ackState = ackState, + .ackDelay = ackDelay, + .ackDelayExponent = static_cast(kDefaultAckDelayExponent), + .connTime = connTime, + .recvTimestampsConfig = defaultAckReceiveTimestmpsConfig, + .maxAckReceiveTimestampsToSend = kMaxReceivedPktsTimestampsStored}; - writeAckFrame(meta, pktBuilder); + auto ackFrameWriteResult = (frameType == FrameType::ACK) + ? *writeAckFrame(ackFrameMetaData, pktBuilder, frameType) + : *writeAckFrameWithReceivedTimestamps( + ackFrameMetaData, pktBuilder, frameType); auto builtOut = std::move(pktBuilder).buildTestPacket(); auto regularPacket = builtOut.first; WriteAckFrame& ackFrame = *regularPacket.frames.back().asWriteAckFrame(); EXPECT_EQ(ackDelay, ackFrame.ackDelay); } -TEST_F(QuicWriteCodecTest, VerifyNumAckBlocksSizeAccounted) { - // Tests that if we restrict the size to be exactly the size required for a 1 - // byte num blocks size, if the num blocks requires 2 bytes (practically this - // will never happen), then we won't write the additional blocks. +TEST_P(QuicWriteCodecTest, VerifyNumAckBlocksSizeAccounted) { + // Tests that if we restrict the size to be exactly the size required + // for a byte num blocks size, if the num blocks requires 2 bytes + // practically will never happen), then we won't write the additional MockQuicPacketBuilder pktBuilder; pktBuilder.remaining_ = 134; setupCommonExpects(pktBuilder); + auto frameType = GetParam(); // 1 type byte, // 2 byte for largest acked, 1 bytes for ack delay => 3 bytes @@ -861,31 +1149,91 @@ TEST_F(QuicWriteCodecTest, VerifyNumAckBlocksSizeAccounted) { currentEnd -= blockLength + gap; } ackBlocks.insert({largest, largest}); - AckFrameMetaData ackMetadata(ackBlocks, 0us, kDefaultAckDelayExponent); - - auto ackFrameWriteResult = *writeAckFrame(ackMetadata, pktBuilder); - - EXPECT_EQ(ackFrameWriteResult.bytesWritten, 132); - EXPECT_EQ(pktBuilder.remainingSpaceInPkt(), 2); + TimePoint connTime = Clock::now(); + WriteAckState ackState = + createTestWriteAckState(frameType, connTime, ackBlocks); + AckFrameMetaData ackFrameMetaData = { + .ackState = ackState, + .ackDelay = 0us, + .ackDelayExponent = static_cast(kDefaultAckDelayExponent), + .connTime = connTime, + .recvTimestampsConfig = defaultAckReceiveTimestmpsConfig, + .maxAckReceiveTimestampsToSend = kMaxReceivedPktsTimestampsStored}; + // 1 type byte, + // 2 bytes for largest acked, 1 bytes for ack delay => 3 bytes + // 1 byte for ack block count + // There is 1 gap => each represented by 2 bytes => 2 bytes + // 2 byte for first ack block length, then 2 bytes for the next len => 4 bytes + // total 11 bytes + auto ackFrameWriteResult = (frameType == FrameType::ACK) + ? *writeAckFrame(ackFrameMetaData, pktBuilder, frameType) + : *writeAckFrameWithReceivedTimestamps( + ackFrameMetaData, pktBuilder, frameType); + auto addlBytesConsumed = computeBytesForAckReceivedTimestamps( + ackFrameMetaData, ackFrameWriteResult, frameType); auto builtOut = std::move(pktBuilder).buildTestPacket(); auto regularPacket = builtOut.first; - EXPECT_EQ(regularPacket.frames.size(), 1); - WriteAckFrame& ackFrame = *regularPacket.frames.back().asWriteAckFrame(); - EXPECT_EQ(ackFrame.ackBlocks.size(), 64); - EXPECT_EQ(ackFrame.ackBlocks.back().start, 746); - EXPECT_EQ(ackFrame.ackBlocks.back().end, 748); + if (frameType == FrameType::ACK) { + EXPECT_EQ(ackFrameWriteResult.bytesWritten, 132); + EXPECT_EQ(pktBuilder.remainingSpaceInPkt(), 2); + EXPECT_EQ(regularPacket.frames.size(), 1); + WriteAckFrame& ackFrame = *regularPacket.frames.back().asWriteAckFrame(); + EXPECT_EQ(ackFrame.ackBlocks.size(), 64); + + EXPECT_EQ(ackFrame.ackBlocks.back().start, 746); + EXPECT_EQ(ackFrame.ackBlocks.back().end, 748); + } else if (frameType == FrameType::ACK_RECEIVE_TIMESTAMPS) { + EXPECT_EQ(ackFrameWriteResult.bytesWritten, 128 + addlBytesConsumed); + EXPECT_EQ(pktBuilder.remainingSpaceInPkt(), 0); + EXPECT_EQ(regularPacket.frames.size(), 1); + WriteAckFrame& ackFrame = *regularPacket.frames.back().asWriteAckFrame(); + EXPECT_EQ(ackFrame.ackBlocks.size(), 62); + + EXPECT_EQ(ackFrame.ackBlocks.back().start, 754); + EXPECT_EQ(ackFrame.ackBlocks.back().end, 756); + + auto wireBuf = std::move(builtOut.second); + BufQueue queue; + queue.append(wireBuf->clone()); + QuicFrame decodedFrame = + parseQuicFrame(queue, frameType == FrameType::ACK_RECEIVE_TIMESTAMPS); + auto& decodedAckFrame = *decodedFrame.asReadAckFrame(); + // Multiple ack blocks, however no space remaining to send received + // timestamps + assertsOnDecodedReceiveTimestamps( + ackFrameMetaData, + ackFrameWriteResult.writeAckFrame, + + decodedAckFrame, + 0 /* timestamp ranges count */, + 0 /* timestamps count */); + } } -TEST_F(QuicWriteCodecTest, WriteWithDifferentAckDelayExponent) { +TEST_P(QuicWriteCodecTest, WriteWithDifferentAckDelayExponent) { MockQuicPacketBuilder pktBuilder; setupCommonExpects(pktBuilder); AckBlocks ackBlocks{{1000, 1000}}; uint8_t ackDelayExponent = 6; - AckFrameMetaData ackMetadata(ackBlocks, 1240us, ackDelayExponent); + auto frameType = GetParam(); - writeAckFrame(ackMetadata, pktBuilder); + TimePoint connTime = Clock::now(); + WriteAckState ackState = + createTestWriteAckState(frameType, connTime, ackBlocks); + AckFrameMetaData ackFrameMetaData = { + .ackState = ackState, + .ackDelay = 1240us, + .ackDelayExponent = static_cast(ackDelayExponent), + .connTime = connTime, + .recvTimestampsConfig = defaultAckReceiveTimestmpsConfig, + .maxAckReceiveTimestampsToSend = kMaxReceivedPktsTimestampsStored}; + + (frameType == FrameType::ACK) + ? writeAckFrame(ackFrameMetaData, pktBuilder, frameType) + : *writeAckFrameWithReceivedTimestamps( + ackFrameMetaData, pktBuilder, frameType); auto builtOut = std::move(pktBuilder).buildTestPacket(); auto wireBuf = std::move(builtOut.second); BufQueue queue; @@ -897,18 +1245,31 @@ TEST_F(QuicWriteCodecTest, WriteWithDifferentAckDelayExponent) { auto& decodedAckFrame = *decodedFrame.asReadAckFrame(); EXPECT_EQ( decodedAckFrame.ackDelay.count(), - computeExpectedDelay(ackMetadata.ackDelay, ackDelayExponent)); + computeExpectedDelay(ackFrameMetaData.ackDelay, ackDelayExponent)); } -TEST_F(QuicWriteCodecTest, WriteExponentInLongHeaderPacket) { +TEST_P(QuicWriteCodecTest, WriteExponentInLongHeaderPacket) { MockQuicPacketBuilder pktBuilder; setupCommonExpects(pktBuilder); AckBlocks ackBlocks{{1000, 1000}}; uint8_t ackDelayExponent = 6; - AckFrameMetaData ackMetadata(ackBlocks, 1240us, ackDelayExponent); + auto frameType = GetParam(); + TimePoint connTime = Clock::now(); + WriteAckState ackState = + createTestWriteAckState(frameType, connTime, ackBlocks); + AckFrameMetaData ackFrameMetaData = { + .ackState = ackState, + .ackDelay = 1240us, + .ackDelayExponent = static_cast(ackDelayExponent), + .connTime = connTime, + .recvTimestampsConfig = defaultAckReceiveTimestmpsConfig, + .maxAckReceiveTimestampsToSend = kMaxReceivedPktsTimestampsStored}; - writeAckFrame(ackMetadata, pktBuilder); + (frameType == FrameType::ACK) + ? writeAckFrame(ackFrameMetaData, pktBuilder, frameType) + : writeAckFrameWithReceivedTimestamps( + ackFrameMetaData, pktBuilder, frameType); auto builtOut = std::move(pktBuilder).buildLongHeaderPacket(); auto wireBuf = std::move(builtOut.second); folly::io::Cursor cursor(wireBuf.get()); @@ -921,11 +1282,11 @@ TEST_F(QuicWriteCodecTest, WriteExponentInLongHeaderPacket) { auto& decodedAckFrame = *decodedFrame.asReadAckFrame(); EXPECT_EQ( decodedAckFrame.ackDelay.count(), - (uint64_t(ackMetadata.ackDelay.count()) >> ackDelayExponent) + (uint64_t(ackFrameMetaData.ackDelay.count()) >> ackDelayExponent) << kDefaultAckDelayExponent); } -TEST_F(QuicWriteCodecTest, OnlyAckLargestPacket) { +TEST_P(QuicWriteCodecTest, OnlyAckLargestPacket) { MockQuicPacketBuilder pktBuilder; setupCommonExpects(pktBuilder); @@ -935,13 +1296,33 @@ TEST_F(QuicWriteCodecTest, OnlyAckLargestPacket) { // 1 byte for first ack block length // total 7 bytes AckBlocks ackBlocks{{1000, 1000}}; - AckFrameMetaData ackMetadata(ackBlocks, 555us, kDefaultAckDelayExponent); // No AckBlock is added to the metadata. There will still be one block // generated as the first block to cover largestAcked => 2 bytes - auto ackFrameWriteResult = *writeAckFrame(ackMetadata, pktBuilder); - EXPECT_EQ(7, ackFrameWriteResult.bytesWritten); - EXPECT_EQ(kDefaultUDPSendPacketLen - 7, pktBuilder.remainingSpaceInPkt()); + auto frameType = GetParam(); + TimePoint connTime = Clock::now(); + WriteAckState ackState = + createTestWriteAckState(frameType, connTime, ackBlocks); + AckFrameMetaData ackFrameMetaData = { + .ackState = ackState, + .ackDelay = 555us, + .ackDelayExponent = static_cast(kDefaultAckDelayExponent), + .connTime = connTime, + .recvTimestampsConfig = defaultAckReceiveTimestmpsConfig, + .maxAckReceiveTimestampsToSend = kMaxReceivedPktsTimestampsStored}; + + auto ackFrameWriteResult = (frameType == FrameType::ACK) + ? *writeAckFrame(ackFrameMetaData, pktBuilder, frameType) + : *writeAckFrameWithReceivedTimestamps( + ackFrameMetaData, pktBuilder, frameType); + auto addlBytesConsumed = computeBytesForAckReceivedTimestamps( + ackFrameMetaData, ackFrameWriteResult, frameType); + + EXPECT_EQ(ackFrameWriteResult.bytesWritten, 7 + addlBytesConsumed); + EXPECT_EQ( + kDefaultUDPSendPacketLen - 7 - addlBytesConsumed, + pktBuilder.remainingSpaceInPkt()); + auto builtOut = std::move(pktBuilder).buildTestPacket(); auto regularPacket = builtOut.first; EXPECT_EQ(regularPacket.frames.size(), 1); @@ -960,16 +1341,27 @@ TEST_F(QuicWriteCodecTest, OnlyAckLargestPacket) { EXPECT_EQ(decodedAckFrame.largestAcked, 1000); EXPECT_EQ( decodedAckFrame.ackDelay.count(), - computeExpectedDelay(ackMetadata.ackDelay, kDefaultAckDelayExponent)); + computeExpectedDelay( + ackFrameMetaData.ackDelay, kDefaultAckDelayExponent)); EXPECT_EQ(decodedAckFrame.ackBlocks.size(), 1); EXPECT_EQ(decodedAckFrame.ackBlocks[0].startPacket, 1000); EXPECT_EQ(decodedAckFrame.ackBlocks[0].endPacket, 1000); + + if (frameType == FrameType::ACK_RECEIVE_TIMESTAMPS) { + // Single packet received and acked + assertsOnDecodedReceiveTimestamps( + ackFrameMetaData, + ackFrameWriteResult.writeAckFrame, + + decodedAckFrame, + 1 /* timestamp ranges count */, + 1 /* timestamps count */); + } } -TEST_F(QuicWriteCodecTest, WriteSomeAckBlocks) { +TEST_P(QuicWriteCodecTest, WriteSomeAckBlocks) { // Too many ack blocks passed in, we can only write some of them MockQuicPacketBuilder pktBuilder; - pktBuilder.remaining_ = 36; setupCommonExpects(pktBuilder); // 1 type byte, @@ -978,26 +1370,50 @@ TEST_F(QuicWriteCodecTest, WriteSomeAckBlocks) { // 1 byte for first ack block length // each additional ack block 1 byte gap + 1 byte length => 2 bytes // total 7 bytes - AckBlocks testAckBlocks; + AckBlocks ackBlocks; PacketNum currentEnd = 1000; auto blockLength = 5; auto gap = 10; for (int i = 0; i < 30; i++) { - testAckBlocks.insert({currentEnd - blockLength + 1, currentEnd}); + ackBlocks.insert({currentEnd - blockLength + 1, currentEnd}); currentEnd -= blockLength + gap; } - testAckBlocks.insert({1000, 1000}); + ackBlocks.insert({1000, 1000}); - AckFrameMetaData ackMetadata(testAckBlocks, 555ms, kDefaultAckDelayExponent); - auto ackFrameWriteResult = *writeAckFrame(ackMetadata, pktBuilder); + auto frameType = GetParam(); - EXPECT_EQ(ackFrameWriteResult.bytesWritten, 35); + if (frameType == FrameType::ACK) { + pktBuilder.remaining_ = 36; + } else { + pktBuilder.remaining_ = 42; + } + TimePoint connTime = Clock::now(); + WriteAckState ackState = + createTestWriteAckState(frameType, connTime, ackBlocks); + AckFrameMetaData ackFrameMetaData = { + .ackState = ackState, + .ackDelay = 555us, + .ackDelayExponent = static_cast(kDefaultAckDelayExponent), + .connTime = connTime, + .recvTimestampsConfig = defaultAckReceiveTimestmpsConfig, + .maxAckReceiveTimestampsToSend = kMaxReceivedPktsTimestampsStored}; + + auto ackFrameWriteResult = (frameType == FrameType::ACK) + ? *writeAckFrame(ackFrameMetaData, pktBuilder, frameType) + : *writeAckFrameWithReceivedTimestamps( + ackFrameMetaData, pktBuilder, frameType); + auto addlBytesConsumed = computeBytesForAckReceivedTimestamps( + ackFrameMetaData, ackFrameWriteResult, frameType); + + EXPECT_EQ(ackFrameWriteResult.bytesWritten, 35 + addlBytesConsumed); EXPECT_EQ(pktBuilder.remainingSpaceInPkt(), 1); + auto builtOut = std::move(pktBuilder).buildTestPacket(); auto regularPacket = builtOut.first; EXPECT_EQ(regularPacket.frames.size(), 1); WriteAckFrame& ackFrame = *regularPacket.frames.back().asWriteAckFrame(); - EXPECT_EQ(ackFrame.ackBlocks.size(), 14); + + EXPECT_EQ(ackFrame.ackBlocks.size(), 15); // Verify the on wire bytes via decoder: // (Awkwardly, this assumes the decoder is correct) @@ -1009,11 +1425,23 @@ TEST_F(QuicWriteCodecTest, WriteSomeAckBlocks) { EXPECT_EQ(decodedAckFrame.largestAcked, 1000); EXPECT_EQ( decodedAckFrame.ackDelay.count(), - computeExpectedDelay(ackMetadata.ackDelay, kDefaultAckDelayExponent)); - EXPECT_EQ(decodedAckFrame.ackBlocks.size(), 14); + computeExpectedDelay( + ackFrameMetaData.ackDelay, kDefaultAckDelayExponent)); + + EXPECT_EQ(decodedAckFrame.ackBlocks.size(), 15); + + if (frameType == FrameType::ACK_RECEIVE_TIMESTAMPS) { + assertsOnDecodedReceiveTimestamps( + ackFrameMetaData, + ackFrameWriteResult.writeAckFrame, + + decodedAckFrame, + 0 /* timestamp ranges count */, + 0 /* timestamps count */); + } } -TEST_F(QuicWriteCodecTest, NoSpaceForAckBlockSection) { +TEST_P(QuicWriteCodecTest, NoSpaceForAckBlockSection) { MockQuicPacketBuilder pktBuilder; pktBuilder.remaining_ = 6; setupCommonExpects(pktBuilder); @@ -1023,14 +1451,36 @@ TEST_F(QuicWriteCodecTest, NoSpaceForAckBlockSection) { // 1 byte for num ack blocks // 1 byte for first ack block length AckBlocks ackBlocks = {{1000, 1000}, {701, 900}, {501, 600}}; - AckFrameMetaData ackMetadata(ackBlocks, 555us, kDefaultAckDelayExponent); - auto ackFrameWriteResult = writeAckFrame(ackMetadata, pktBuilder); + auto frameType = GetParam(); + TimePoint connTime = Clock::now(); + WriteAckState ackState = + createTestWriteAckState(frameType, connTime, ackBlocks); + AckFrameMetaData ackFrameMetaData = { + .ackState = ackState, + .ackDelay = 555us, + .ackDelayExponent = static_cast(kDefaultAckDelayExponent), + .connTime = connTime, + .recvTimestampsConfig = defaultAckReceiveTimestmpsConfig, + .maxAckReceiveTimestampsToSend = kMaxReceivedPktsTimestampsStored}; + + auto ackFrameWriteResult = (frameType == FrameType::ACK) + ? writeAckFrame(ackFrameMetaData, pktBuilder, frameType) + : writeAckFrameWithReceivedTimestamps( + ackFrameMetaData, pktBuilder, frameType); EXPECT_FALSE(ackFrameWriteResult.has_value()); } -TEST_F(QuicWriteCodecTest, OnlyHasSpaceForFirstAckBlock) { +TEST_P(QuicWriteCodecTest, OnlyHasSpaceForFirstAckBlock) { MockQuicPacketBuilder pktBuilder; - pktBuilder.remaining_ = 10; + auto frameType = GetParam(); + + if (frameType == FrameType::ACK) { + pktBuilder.remaining_ = 10; + + } else if (frameType == FrameType::ACK_RECEIVE_TIMESTAMPS) { + pktBuilder.remaining_ = 16; + } + setupCommonExpects(pktBuilder); // 1 type byte, @@ -1038,11 +1488,27 @@ TEST_F(QuicWriteCodecTest, OnlyHasSpaceForFirstAckBlock) { // 1 byte for num ack blocks // 1 byte for first ack block length AckBlocks ackBlocks = {{1000, 1000}, {701, 900}, {501, 600}}; - AckFrameMetaData ackMetadata(ackBlocks, 555us, kDefaultAckDelayExponent); - auto ackFrameWriteResult = *writeAckFrame(ackMetadata, pktBuilder); + TimePoint connTime = Clock::now(); + WriteAckState ackState = + createTestWriteAckState(frameType, connTime, ackBlocks); + AckFrameMetaData ackFrameMetaData = { + .ackState = ackState, + .ackDelay = 555us, + .ackDelayExponent = static_cast(kDefaultAckDelayExponent), + .connTime = connTime, + .recvTimestampsConfig = defaultAckReceiveTimestmpsConfig, + .maxAckReceiveTimestampsToSend = kMaxReceivedPktsTimestampsStored}; - EXPECT_EQ(ackFrameWriteResult.bytesWritten, 7); + auto ackFrameWriteResult = (frameType == FrameType::ACK) + ? *writeAckFrame(ackFrameMetaData, pktBuilder, frameType) + : *writeAckFrameWithReceivedTimestamps( + ackFrameMetaData, pktBuilder, frameType); + auto addlBytesConsumed = computeBytesForAckReceivedTimestamps( + ackFrameMetaData, ackFrameWriteResult, frameType); + + EXPECT_EQ(ackFrameWriteResult.bytesWritten, 7 + addlBytesConsumed); EXPECT_EQ(pktBuilder.remainingSpaceInPkt(), 3); + auto builtOut = std::move(pktBuilder).buildTestPacket(); WriteAckFrame& ackFrame = *builtOut.first.frames.back().asWriteAckFrame(); EXPECT_EQ(ackFrame.ackBlocks.size(), 1); @@ -1057,10 +1523,179 @@ TEST_F(QuicWriteCodecTest, OnlyHasSpaceForFirstAckBlock) { EXPECT_EQ(decodedAckFrame.largestAcked, 1000); EXPECT_EQ( decodedAckFrame.ackDelay.count(), - computeExpectedDelay(ackMetadata.ackDelay, kDefaultAckDelayExponent)); + computeExpectedDelay( + ackFrameMetaData.ackDelay, kDefaultAckDelayExponent)); EXPECT_EQ(decodedAckFrame.ackBlocks.size(), 1); EXPECT_EQ(decodedAckFrame.ackBlocks[0].startPacket, 1000); EXPECT_EQ(decodedAckFrame.ackBlocks[0].endPacket, 1000); + + if (frameType == FrameType::ACK_RECEIVE_TIMESTAMPS) { + // No space left for ack blocks, and hence none for received timestamps + assertsOnDecodedReceiveTimestamps( + ackFrameMetaData, + ackFrameWriteResult.writeAckFrame, + + decodedAckFrame, + 0 /* timestamp ranges count */, + 0 /* timestamps count */); + } +} + +TEST_P(QuicWriteCodecTest, WriteAckFrameWithMultipleTimestampRanges) { + MockQuicPacketBuilder pktBuilder; + setupCommonExpects(pktBuilder); + auto ackDelay = 111us; + AckBlocks ackBlocks = {{501, 520}, {471, 490}, {431, 460}}; + // 1 type byte, + // 2 bytes for largest acked, 1 bytes for ack delay => 3 bytes + // 1 byte for ack block count + // There is 1 gap => each represented by 2 bytes => 2 bytes + // 2 byte for first ack block length, then 2 bytes for the next len => 4 + // bytes + // total 11 bytes + auto frameType = GetParam(); + TimePoint connTime = Clock::now(); + WriteAckState ackState = + createTestWriteAckState(frameType, connTime, ackBlocks, 50); + AckFrameMetaData ackFrameMetaData = { + .ackState = ackState, + .ackDelay = ackDelay, + .ackDelayExponent = static_cast(kDefaultAckDelayExponent), + .connTime = connTime, + .recvTimestampsConfig = defaultAckReceiveTimestmpsConfig, + .maxAckReceiveTimestampsToSend = 50}; + + auto ackFrameWriteResult = (frameType == FrameType::ACK) + ? *writeAckFrame(ackFrameMetaData, pktBuilder, frameType) + : *writeAckFrameWithReceivedTimestamps( + ackFrameMetaData, pktBuilder, frameType); + auto addlBytesConsumed = computeBytesForAckReceivedTimestamps( + ackFrameMetaData, ackFrameWriteResult, frameType); + + EXPECT_EQ(ackFrameWriteResult.bytesWritten, 10 + addlBytesConsumed); + EXPECT_EQ( + kDefaultUDPSendPacketLen - 10 - addlBytesConsumed, + pktBuilder.remainingSpaceInPkt()); + + auto builtOut = std::move(pktBuilder).buildTestPacket(); + auto regularPacket = builtOut.first; + WriteAckFrame& ackFrame = *regularPacket.frames.back().asWriteAckFrame(); + EXPECT_EQ(ackFrame.ackBlocks.size(), 3); + auto iter = ackFrame.ackBlocks.crbegin(); + EXPECT_EQ(iter->start, 431); + EXPECT_EQ(iter->end, 460); + iter++; + EXPECT_EQ(iter->start, 471); + EXPECT_EQ(iter->end, 490); + + auto wireBuf = std::move(builtOut.second); + BufQueue queue; + queue.append(wireBuf->clone()); + QuicFrame decodedFrame = + parseQuicFrame(queue, frameType == FrameType::ACK_RECEIVE_TIMESTAMPS); + auto& decodedAckFrame = *decodedFrame.asReadAckFrame(); + EXPECT_EQ(decodedAckFrame.largestAcked, 520); + EXPECT_EQ( + decodedAckFrame.ackDelay.count(), + computeExpectedDelay(ackDelay, kDefaultAckDelayExponent)); + EXPECT_EQ(decodedAckFrame.ackBlocks.size(), 3); + EXPECT_EQ(decodedAckFrame.ackBlocks[0].startPacket, 501); + EXPECT_EQ(decodedAckFrame.ackBlocks[0].endPacket, 520); + EXPECT_EQ(decodedAckFrame.ackBlocks[1].startPacket, 471); + EXPECT_EQ(decodedAckFrame.ackBlocks[1].endPacket, 490); + if (frameType == FrameType::ACK_RECEIVE_TIMESTAMPS) { + // Multiple ack blocks, and received timestamps up to the configured + // allowed received timestamps + assertsOnDecodedReceiveTimestamps( + ackFrameMetaData, + ackFrameWriteResult.writeAckFrame, + + decodedAckFrame, + 3 /* timestamp ranges count */, + 50 /* timestamps count */); + } +} + +TEST_P( + QuicWriteCodecTest, + WriteAckFrameWithMultipleTimestampRangesPartiallySent) { + MockQuicPacketBuilder pktBuilder; + setupCommonExpects(pktBuilder); + auto ackDelay = 111us; + AckBlocks ackBlocks = {{501, 520}, {471, 490}, {431, 460}}; + // 1 type byte, + // 2 bytes for largest acked, 1 bytes for ack delay => 3 bytes + // 1 byte for ack block count + // There is 1 gap => each represented by 2 bytes => 2 bytes + // 2 byte for first ack block length, then 2 bytes for the next len => 4 + // bytes + // total 11 bytes + auto frameType = GetParam(); + TimePoint connTime = Clock::now(); + WriteAckState ackState = + createTestWriteAckState(frameType, connTime, ackBlocks, 100); + AckFrameMetaData ackFrameMetaData = { + .ackState = ackState, + .ackDelay = ackDelay, + .ackDelayExponent = static_cast(kDefaultAckDelayExponent), + .connTime = connTime, + .recvTimestampsConfig = defaultAckReceiveTimestmpsConfig, + .maxAckReceiveTimestampsToSend = 100}; + + if (frameType == FrameType::ACK_RECEIVE_TIMESTAMPS) { + pktBuilder.remaining_ = 80; + } + auto ackFrameWriteResult = (frameType == FrameType::ACK) + ? *writeAckFrame(ackFrameMetaData, pktBuilder, frameType) + : *writeAckFrameWithReceivedTimestamps( + ackFrameMetaData, pktBuilder, frameType); + auto addlBytesConsumed = computeBytesForAckReceivedTimestamps( + ackFrameMetaData, ackFrameWriteResult, frameType); + + if (frameType == FrameType::ACK) { + EXPECT_EQ(10, ackFrameWriteResult.bytesWritten); + EXPECT_EQ(kDefaultUDPSendPacketLen - 10, pktBuilder.remainingSpaceInPkt()); + } else { + EXPECT_EQ(10 + addlBytesConsumed, ackFrameWriteResult.bytesWritten); + EXPECT_EQ(80 - (10 + addlBytesConsumed), pktBuilder.remainingSpaceInPkt()); + } + auto builtOut = std::move(pktBuilder).buildTestPacket(); + auto regularPacket = builtOut.first; + WriteAckFrame& ackFrame = *regularPacket.frames.back().asWriteAckFrame(); + EXPECT_EQ(ackFrame.ackBlocks.size(), 3); + auto iter = ackFrame.ackBlocks.crbegin(); + EXPECT_EQ(iter->start, 431); + EXPECT_EQ(iter->end, 460); + iter++; + EXPECT_EQ(iter->start, 471); + EXPECT_EQ(iter->end, 490); + + auto wireBuf = std::move(builtOut.second); + BufQueue queue; + queue.append(wireBuf->clone()); + QuicFrame decodedFrame = + parseQuicFrame(queue, frameType == FrameType::ACK_RECEIVE_TIMESTAMPS); + auto& decodedAckFrame = *decodedFrame.asReadAckFrame(); + EXPECT_EQ(decodedAckFrame.largestAcked, 520); + EXPECT_EQ( + decodedAckFrame.ackDelay.count(), + computeExpectedDelay(ackDelay, kDefaultAckDelayExponent)); + EXPECT_EQ(decodedAckFrame.ackBlocks.size(), 3); + EXPECT_EQ(decodedAckFrame.ackBlocks[0].startPacket, 501); + EXPECT_EQ(decodedAckFrame.ackBlocks[0].endPacket, 520); + EXPECT_EQ(decodedAckFrame.ackBlocks[1].startPacket, 471); + EXPECT_EQ(decodedAckFrame.ackBlocks[1].endPacket, 490); + + if (frameType == FrameType::ACK_RECEIVE_TIMESTAMPS) { + // Multiple ack blocks, and received timestamps up to the space available + assertsOnDecodedReceiveTimestamps( + ackFrameMetaData, + ackFrameWriteResult.writeAckFrame, + + decodedAckFrame, + 3 /* timestamp ranges count */, + 57 /* timestamps count */); + } } TEST_F(QuicWriteCodecTest, WriteMaxStreamData) { @@ -1723,5 +2358,9 @@ TEST_F(QuicWriteCodecTest, WriteAckFrequencyFrame) { EXPECT_EQ(decodedFrame->reorderThreshold, frame.reorderThreshold); } +INSTANTIATE_TEST_SUITE_P( + QuicWriteCodecTests, + QuicWriteCodecTest, + Values(FrameType::ACK, FrameType::ACK_RECEIVE_TIMESTAMPS)); } // namespace test } // namespace quic diff --git a/quic/common/test/TestPacketBuilders.cpp b/quic/common/test/TestPacketBuilders.cpp index c30367200..1b1a38749 100644 --- a/quic/common/test/TestPacketBuilders.cpp +++ b/quic/common/test/TestPacketBuilders.cpp @@ -131,10 +131,13 @@ RegularQuicPacketBuilder::Packet AckPacketBuilder::build() && { builder.accountForCipherOverhead(maybeAead.value()->getCipherOverhead()); } DCHECK(builder.canBuildPacket()); - AckFrameMetaData ackData( - *CHECK_NOTNULL(maybeAckBlocks.get_pointer()), - *CHECK_NOTNULL(maybeAckDelay.get_pointer()), - CHECK_NOTNULL(dstConn)->transportSettings.ackDelayExponent); + WriteAckState ackState; + ackState.acks = *CHECK_NOTNULL(maybeAckBlocks.get_pointer()); + AckFrameMetaData ackData = { + .ackState = ackState, + .ackDelay = *CHECK_NOTNULL(maybeAckDelay.get_pointer()), + .ackDelayExponent = static_cast( + CHECK_NOTNULL(dstConn)->transportSettings.ackDelayExponent)}; writeAckFrame(ackData, builder); return std::move(builder).buildPacket(); } diff --git a/quic/fizz/client/test/QuicClientTransportTest.cpp b/quic/fizz/client/test/QuicClientTransportTest.cpp index 94c1102ca..f283a523f 100644 --- a/quic/fizz/client/test/QuicClientTransportTest.cpp +++ b/quic/fizz/client/test/QuicClientTransportTest.cpp @@ -4086,7 +4086,12 @@ TEST_F(QuicClientTransportVersionAndRetryTest, UnencryptedAckData) { kDefaultUDPSendPacketLen, std::move(header), 0 /* largestAcked */); builder.encodePacketHeader(); DCHECK(builder.canBuildPacket()); - AckFrameMetaData ackData(acks, 0us, 0); + // AckFrameMetaData ackData(acks, 0us, 0); + WriteAckState writeAckState = {.acks = acks}; + AckFrameMetaData ackData = { + .ackState = writeAckState, + .ackDelay = 0us, + .ackDelayExponent = static_cast(kDefaultAckDelayExponent)}; writeAckFrame(ackData, builder); auto packet = packetToBufCleartext( std::move(builder).buildPacket(), diff --git a/quic/handshake/TransportParameters.h b/quic/handshake/TransportParameters.h index 0399612a6..283fcfb60 100644 --- a/quic/handshake/TransportParameters.h +++ b/quic/handshake/TransportParameters.h @@ -33,6 +33,9 @@ enum class TransportParameterId : uint64_t { retry_source_connection_id = 0x0010, max_datagram_frame_size = 0x0020, min_ack_delay = 0xff02de1a, + ack_receive_timestamps_enabled = 0xff0a001, + max_receive_timestamps_per_ack = 0xff0a002, + receive_timestamps_exponent = 0xff0a003 }; struct TransportParameter { diff --git a/quic/logging/QLoggerConstants.cpp b/quic/logging/QLoggerConstants.cpp index 28408077f..8d121a018 100644 --- a/quic/logging/QLoggerConstants.cpp +++ b/quic/logging/QLoggerConstants.cpp @@ -90,6 +90,8 @@ folly::StringPiece toQlogString(FrameType frame) { case FrameType::GROUP_STREAM_OFF_LEN: case FrameType::GROUP_STREAM_OFF_LEN_FIN: return "group_stream"; + case FrameType::ACK_RECEIVE_TIMESTAMPS: + return "ack_receive_timestamps"; } folly::assume_unreachable(); } diff --git a/quic/server/state/ServerStateMachine.cpp b/quic/server/state/ServerStateMachine.cpp index 83687d734..eb6ec1c48 100644 --- a/quic/server/state/ServerStateMachine.cpp +++ b/quic/server/state/ServerStateMachine.cpp @@ -225,6 +225,15 @@ void processClientInitialParams( static_cast(kStreamGroupsEnabledCustomParamId), clientParams.parameters); + auto isAckReceiveTimestampsEnabled = getIntegerParameter( + TransportParameterId::ack_receive_timestamps_enabled, + clientParams.parameters); + auto maxReceiveTimestampsPerAck = getIntegerParameter( + TransportParameterId::max_receive_timestamps_per_ack, + clientParams.parameters); + auto receiveTimestampsExponent = getIntegerParameter( + TransportParameterId::receive_timestamps_exponent, + clientParams.parameters); if (conn.version == QuicVersion::QUIC_DRAFT || conn.version == QuicVersion::QUIC_V1 || conn.version == QuicVersion::QUIC_V1_ALIAS) { @@ -397,6 +406,19 @@ void processClientInitialParams( if (peerMaxStreamGroupsAdvertized) { conn.peerMaxStreamGroupsAdvertized = *peerMaxStreamGroupsAdvertized; } + if (isAckReceiveTimestampsEnabled.has_value() && + isAckReceiveTimestampsEnabled.value() == 1) { + if (maxReceiveTimestampsPerAck.has_value() && + receiveTimestampsExponent.has_value()) { + conn.maybePeerAckReceiveTimestampsConfig.assign( + {std::min( + static_cast(maxReceiveTimestampsPerAck.value()), + kMaxReceivedPktsTimestampsStored), + std::max( + static_cast(receiveTimestampsExponent.value()), + static_cast(0))}); + } + } } void updateHandshakeState(QuicServerConnectionState& conn) { @@ -762,10 +784,10 @@ void onServerReadDataFromOpen( readData.networkData.data->computeChainDataLength() == 0) { return; } - bool firstPacketFromPeer = false; if (!conn.readCodec) { firstPacketFromPeer = true; + folly::io::Cursor cursor(readData.networkData.data.get()); auto initialByte = cursor.readBE(); auto parsedLongHeader = parseLongHeaderInvariant(initialByte, cursor); @@ -857,8 +879,10 @@ void onServerReadDataFromOpen( conn.qLogger->setScid(conn.serverConnectionId); conn.qLogger->setDcid(initialDestinationConnectionId); } - conn.readCodec->setCodecParameters( - CodecParameters(conn.peerAckDelayExponent, version)); + conn.readCodec->setCodecParameters(CodecParameters( + conn.peerAckDelayExponent, + version, + conn.transportSettings.maybeAckReceiveTimestampsConfigSentToPeer)); conn.initialWriteCipher = cryptoFactory.getServerInitialCipher( initialDestinationConnectionId, version); @@ -1607,6 +1631,34 @@ std::vector setSupportedExtensionTransportParameters( } } + auto ackReceiveTimestampsEnabled = + std::make_unique( + static_cast( + TransportParameterId::ack_receive_timestamps_enabled), + conn.transportSettings.maybeAckReceiveTimestampsConfigSentToPeer + .has_value() + ? 1 + : 0); + customTransportParams.push_back(ackReceiveTimestampsEnabled->encode()); + if (conn.transportSettings.maybeAckReceiveTimestampsConfigSentToPeer + .has_value()) { + auto maxReceiveTimestampsPerAck = + std::make_unique( + static_cast( + TransportParameterId::max_receive_timestamps_per_ack), + conn.transportSettings.maybeAckReceiveTimestampsConfigSentToPeer + .value() + .max_receive_timestamps_per_ack); + customTransportParams.push_back(maxReceiveTimestampsPerAck->encode()); + auto receiveTimestampsExponent = + std::make_unique( + static_cast( + TransportParameterId::receive_timestamps_exponent), + conn.transportSettings.maybeAckReceiveTimestampsConfigSentToPeer + .value() + .receive_timestamps_exponent); + customTransportParams.push_back(receiveTimestampsExponent->encode()); + } return customTransportParams; } } // namespace quic diff --git a/quic/server/test/QuicServerTransportTest.cpp b/quic/server/test/QuicServerTransportTest.cpp index 70e357c13..4fbd1ba6b 100644 --- a/quic/server/test/QuicServerTransportTest.cpp +++ b/quic/server/test/QuicServerTransportTest.cpp @@ -3287,7 +3287,11 @@ TEST_F(QuicUnencryptedServerTransportTest, TestUnencryptedAck) { kDefaultUDPSendPacketLen, std::move(header), 0 /* largestAcked */); builder.encodePacketHeader(); DCHECK(builder.canBuildPacket()); - AckFrameMetaData ackData(acks, 0us, 0); + WriteAckState writeAckState = {.acks = acks}; + AckFrameMetaData ackData = { + .ackState = writeAckState, + .ackDelay = 0us, + .ackDelayExponent = static_cast(kDefaultAckDelayExponent)}; writeAckFrame(ackData, builder); auto packet = packetToBufCleartext( std::move(builder).buildPacket(), diff --git a/quic/state/AckStates.h b/quic/state/AckStates.h index ebe5c29d5..7cac24cd7 100644 --- a/quic/state/AckStates.h +++ b/quic/state/AckStates.h @@ -8,8 +8,8 @@ #pragma once #include +#include #include -#include #include #include @@ -17,7 +17,7 @@ namespace quic { // Ack and PacketNumber states. This is per-packet number space. -struct AckState { +struct AckState : WriteAckState { // Largest ack that has been written to a packet folly::Optional largestAckScheduled; // Count of outstanding packets received with only non-retransmittable data. @@ -26,26 +26,12 @@ struct AckState { folly::Optional largestRecvdPacketTime; // Largest received packet numbers on the connection. folly::Optional largestRecvdPacketNum; - // Receive timestamp and packet number for the largest received packet. - // - // Updated whenever we receive a packet with a larger packet number - // than all previously received packets in the packet number space - // tracked by this AckState. - folly::Optional largestRecvdPacketInfo; - // Receive timestamp and packet number for the last received packet. - // - // Will be different from the value stored in largestRecvdPacketInfo - // if the last packet was received out of order and thus had a packet - // number less than that of a previously received packet in the packet - // number space tracked by this AckState. - folly::Optional lastRecvdPacketInfo; // Latest packet number acked by peer folly::Optional largestAckedByPeer; // Largest received packet number at the time we sent our last close message. folly::Optional largestReceivedAtLastCloseSent; // Next PacketNum we will send for packet in this packet number space PacketNum nextPacketNum{0}; - AckBlocks acks; uint64_t reorderThreshold{0}; folly::Optional tolerance; folly::Optional ackFrequencySequenceNumber; @@ -61,17 +47,6 @@ struct AckState { folly::Optional latestRecvdPacketTime; // Packet number of the latest packet folly::Optional latestReceivedPacketNum; - // Packet number and timestamp of recently received packets. - // - // The maximum number of packets stored in pktsReceivedTimestamps is - // controlled by kMaxReceivedPktsTimestampsStored. - // - // The packet number of entries in the deque is guarenteed to increase - // monotonically because an entry is only added for a received packet - // if the packet number is greater than the packet number of the last - // element in the deque (e.g., entries are not added for packets that - // arrive out of order relative to previously received packets). - CircularDeque recvdPacketInfos; }; struct AckStates { diff --git a/quic/state/StateData.h b/quic/state/StateData.h index f06dc1197..21027829d 100644 --- a/quic/state/StateData.h +++ b/quic/state/StateData.h @@ -730,6 +730,13 @@ struct QuicConnectionStateBase : public folly::DelayedDestruction { // GSO supported on conn. folly::Optional gsoSupported; + + struct AckReceiveTimestampsConfig { + uint64_t maxReceiveTimestampsPerAck; + uint64_t receiveTimestampsExponent; + }; + folly::Optional + maybePeerAckReceiveTimestampsConfig; }; std::ostream& operator<<(std::ostream& os, const QuicConnectionStateBase& st); diff --git a/quic/state/TransportSettings.h b/quic/state/TransportSettings.h index bd9885798..b2e1d71ae 100644 --- a/quic/state/TransportSettings.h +++ b/quic/state/TransportSettings.h @@ -11,6 +11,7 @@ #include #include #include +#include namespace quic { @@ -133,6 +134,11 @@ struct DatagramConfig { uint32_t writeBufSize{kDefaultMaxDatagramsBuffered}; }; +struct AckReceiveTimestampsConfig { + uint64_t max_receive_timestamps_per_ack{kMaxReceivedPktsTimestampsStored}; + uint64_t receive_timestamps_exponent{kDefaultReceiveTimestampsExponent}; +}; + // JSON-serialized transport knobs struct SerializedKnob { uint64_t space; @@ -325,6 +331,19 @@ struct TransportSettings { bool experimentalPacer{false}; // experimental flag to close ingress SM when invoking stopSending bool dropIngressOnStopSending{false}; + + // Local configuration for ACK receive timestamps. + // + // Determines the ACK receive timestamp configuration sent to peer, + // which in turn determines the maximum number of timestamps and + // timestamp resolution included in ACK messages sent by the peer + // if the peer supports ACK receive timestamps. + // + // If structure is not initialized, ACK receive timestamps are + // not requested from peer regardless of whether the peer + // supports them. + folly::Optional + maybeAckReceiveTimestampsConfigSentToPeer; }; } // namespace quic