Codec changes to support ACK_RECEIVE_TIMESTAMPS

Summary: Create a new ACK_RECEIVE_TIMESTAMPS frame, as outlined in https://www.ietf.org/archive/id/draft-smith-quic-receive-ts-00.html#name-ack_receive_timestamps-fram

Reviewed By: mjoras

Differential Revision: D37799050

fbshipit-source-id: 0157c7fa7c4e489bb310f7c9cd6c0c1877e4967f

quic/QuicConstants.h

@@ -10,6 +10,7 @@
 #include <folly/Range.h>
 #include <folly/String.h>
 #include <quic/common/third-party/enum.h>
+#include <sys/types.h>
 #include <chrono>
 #include <cstdint>
 #include <ostream>
@@ -251,6 +252,7 @@ enum class FrameType : uint64_t {
   GROUP_STREAM_OFF_FIN = 0x37,
   GROUP_STREAM_OFF_LEN = 0x38,
   GROUP_STREAM_OFF_LEN_FIN = 0x39,
+  ACK_RECEIVE_TIMESTAMPS = 0xB0
 };
 
 inline constexpr uint16_t toFrameError(FrameType frame) {
@@ -723,5 +725,6 @@ constexpr uint16_t kStreamGroupsEnabledCustomParamId = 0xFF99;
 
 // Maximum packet receive timestamps stored.
 constexpr uint8_t kMaxReceivedPktsTimestampsStored = 5;
+constexpr uint8_t kDefaultReceiveTimestampsExponent = 3;
 
 } // namespace quic

quic/api/QuicPacketScheduler.cpp

@@ -5,8 +5,10 @@
  * LICENSE file in the root directory of this source tree.
  */
 
+#include <quic/QuicConstants.h>
 #include <quic/api/QuicPacketScheduler.h>
 #include <quic/flowcontrol/QuicFlowController.h>
+#include <cstdint>
 
 namespace {
 using namespace quic;
@@ -589,8 +591,39 @@ folly::Optional<PacketNum> AckScheduler::writeNextAcks(
            ? std::chrono::duration_cast<std::chrono::microseconds>(
                  ackingTime - receivedTime)
            : 0us);
-  AckFrameMetaData meta(ackState_.acks, ackDelay, ackDelayExponentToUse);
+
-  auto ackWriteResult = writeAckFrame(meta, builder);
+  AckFrameMetaData meta = {
+      ackState_, /* ackState*/
+      ackDelay, /* ackDelay */
+      static_cast<uint8_t>(ackDelayExponentToUse), /* ackDelayExponent */
+      conn_.connectionTime, /* connect timestamp */
+      folly::none, /* recvTimestampsConfig */
+      folly::none /* maxAckReceiveTimestampsToSend */};
+
+  folly::Optional<AckFrameWriteResult> ackWriteResult;
+
+  bool isAckReceiveTimestampsSupported =
+      conn_.transportSettings.maybeAckReceiveTimestampsConfigSentToPeer &&
+      conn_.maybePeerAckReceiveTimestampsConfig;
+
+  uint64_t peerRequestedTimestampsCount =
+      conn_.maybePeerAckReceiveTimestampsConfig.has_value()
+      ? conn_.maybePeerAckReceiveTimestampsConfig.value()
+            .maxReceiveTimestampsPerAck
+      : 0;
+
+  // If ack_receive_timestamps are not enabled on *either* end-points OR
+  // the peer requests 0 timestamps, we fall-back to using FrameType::ACK
+  if (!isAckReceiveTimestampsSupported || !peerRequestedTimestampsCount) {
+    ackWriteResult = writeAckFrame(meta, builder, FrameType::ACK);
+  } else {
+    meta.recvTimestampsConfig =
+        conn_.transportSettings.maybeAckReceiveTimestampsConfigSentToPeer
+            .value();
+    meta.maxAckReceiveTimestampsToSend = peerRequestedTimestampsCount;
+    ackWriteResult = writeAckFrameWithReceivedTimestamps(
+        meta, builder, FrameType::ACK_RECEIVE_TIMESTAMPS);
+  }
   if (!ackWriteResult) {
     return folly::none;
   }

quic/api/test/QuicPacketSchedulerTest.cpp

@@ -646,7 +646,11 @@ TEST_F(QuicPacketSchedulerTest, WriteOnlyOutstandingPacketsTest) {
   AckBlocks ackBlocks;
   ackBlocks.insert(10, 100);
   ackBlocks.insert(200, 1000);
-  AckFrameMetaData ackMeta(ackBlocks, 0us, kDefaultAckDelayExponent);
+  WriteAckState writeAckState = {.acks = ackBlocks};
+  AckFrameMetaData ackMeta = {
+      .ackState = writeAckState,
+      .ackDelay = 0us,
+      .ackDelayExponent = static_cast<uint8_t>(kDefaultAckDelayExponent)};
 
   // Write those framses with a regular builder
   writeFrame(connCloseFrame, regularBuilder);

quic/client/QuicClientTransport.cpp

@@ -90,7 +90,9 @@ QuicClientTransport::QuicClientTransport(
   }
 
   conn_->readCodec->setCodecParameters(CodecParameters(
-      conn_->peerAckDelayExponent, conn_->originalVersion.value()));
+      conn_->peerAckDelayExponent,
+      conn_->originalVersion.value(),
+      conn_->transportSettings.maybeAckReceiveTimestampsConfigSentToPeer));
 
   VLOG(10) << "client created " << *conn_;
 }
@@ -1016,7 +1018,6 @@ void QuicClientTransport::startCryptoHandshake() {
   setD6DProbeTimeoutTransportParameter();
   setSupportedExtensionTransportParameters();
   maybeEnableStreamGroups();
-
   auto paramsExtension = std::make_shared<ClientTransportParametersExtension>(
       conn_->originalVersion.value(),
       conn_->transportSettings.advertisedInitialConnectionWindowSize,
@@ -1735,6 +1736,35 @@ void QuicClientTransport::setSupportedExtensionTransportParameters() {
             conn_->datagramState.maxReadFrameSize);
     customTransportParameters_.push_back(maxDatagramFrameSize->encode());
   }
+
+  auto ackReceiveTimestampsEnabled =
+      std::make_unique<CustomIntegralTransportParameter>(
+          static_cast<uint64_t>(
+              TransportParameterId::ack_receive_timestamps_enabled),
+          conn_->transportSettings.maybeAckReceiveTimestampsConfigSentToPeer
+                  .has_value()
+              ? 1
+              : 0);
+  customTransportParameters_.push_back(ackReceiveTimestampsEnabled->encode());
+  if (conn_->transportSettings.maybeAckReceiveTimestampsConfigSentToPeer
+          .has_value()) {
+    auto maxReceiveTimestampsPerAck =
+        std::make_unique<CustomIntegralTransportParameter>(
+            static_cast<uint64_t>(
+                TransportParameterId::max_receive_timestamps_per_ack),
+            conn_->transportSettings.maybeAckReceiveTimestampsConfigSentToPeer
+                .value()
+                .max_receive_timestamps_per_ack);
+    customTransportParameters_.push_back(maxReceiveTimestampsPerAck->encode());
+    auto receiveTimestampsExponent =
+        std::make_unique<CustomIntegralTransportParameter>(
+            static_cast<uint64_t>(
+                TransportParameterId::receive_timestamps_exponent),
+            conn_->transportSettings.maybeAckReceiveTimestampsConfigSentToPeer
+                .value()
+                .receive_timestamps_exponent);
+    customTransportParameters_.push_back(receiveTimestampsExponent->encode());
+  }
 }
 
 void QuicClientTransport::adjustGROBuffers() {

quic/client/state/ClientStateMachine.cpp

@@ -10,7 +10,9 @@
 #include <quic/loss/QuicLossFunctions.h>
 
 #include <folly/io/async/AsyncSocketException.h>
+#include <quic/QuicConstants.h>
 #include <quic/client/handshake/CachedServerTransportParameters.h>
+#include <quic/codec/Decode.h>
 #include <quic/common/TimeUtil.h>
 #include <quic/congestion_control/CongestionControllerFactory.h>
 #include <quic/congestion_control/QuicCubic.h>
@@ -54,7 +56,9 @@ std::unique_ptr<QuicClientConnectionState> undoAllClientStateForRetry(
   newConn->readCodec = std::make_unique<QuicReadCodec>(QuicNodeType::Client);
   newConn->readCodec->setClientConnectionId(*conn->clientConnectionId);
   newConn->readCodec->setCodecParameters(CodecParameters(
-      conn->peerAckDelayExponent, conn->originalVersion.value()));
+      conn->peerAckDelayExponent,
+      conn->originalVersion.value(),
+      conn->transportSettings.maybeAckReceiveTimestampsConfigSentToPeer));
   newConn->earlyDataAppParamsValidator =
       std::move(conn->earlyDataAppParamsValidator);
   newConn->earlyDataAppParamsGetter = std::move(conn->earlyDataAppParamsGetter);
@@ -134,6 +138,15 @@ void processServerInitialParams(
   auto minAckDelay = getIntegerParameter(
       TransportParameterId::min_ack_delay, serverParams.parameters);
 
+  auto isAckReceiveTimestampsEnabled = getIntegerParameter(
+      TransportParameterId::ack_receive_timestamps_enabled,
+      serverParams.parameters);
+  auto maxReceiveTimestampsPerAck = getIntegerParameter(
+      TransportParameterId::max_receive_timestamps_per_ack,
+      serverParams.parameters);
+  auto receiveTimestampsExponent = getIntegerParameter(
+      TransportParameterId::receive_timestamps_exponent,
+      serverParams.parameters);
   if (conn.version == QuicVersion::QUIC_DRAFT ||
       conn.version == QuicVersion::QUIC_V1 ||
       conn.version == QuicVersion::QUIC_V1_ALIAS) {
@@ -224,7 +237,7 @@ void processServerInitialParams(
         : conn.transportSettings.advertisedInitialBidiRemoteStreamWindowSize;
     handleStreamWindowUpdate(s, windowSize, packetNum);
   });
-  if (maxDatagramFrameSize.hasValue()) {
+  if (maxDatagramFrameSize.has_value()) {
     if (maxDatagramFrameSize.value() > 0 &&
         maxDatagramFrameSize.value() <= kMaxDatagramPacketOverhead) {
       throw QuicTransportException(
@@ -237,6 +250,20 @@ void processServerInitialParams(
   if (peerMaxStreamGroupsAdvertized) {
     conn.peerMaxStreamGroupsAdvertized = *peerMaxStreamGroupsAdvertized;
   }
+
+  if (isAckReceiveTimestampsEnabled.has_value() &&
+      isAckReceiveTimestampsEnabled.value() == 1) {
+    if (maxReceiveTimestampsPerAck.has_value() &&
+        receiveTimestampsExponent.has_value()) {
+      conn.maybePeerAckReceiveTimestampsConfig.assign(
+          {std::min(
+               static_cast<uint8_t>(maxReceiveTimestampsPerAck.value()),
+               kMaxReceivedPktsTimestampsStored),
+           std::max(
+               static_cast<uint8_t>(receiveTimestampsExponent.value()),
+               static_cast<uint8_t>(0))});
+    }
+  }
 }
 
 void cacheServerInitialParams(

quic/codec/Decode.cpp

@@ -8,9 +8,13 @@
 #include <quic/codec/Decode.h>
 
 #include <folly/String.h>
+#include <quic/QuicConstants.h>
 #include <quic/QuicException.h>
 #include <quic/codec/PacketNumber.h>
 #include <quic/codec/QuicInteger.h>
+#include <chrono>
+#include <cstdint>
+#include <sstream>
 
 namespace {
 
@@ -140,11 +144,41 @@ ImmediateAckFrame decodeImmediateAckFrame(folly::io::Cursor&) {
   return ImmediateAckFrame();
 }
 
+uint64_t computeAdjustedDelay(
+    FrameType frameType,
+    uint8_t exponentToUse,
+    folly::Optional<std::pair<uint64_t, size_t>> delay) {
+  // ackDelayExponentToUse is guaranteed to be less than the size of uint64_t
+  uint64_t delayOverflowMask = 0xFFFFFFFFFFFFFFFF;
+  uint8_t leftShift = (sizeof(delay->first) * 8 - exponentToUse);
+  DCHECK_LT(leftShift, sizeof(delayOverflowMask) * 8);
+  delayOverflowMask = delayOverflowMask << leftShift;
+  if ((delay->first & delayOverflowMask) != 0) {
+    throw QuicTransportException(
+        "Decoded delay overflows",
+        quic::TransportErrorCode::FRAME_ENCODING_ERROR,
+        frameType);
+  }
+  uint64_t adjustedDelay = delay->first << exponentToUse;
+  if (adjustedDelay >
+      static_cast<uint64_t>(
+          std::numeric_limits<std::chrono::microseconds::rep>::max())) {
+    throw QuicTransportException(
+        "Bad delay", quic::TransportErrorCode::FRAME_ENCODING_ERROR, frameType);
+  } else if (UNLIKELY(adjustedDelay > 1000 * 1000 * 1000 /* 1000s */)) {
+    LOG(ERROR) << "Quic recvd long ack delay=" << adjustedDelay;
+    adjustedDelay = 0;
+  }
+  return adjustedDelay;
+}
+
 ReadAckFrame decodeAckFrame(
     folly::io::Cursor& cursor,
     const PacketHeader& header,
-    const CodecParameters& params) {
+    const CodecParameters& params,
+    FrameType frameType) {
   ReadAckFrame frame;
+  frame.frameType = frameType;
   auto largestAckedInt = decodeQuicInteger(cursor);
   if (!largestAckedInt) {
     throw QuicTransportException(
@@ -182,33 +216,12 @@ ReadAckFrame decodeAckFrame(
       ? kDefaultAckDelayExponent
       : params.peerAckDelayExponent;
   DCHECK_LT(ackDelayExponentToUse, sizeof(ackDelay->first) * 8);
-  // ackDelayExponentToUse is guaranteed to be less than the size of uint64_t
+
-  uint64_t delayOverflowMask = 0xFFFFFFFFFFFFFFFF;
-  uint8_t leftShift = (sizeof(ackDelay->first) * 8 - ackDelayExponentToUse);
-  DCHECK_LT(leftShift, sizeof(delayOverflowMask) * 8);
-  delayOverflowMask = delayOverflowMask << leftShift;
-  if ((ackDelay->first & delayOverflowMask) != 0) {
-    throw QuicTransportException(
-        "Decoded ack delay overflows",
-        quic::TransportErrorCode::FRAME_ENCODING_ERROR,
-        quic::FrameType::ACK);
-  }
-  uint64_t adjustedAckDelay = ackDelay->first << ackDelayExponentToUse;
-  if (adjustedAckDelay >
-      static_cast<uint64_t>(
-          std::numeric_limits<std::chrono::microseconds::rep>::max())) {
-    throw QuicTransportException(
-        "Bad ack delay",
-        quic::TransportErrorCode::FRAME_ENCODING_ERROR,
-        quic::FrameType::ACK);
-  } else if (UNLIKELY(adjustedAckDelay > 1000 * 1000 * 1000 /* 1000s */)) {
-    LOG(ERROR) << "Quic recvd long ack delay=" << adjustedAckDelay;
-    adjustedAckDelay = 0;
-  }
   PacketNum currentPacketNum =
       nextAckedPacketLen(largestAcked, firstAckBlockLen->first);
   frame.largestAcked = largestAcked;
-  frame.ackDelay = std::chrono::microseconds(adjustedAckDelay);
+  frame.ackDelay = std::chrono::microseconds(
+      computeAdjustedDelay(frameType, ackDelayExponentToUse, ackDelay));
   frame.ackBlocks.emplace_back(currentPacketNum, largestAcked);
   for (uint64_t numBlocks = 0; numBlocks < additionalAckBlocks->first;
        ++numBlocks) {
@@ -233,6 +246,84 @@ ReadAckFrame decodeAckFrame(
     // already would have processed it in the previous iteration.
     frame.ackBlocks.emplace_back(currentPacketNum, nextEndPacket);
   }
+
+  return frame;
+}
+
+ReadAckFrame decodeAckFrameWithReceivedTimestamps(
+    folly::io::Cursor& cursor,
+    const PacketHeader& header,
+    const CodecParameters& params,
+    FrameType frameType) {
+  ReadAckFrame frame;
+
+  frame = decodeAckFrame(cursor, header, params, frameType);
+
+  auto latestRecvdPacketNum = decodeQuicInteger(cursor);
+  if (!latestRecvdPacketNum) {
+    throw QuicTransportException(
+        "Bad latest received packet number",
+        quic::TransportErrorCode::FRAME_ENCODING_ERROR,
+        quic::FrameType::ACK_RECEIVE_TIMESTAMPS);
+  }
+  frame.maybeLatestRecvdPacketNum = latestRecvdPacketNum->first;
+
+  auto latestRecvdPacketTimeDelta = decodeQuicInteger(cursor);
+  if (!latestRecvdPacketTimeDelta) {
+    throw QuicTransportException(
+        "Bad receive packet timestamp delta",
+        quic::TransportErrorCode::FRAME_ENCODING_ERROR,
+        quic::FrameType::ACK_RECEIVE_TIMESTAMPS);
+  }
+  frame.maybeLatestRecvdPacketTime =
+      std::chrono::microseconds(latestRecvdPacketTimeDelta->first);
+
+  auto timeStampRangeCount = decodeQuicInteger(cursor);
+  if (!timeStampRangeCount) {
+    throw QuicTransportException(
+        "Bad receive timestamps range count",
+        quic::TransportErrorCode::FRAME_ENCODING_ERROR,
+        quic::FrameType::ACK_RECEIVE_TIMESTAMPS);
+  }
+  for (uint64_t numRanges = 0; numRanges < timeStampRangeCount->first;
+       numRanges++) {
+    RecvdPacketsTimestampsRange timeStampRange;
+    auto receiveTimeStampsGap = decodeQuicInteger(cursor);
+    if (!receiveTimeStampsGap) {
+      throw QuicTransportException(
+          "Bad receive timestamps gap",
+          quic::TransportErrorCode::FRAME_ENCODING_ERROR,
+          quic::FrameType::ACK_RECEIVE_TIMESTAMPS);
+    }
+    timeStampRange.gap = receiveTimeStampsGap->first;
+    auto receiveTimeStampsLen = decodeQuicInteger(cursor);
+    if (!receiveTimeStampsLen) {
+      throw QuicTransportException(
+          "Bad receive timestamps block length",
+          quic::TransportErrorCode::FRAME_ENCODING_ERROR,
+          quic::FrameType::ACK_RECEIVE_TIMESTAMPS);
+    }
+    timeStampRange.timestamp_delta_count = receiveTimeStampsLen->first;
+    uint8_t receiveTimestampsExponentToUse =
+        (params.maybeAckReceiveTimestampsConfig)
+        ? params.maybeAckReceiveTimestampsConfig.value()
+              .receive_timestamps_exponent
+        : kDefaultReceiveTimestampsExponent;
+    for (uint64_t i = 0; i < receiveTimeStampsLen->first; i++) {
+      auto delta = decodeQuicInteger(cursor);
+      if (!delta) {
+        throw QuicTransportException(
+            "Bad receive timestamps delta",
+            quic::TransportErrorCode::FRAME_ENCODING_ERROR,
+            quic::FrameType::ACK_RECEIVE_TIMESTAMPS);
+      }
+      DCHECK_LT(receiveTimestampsExponentToUse, sizeof(delta->first) * 8);
+      auto adjustedDelta = computeAdjustedDelay(
+          frameType, receiveTimestampsExponentToUse, delta);
+      timeStampRange.deltas.push_back(adjustedDelta);
+    }
+    frame.recvdPacketsTimestampRanges.emplace_back(timeStampRange);
+  }
   return frame;
 }
 
@@ -754,7 +845,9 @@ QuicFrame parseFrame(
   };
   cursor.reset(queue.front());
   FrameType frameType = static_cast<FrameType>(frameTypeInt->first);
-  try {
+  try
+
+  {
     switch (frameType) {
       case FrameType::PADDING:
         return QuicFrame(decodePaddingFrame(cursor));
@@ -842,6 +935,10 @@ QuicFrame parseFrame(
         return QuicFrame(decodeAckFrequencyFrame(cursor));
       case FrameType::IMMEDIATE_ACK:
         return QuicFrame(decodeImmediateAckFrame(cursor));
+      case FrameType::ACK_RECEIVE_TIMESTAMPS:
+        auto frame = QuicFrame(decodeAckFrameWithReceivedTimestamps(
+            cursor, header, params, FrameType::ACK_RECEIVE_TIMESTAMPS));
+        return frame;
     }
   } catch (const std::exception& e) {
     error = true;

quic/codec/Decode.h

@@ -8,8 +8,10 @@
 #pragma once
 
 #include <folly/io/Cursor.h>
+#include <quic/QuicConstants.h>
 #include <quic/codec/PacketNumber.h>
 #include <quic/codec/Types.h>
+#include <quic/state/TransportSettings.h>
 
 namespace quic {
 
@@ -21,13 +23,23 @@ struct CodecParameters {
   // This must not be set to zero.
   uint8_t peerAckDelayExponent{kDefaultAckDelayExponent};
   QuicVersion version{QuicVersion::MVFST};
+  folly::Optional<AckReceiveTimestampsConfig> maybeAckReceiveTimestampsConfig =
+      folly::none;
 
   CodecParameters() = default;
 
+  CodecParameters(
+      uint8_t peerAckDelayExponentIn,
+      QuicVersion versionIn,
+      folly::Optional<AckReceiveTimestampsConfig>
+          maybeAckReceiveTimestampsConfigIn)
+      : peerAckDelayExponent(peerAckDelayExponentIn),
+        version(versionIn),
+        maybeAckReceiveTimestampsConfig(maybeAckReceiveTimestampsConfigIn) {}
+
   CodecParameters(uint8_t peerAckDelayExponentIn, QuicVersion versionIn)
       : peerAckDelayExponent(peerAckDelayExponentIn), version(versionIn) {}
 };
-
 struct ParsedLongHeaderInvariant {
   uint8_t initialByte;
   LongHeaderInvariant invariant;
@@ -117,7 +129,14 @@ PathResponseFrame decodePathResponseFrame(folly::io::Cursor& cursor);
 ReadAckFrame decodeAckFrame(
     folly::io::Cursor& cursor,
     const PacketHeader& header,
-    const CodecParameters& params);
+    const CodecParameters& params,
+    FrameType frameType = FrameType::ACK);
+
+ReadAckFrame decodeAckFrameWithReceivedTimestamps(
+    folly::io::Cursor& cursor,
+    const PacketHeader& header,
+    const CodecParameters& params,
+    FrameType frameType);
 
 ReadAckFrame decodeAckFrameWithECN(
     folly::io::Cursor& cursor,
@@ -143,8 +162,8 @@ DatagramFrame decodeDatagramFrame(BufQueue& queue, bool hasLen);
 /**
  * Parse the Invariant fields in Long Header.
  *
- * cursor: points to the byte just past initialByte. After parsing, cursor will
+ * cursor: points to the byte just past initialByte. After parsing, cursor
- *         be moved to the byte right after Source Connection ID.
+ * will be moved to the byte right after Source Connection ID.
  */
 folly::Expected<ParsedLongHeaderInvariant, TransportErrorCode>
 parseLongHeaderInvariant(uint8_t initalByte, folly::io::Cursor& cursor);
@@ -213,4 +232,9 @@ folly::Expected<ShortHeader, TransportErrorCode> parseShortHeader(
     uint8_t initialByte,
     folly::io::Cursor& cursor,
     size_t dstConnIdSize = kDefaultConnectionIdSize);
+
+uint64_t computeAdjustedDelay(
+    FrameType frameType,
+    uint8_t exponentToUse,
+    folly::Optional<std::pair<uint64_t, size_t>> delay);
 } // namespace quic

quic/codec/QuicPacketRebuilder.cpp

@@ -228,9 +228,38 @@ folly::Optional<PacketEvent> PacketRebuilder::rebuildFromPacket(
              ? std::chrono::duration_cast<std::chrono::microseconds>(
                    ackingTime - receivedTime)
              : 0us);
-    AckFrameMetaData meta(ackState_.acks, ackDelay, ackDelayExponent);
+
+    AckFrameMetaData meta = {
+        ackState_, /* ackState*/
+        ackDelay, /* ackDelay */
+        static_cast<uint8_t>(ackDelayExponent), /* ackDelayExponent */
+        conn_.connectionTime, /* connect timestamp */
+        folly::none, /* recvTimestampsConfig */
+        folly::none /* maxAckReceiveTimestampsToSend */};
+
+    folly::Optional<AckFrameWriteResult> ackWriteResult;
+
+    uint64_t peerRequestedTimestampsCount =
+        conn_.maybePeerAckReceiveTimestampsConfig.has_value()
+        ? conn_.maybePeerAckReceiveTimestampsConfig.value()
+              .maxReceiveTimestampsPerAck
+        : 0;
+
     // Write the AckFrame ignoring the result. This is best-effort.
-    writeAckFrame(meta, builder_);
+    bool isAckReceiveTimestampsSupported =
+        conn_.transportSettings.maybeAckReceiveTimestampsConfigSentToPeer &&
+        conn_.maybePeerAckReceiveTimestampsConfig;
+
+    if (!isAckReceiveTimestampsSupported || !peerRequestedTimestampsCount) {
+      writeAckFrame(meta, builder_, FrameType::ACK);
+    } else {
+      meta.recvTimestampsConfig =
+          conn_.transportSettings.maybeAckReceiveTimestampsConfigSentToPeer
+              .value();
+      meta.maxAckReceiveTimestampsToSend = peerRequestedTimestampsCount;
+      writeAckFrameWithReceivedTimestamps(
+          meta, builder_, FrameType::ACK_RECEIVE_TIMESTAMPS);
+    }
   }
   // We shouldn't clone if:
   // (1) we only end up cloning only acks, ping, or paddings.

quic/codec/QuicWriteCodec.cpp

@@ -12,6 +12,8 @@
 #include <quic/QuicConstants.h>
 #include <quic/QuicException.h>
 #include <quic/codec/QuicInteger.h>
+#include <cstdint>
+#include <sstream>
 
 namespace {
 
@@ -265,22 +267,131 @@ static size_t fillFrameWithAckBlocks(
   return numAdditionalAckBlocks;
 }
 
-folly::Optional<AckFrameWriteResult> writeAckFrame(
+size_t computeSizeUsedByRecvdTimestamps(WriteAckFrame& ackFrame) {
+  size_t usedSize = 0;
+  for (auto& recvdPacketsTimestampRanges :
+       ackFrame.recvdPacketsTimestampRanges) {
+    usedSize += getQuicIntegerSizeThrows(recvdPacketsTimestampRanges.gap);
+    usedSize += getQuicIntegerSizeThrows(
+        recvdPacketsTimestampRanges.timestamp_delta_count);
+    for (auto& timestampDelta : recvdPacketsTimestampRanges.deltas) {
+      usedSize += getQuicIntegerSizeThrows(timestampDelta);
+    }
+  }
+  return usedSize;
+}
+
+static size_t fillPacketReceiveTimestamps(
     const quic::AckFrameMetaData& ackFrameMetaData,
-    PacketBuilderInterface& builder) {
+    WriteAckFrame& ackFrame,
-  if (ackFrameMetaData.ackBlocks.empty()) {
+    uint64_t largestAckedPacketNum,
+    uint64_t spaceLeft,
+    uint64_t receiveTimestampsExponent) {
+  if (ackFrameMetaData.ackState.recvdPacketInfos.size() == 0) {
+    return 0;
+  }
+  auto recvdPacketInfos = ackFrameMetaData.ackState.recvdPacketInfos;
+  // Insert all received packet timestamps into an interval set, to identify
+  // continguous ranges
+
+  DCHECK(ackFrameMetaData.maxAckReceiveTimestampsToSend.has_value());
+  auto maxRecvTimestampsToSend =
+      ackFrameMetaData.maxAckReceiveTimestampsToSend.value();
+  uint64_t pktsAdded = 0;
+  IntervalSet<PacketNum> receivedPktNumsIntervalSet;
+  for (auto& recvdPkt : recvdPacketInfos) {
+    // Add up to the peer requested max ack receive timestamps;
+    if (pktsAdded == maxRecvTimestampsToSend) {
+      break;
+    }
+    receivedPktNumsIntervalSet.insert(recvdPkt.pktNum);
+    pktsAdded++;
+  }
+  auto prevPktNum = largestAckedPacketNum;
+  auto timestampIt = recvdPacketInfos.crbegin();
+  size_t cumUsedSpace = 0;
+  // We start from the latest timestamp intervals
+  bool outOfSpace = false;
+  for (auto timestampIntervalsIt = receivedPktNumsIntervalSet.crbegin();
+       timestampIntervalsIt != receivedPktNumsIntervalSet.crend();
+       timestampIntervalsIt++) {
+    RecvdPacketsTimestampsRange nextTimestampRange;
+    size_t nextTimestampRangeUsedSpace = 0;
+    // Compute pktNum gap for each time-stamp range
+    if (ackFrame.recvdPacketsTimestampRanges.empty()) {
+      nextTimestampRange.gap = prevPktNum - timestampIntervalsIt->end;
+    } else {
+      nextTimestampRange.gap = prevPktNum - 2 - timestampIntervalsIt->end;
+    }
+    // Intialize spaced used by the next candidate time-stamp range
+    nextTimestampRangeUsedSpace +=
+        getQuicIntegerSizeThrows(nextTimestampRange.gap);
+
+    while (timestampIt != recvdPacketInfos.crend() &&
+           timestampIt->pktNum >= timestampIntervalsIt->start &&
+           timestampIt->pktNum <= timestampIntervalsIt->end) {
+      std::chrono::microseconds deltaDuration;
+      if (timestampIt == recvdPacketInfos.crbegin()) {
+        deltaDuration = (timestampIt->timeStamp > ackFrameMetaData.connTime)
+            ? std::chrono::duration_cast<std::chrono::microseconds>(
+                  timestampIt->timeStamp - ackFrameMetaData.connTime)
+            : 0us;
+      } else {
+        deltaDuration = std::chrono::duration_cast<std::chrono::microseconds>(
+            (timestampIt - 1)->timeStamp - timestampIt->timeStamp);
+      }
+      auto delta = deltaDuration.count() >> receiveTimestampsExponent;
+      // Check if adding a new time-stamp delta from the current time-stamp
+      // interval Will allow us to run out of space. Since adding a new delta
+      // impacts cumulative counts of deltas these are not already incorporated
+      // into nextTimestampRangeUsedSpace.
+      if (spaceLeft <
+          (cumUsedSpace + nextTimestampRangeUsedSpace +
+           getQuicIntegerSizeThrows(delta) +
+           getQuicIntegerSizeThrows(nextTimestampRange.deltas.size() + 1) +
+           getQuicIntegerSizeThrows(
+               ackFrame.recvdPacketsTimestampRanges.size() + 1))) {
+        outOfSpace = true;
+        break;
+      }
+      nextTimestampRange.deltas.push_back(delta);
+      nextTimestampRangeUsedSpace += getQuicIntegerSizeThrows(delta);
+      timestampIt++;
+    }
+    if (nextTimestampRange.deltas.size() > 0) {
+      nextTimestampRange.timestamp_delta_count =
+          nextTimestampRange.deltas.size();
+      cumUsedSpace += nextTimestampRangeUsedSpace +
+          getQuicIntegerSizeThrows(nextTimestampRange.deltas.size());
+      ackFrame.recvdPacketsTimestampRanges.push_back(nextTimestampRange);
+      prevPktNum = timestampIntervalsIt->start;
+      DCHECK(cumUsedSpace <= spaceLeft);
+    }
+    if (outOfSpace) {
+      break;
+    }
+  }
+  DCHECK(cumUsedSpace == computeSizeUsedByRecvdTimestamps(ackFrame));
+  return ackFrame.recvdPacketsTimestampRanges.size();
+}
+
+folly::Optional<WriteAckFrame> writeAckFrameToPacketBuilder(
+    const quic::AckFrameMetaData& ackFrameMetaData,
+    PacketBuilderInterface& builder,
+    FrameType frameType) {
+  if (ackFrameMetaData.ackState.acks.empty()) {
     return folly::none;
   }
+  const WriteAckState& ackState = ackFrameMetaData.ackState;
   // The last block must be the largest block.
-  auto largestAckedPacket = ackFrameMetaData.ackBlocks.back().end;
+  auto largestAckedPacket = ackState.acks.back().end;
   // ackBlocks are already an interval set so each value is naturally
   // non-overlapping.
-  auto firstAckBlockLength =
+  auto firstAckBlockLength = largestAckedPacket - ackState.acks.back().start;
-      largestAckedPacket - ackFrameMetaData.ackBlocks.back().start;
 
   WriteAckFrame ackFrame;
+  ackFrame.frameType = frameType;
   uint64_t spaceLeft = builder.remainingSpaceInPkt();
-  uint64_t beginningSpace = spaceLeft;
   ackFrame.ackBlocks.reserve(spaceLeft / 4);
 
   // We could technically split the range if the size of the representation of
@@ -297,18 +408,42 @@ folly::Optional<AckFrameWriteResult> writeAckFrame(
 
   // Required fields are Type, LargestAcked, AckDelay, AckBlockCount,
   // firstAckBlockLength
-  QuicInteger encodedintFrameType(static_cast<uint8_t>(FrameType::ACK));
+  QuicInteger encodedintFrameType(static_cast<uint8_t>(frameType));
   auto headerSize = encodedintFrameType.getSize() +
       largestAckedPacketInt.getSize() + ackDelayInt.getSize() +
       minAdditionalAckBlockCount.getSize() + firstAckBlockLengthInt.getSize();
-  if (spaceLeft < headerSize) {
+
+  size_t minAdditionalAckReceiveTimestampsFieldsSize = 0;
+  if (frameType == FrameType::ACK_RECEIVE_TIMESTAMPS) {
+    // Compute minimum size requirements for 3 fields that must be sent
+    // in every ACK_RECEIVE_TIMESTAMPS frame
+    uint64_t countTimestampRanges = 0;
+    uint64_t maybeLastPktNum = 0;
+    std::chrono::microseconds maybeLastPktTsDelta = 0us;
+    if (ackState.lastRecvdPacketInfo)
+      maybeLastPktNum = ackState.lastRecvdPacketInfo.value().pktNum;
+
+    maybeLastPktTsDelta =
+        (ackState.lastRecvdPacketInfo.value().timeStamp >
+                 ackFrameMetaData.connTime
+             ? std::chrono::duration_cast<std::chrono::microseconds>(
+                   ackState.lastRecvdPacketInfo.value().timeStamp -
+                   ackFrameMetaData.connTime)
+             : 0us);
+
+    minAdditionalAckReceiveTimestampsFieldsSize =
+        getQuicIntegerSize(countTimestampRanges).value_or(0) +
+        getQuicIntegerSize(maybeLastPktNum).value_or(0) +
+        getQuicIntegerSize(maybeLastPktTsDelta.count()).value_or(0);
+  }
+  if (spaceLeft < (headerSize + minAdditionalAckReceiveTimestampsFieldsSize)) {
     return folly::none;
   }
-  spaceLeft -= headerSize;
+  spaceLeft -= (headerSize + minAdditionalAckReceiveTimestampsFieldsSize);
 
-  ackFrame.ackBlocks.push_back(ackFrameMetaData.ackBlocks.back());
+  ackFrame.ackBlocks.push_back(ackState.acks.back());
   auto numAdditionalAckBlocks =
-      fillFrameWithAckBlocks(ackFrameMetaData.ackBlocks, ackFrame, spaceLeft);
+      fillFrameWithAckBlocks(ackState.acks, ackFrame, spaceLeft);
 
   QuicInteger numAdditionalAckBlocksInt(numAdditionalAckBlocks);
   builder.write(encodedintFrameType);
@@ -317,7 +452,7 @@ folly::Optional<AckFrameWriteResult> writeAckFrame(
   builder.write(numAdditionalAckBlocksInt);
   builder.write(firstAckBlockLengthInt);
 
-  PacketNum currentSeqNum = ackFrameMetaData.ackBlocks.back().start;
+  PacketNum currentSeqNum = ackState.acks.back().start;
   for (auto it = ackFrame.ackBlocks.cbegin() + 1;
        it != ackFrame.ackBlocks.cend();
        ++it) {
@@ -331,10 +466,108 @@ folly::Optional<AckFrameWriteResult> writeAckFrame(
     currentSeqNum = it->start;
   }
   ackFrame.ackDelay = ackFrameMetaData.ackDelay;
-  builder.appendFrame(std::move(ackFrame));
+  return ackFrame;
+}
+
+folly::Optional<AckFrameWriteResult> writeAckFrame(
+    const quic::AckFrameMetaData& ackFrameMetaData,
+    PacketBuilderInterface& builder,
+    FrameType frameType) {
+  uint64_t beginningSpace = builder.remainingSpaceInPkt();
+  auto maybeWriteAckFrame =
+      writeAckFrameToPacketBuilder(ackFrameMetaData, builder, frameType);
+
+  if (maybeWriteAckFrame.has_value()) {
+    builder.appendFrame(std::move(maybeWriteAckFrame.value()));
     return AckFrameWriteResult(
         beginningSpace - builder.remainingSpaceInPkt(),
-      1 + numAdditionalAckBlocks);
+        maybeWriteAckFrame.value(),
+        maybeWriteAckFrame.value().ackBlocks.size());
+  } else {
+    return folly::none;
+  }
+}
+
+folly::Optional<AckFrameWriteResult> writeAckFrameWithReceivedTimestamps(
+    const quic::AckFrameMetaData& ackFrameMetaData,
+    PacketBuilderInterface& builder,
+    FrameType frameType) {
+  auto beginningSpace = builder.remainingSpaceInPkt();
+  auto maybeAckFrame =
+      writeAckFrameToPacketBuilder(ackFrameMetaData, builder, frameType);
+  if (!maybeAckFrame.has_value()) {
+    return folly::none;
+  }
+  auto ackFrame = maybeAckFrame.value();
+  const WriteAckState& ackState = ackFrameMetaData.ackState;
+  uint64_t spaceLeft = builder.remainingSpaceInPkt();
+  uint64_t lastPktNum = 0;
+  std::chrono::microseconds lastPktTsDelta = 0us;
+  if (ackState.lastRecvdPacketInfo) {
+    lastPktNum = ackState.lastRecvdPacketInfo.value().pktNum;
+    lastPktTsDelta =
+        (ackState.lastRecvdPacketInfo.value().timeStamp >
+                 ackFrameMetaData.connTime
+             ? std::chrono::duration_cast<std::chrono::microseconds>(
+                   ackState.lastRecvdPacketInfo.value().timeStamp -
+                   ackFrameMetaData.connTime)
+             : 0us);
+  }
+  QuicInteger lastRecvdPacketNumInt(lastPktNum);
+  builder.write(lastRecvdPacketNumInt);
+  ackFrame.maybeLatestRecvdPacketNum = lastRecvdPacketNumInt.getValue();
+  QuicInteger lastRecvdPacketTimeInt(lastPktTsDelta.count());
+  builder.write(lastRecvdPacketTimeInt);
+  ackFrame.maybeLatestRecvdPacketTime =
+      std::chrono::microseconds(lastRecvdPacketTimeInt.getValue());
+
+  size_t countTimestampRanges = 0;
+  size_t countTimestamps = 0;
+  spaceLeft = builder.remainingSpaceInPkt();
+  if (spaceLeft > 0) {
+    auto largestAckedPacket = ackState.acks.back().end;
+    uint8_t receiveTimestampsExponentToUse =
+        ackFrameMetaData.recvTimestampsConfig.has_value()
+        ? ackFrameMetaData.recvTimestampsConfig.value()
+              .receive_timestamps_exponent
+        : kDefaultReceiveTimestampsExponent;
+    countTimestampRanges = fillPacketReceiveTimestamps(
+        ackFrameMetaData,
+        ackFrame,
+        largestAckedPacket,
+        spaceLeft,
+        receiveTimestampsExponentToUse);
+    if (countTimestampRanges > 0) {
+      QuicInteger timeStampRangeCountInt(
+          ackFrame.recvdPacketsTimestampRanges.size());
+      builder.write(timeStampRangeCountInt);
+      for (auto& recvdPacketsTimestampRanges :
+           ackFrame.recvdPacketsTimestampRanges) {
+        QuicInteger gapInt(recvdPacketsTimestampRanges.gap);
+        QuicInteger timestampDeltaCountInt(
+            recvdPacketsTimestampRanges.timestamp_delta_count);
+        builder.write(gapInt);
+        builder.write(timestampDeltaCountInt);
+        for (auto& timestamp : recvdPacketsTimestampRanges.deltas) {
+          QuicInteger deltaInt(timestamp);
+          builder.write(deltaInt);
+          countTimestamps++;
+        }
+      }
+    } else {
+      QuicInteger timeStampRangeCountInt(0);
+      builder.write(timeStampRangeCountInt);
+    }
+  }
+  auto ackFrameWriteResult = AckFrameWriteResult(
+      beginningSpace - builder.remainingSpaceInPkt(),
+      ackFrame,
+      ackFrame.ackBlocks.size(),
+      countTimestampRanges,
+      countTimestamps);
+
+  builder.appendFrame(std::move(ackFrame));
+  return ackFrameWriteResult;
 }
 
 size_t writeSimpleFrame(

quic/codec/QuicWriteCodec.h

@@ -7,37 +7,85 @@
 
 #pragma once
 
+#include <quic/QuicConstants.h>
 #include <quic/codec/QuicPacketBuilder.h>
 #include <quic/codec/Types.h>
+#include <quic/common/CircularDeque.h>
 #include <quic/common/IntervalSet.h>
+#include <quic/state/TransportSettings.h>
+#include <sys/types.h>
 #include <chrono>
+#include <cstdint>
 
 namespace quic {
 
+// Ack and PacketNumber states. This is per-packet number space.
+struct WriteAckState {
+  AckBlocks acks;
+
+  // Receive timestamp and packet number for the largest received packet.
+  //
+  // Updated whenever we receive a packet with a larger packet number
+  // than all previously received packets in the packet number space
+  // tracked by this AckState.
+  folly::Optional<RecvdPacketInfo> largestRecvdPacketInfo;
+  // Receive timestamp and packet number for the last received packet.
+  //
+  // Will be different from the value stored in largestRecvdPacketInfo
+  // if the last packet was received out of order and thus had a packet
+  // number less than that of a previously received packet in the packet
+  // number space tracked by this AckState.
+  folly::Optional<RecvdPacketInfo> lastRecvdPacketInfo;
+
+  // Packet number and timestamp of recently received packets.
+  //
+  // The maximum number of packets stored in pktsReceivedTimestamps is
+  // controlled by kMaxReceivedPktsTimestampsStored.
+  //
+  // The packet number of entries in the deque is guarenteed to increase
+  // monotonically because an entry is only added for a received packet
+  // if the packet number is greater than the packet number of the last
+  // element in the deque (e.g., entries are not added for packets that
+  // arrive out of order relative to previously received packets).
+  CircularDeque<RecvdPacketInfo> recvdPacketInfos;
+};
+
 struct AckFrameMetaData {
-  // Ack blocks. There must be at least 1 ACK block to send.
+  // ACK state.
-  const AckBlocks& ackBlocks;
+  const WriteAckState& ackState;
+
   // Delay in sending ack from time that packet was received.
   std::chrono::microseconds ackDelay;
   // The ack delay exponent to use.
   uint8_t ackDelayExponent;
 
-  AckFrameMetaData(
+  // Receive timestamps basis
-      const AckBlocks& acksIn,
+  TimePoint connTime;
-      std::chrono::microseconds ackDelayIn,
+
-      uint8_t ackDelayExponentIn)
+  folly::Optional<AckReceiveTimestampsConfig> recvTimestampsConfig =
-      : ackBlocks(acksIn),
+      folly::none;
-        ackDelay(ackDelayIn),
+
-        ackDelayExponent(ackDelayExponentIn) {}
+  folly::Optional<uint64_t> maxAckReceiveTimestampsToSend = folly::none;
 };
 
 struct AckFrameWriteResult {
   uint64_t bytesWritten;
+  WriteAckFrame writeAckFrame;
   // This includes the first ack block
   size_t ackBlocksWritten;
-
+  size_t timestampRangesWritten;
-  AckFrameWriteResult(uint64_t bytesWrittenIn, size_t ackBlocksWrittenIn)
+  size_t timestampsWritten;
-      : bytesWritten(bytesWrittenIn), ackBlocksWritten(ackBlocksWrittenIn) {}
+  AckFrameWriteResult(
+      uint64_t bytesWrittenIn,
+      WriteAckFrame writeAckFrameIn,
+      size_t ackBlocksWrittenIn,
+      size_t timestampRangesWrittenIn = 0,
+      size_t timestampsWrittenIn = 0)
+      : bytesWritten(bytesWrittenIn),
+        writeAckFrame(writeAckFrameIn),
+        ackBlocksWritten(ackBlocksWrittenIn),
+        timestampRangesWritten(timestampRangesWrittenIn),
+        timestampsWritten(timestampsWrittenIn) {}
 };
 
 /**
@@ -136,5 +184,25 @@ folly::Optional<WriteCryptoFrame> writeCryptoFrame(
  */
 folly::Optional<AckFrameWriteResult> writeAckFrame(
     const AckFrameMetaData& ackFrameMetaData,
-    PacketBuilderInterface& builder);
+    PacketBuilderInterface& builder,
+    FrameType frameType = FrameType::ACK);
+
+/**
+ * Helper functions to write the fields for ACK_RECEIVE_TIMESTAMPS frame
+ */
+size_t computeSizeUsedByRecvdTimestamps(quic::WriteAckFrame& writeAckFrame);
+
+folly::Optional<AckFrameWriteResult> writeAckFrameWithReceivedTimestamps(
+    const AckFrameMetaData& ackFrameMetaData,
+    PacketBuilderInterface& builder,
+    FrameType frameType = FrameType::ACK_RECEIVE_TIMESTAMPS);
+
+folly::Optional<quic::WriteAckFrame> writeAckFrameToPacketBuilder(
+    const quic::AckFrameMetaData& ackFrameMetaData,
+    quic::PacketBuilderInterface& builder,
+    quic::FrameType frameType);
+
 } // namespace quic
+// namespace quic
+// namespace quic
+// namespace quic

quic/codec/Types.cpp

@@ -446,6 +446,8 @@ std::string toString(FrameType frame) {
     case FrameType::GROUP_STREAM_OFF_LEN:
     case FrameType::GROUP_STREAM_OFF_LEN_FIN:
       return "GROUP_STREAM";
+    case FrameType::ACK_RECEIVE_TIMESTAMPS:
+      return "ACK_RECEIVE_TIMESTAMPS";
   }
   LOG(WARNING) << "toString has unhandled frame type";
   return "UNKNOWN";

quic/codec/Types.h

@@ -153,6 +153,14 @@ struct AckBlock {
       : startPacket(start), endPacket(end) {}
 };
 
+struct RecvdPacketsTimestampsRange {
+  uint64_t gap;
+  uint64_t timestamp_delta_count;
+  std::vector<uint64_t> deltas;
+};
+
+using RecvdPacketsTimestampsRangeVec = std::vector<RecvdPacketsTimestampsRange>;
+
 /**
  0                   1                   2                   3
  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
@@ -182,7 +190,10 @@ struct ReadAckFrame {
   // These are ordered in descending order by start packet.
   using Vec = SmallVec<AckBlock, kNumInitialAckBlocksPerFrame>;
   Vec ackBlocks;
-
+  FrameType frameType = FrameType::ACK;
+  folly::Optional<std::chrono::microseconds> maybeLatestRecvdPacketTime;
+  folly::Optional<PacketNum> maybeLatestRecvdPacketNum;
+  RecvdPacketsTimestampsRangeVec recvdPacketsTimestampRanges;
   bool operator==(const ReadAckFrame& /*rhs*/) const {
     // Can't compare ackBlocks, function is just here to appease compiler.
     return false;
@@ -197,7 +208,10 @@ struct WriteAckFrame {
   AckBlockVec ackBlocks;
   // Delay in sending ack from time that packet was received.
   std::chrono::microseconds ackDelay{0us};
-
+  FrameType frameType = FrameType::ACK;
+  folly::Optional<std::chrono::microseconds> maybeLatestRecvdPacketTime;
+  folly::Optional<PacketNum> maybeLatestRecvdPacketNum;
+  RecvdPacketsTimestampsRangeVec recvdPacketsTimestampRanges;
   bool operator==(const WriteAckFrame& /*rhs*/) const {
     // Can't compare ackBlocks, function is just here to appease compiler.
     return false;

quic/codec/test/QuicPacketRebuilderTest.cpp

@@ -9,6 +9,7 @@
 
 #include <quic/codec/QuicPacketBuilder.h>
 #include <quic/codec/QuicPacketRebuilder.h>
+#include <quic/codec/QuicWriteCodec.h>
 #include <quic/codec/test/Mocks.h>
 #include <quic/common/test/TestUtils.h>
 #include <quic/fizz/server/handshake/FizzServerQuicHandshakeContext.h>
@@ -74,7 +75,12 @@ TEST_F(QuicPacketRebuilderTest, RebuildPacket) {
   AckBlocks ackBlocks;
   ackBlocks.insert(10, 100);
   ackBlocks.insert(200, 1000);
-  AckFrameMetaData ackMeta(ackBlocks, 0us, kDefaultAckDelayExponent);
+  WriteAckState writeAckState = {.acks = ackBlocks};
+  //   AckFrameMetaData ackMeta(ackBlocks, 0us, kDefaultAckDelayExponent);
+  AckFrameMetaData ackMeta = {
+      .ackState = writeAckState,
+      .ackDelay = 0us,
+      .ackDelayExponent = static_cast<uint8_t>(kDefaultAckDelayExponent)};
   QuicServerConnectionState conn(
       FizzServerQuicHandshakeContext::Builder().build());
   conn.streamManager->setMaxLocalBidirectionalStreams(10);
@@ -401,7 +407,11 @@ TEST_F(QuicPacketRebuilderTest, CannotRebuild) {
   AckBlocks ackBlocks;
   ackBlocks.insert(10, 100);
   ackBlocks.insert(200, 1000);
-  AckFrameMetaData ackMeta(ackBlocks, 0us, kDefaultAckDelayExponent);
+  WriteAckState writeAckState = {.acks = ackBlocks};
+  AckFrameMetaData ackMeta = {
+      .ackState = writeAckState,
+      .ackDelay = 0us,
+      .ackDelayExponent = static_cast<uint8_t>(kDefaultAckDelayExponent)};
   QuicServerConnectionState conn(
       FizzServerQuicHandshakeContext::Builder().build());
   conn.streamManager->setMaxLocalBidirectionalStreams(10);

quic/common/test/TestPacketBuilders.cpp

@@ -131,10 +131,13 @@ RegularQuicPacketBuilder::Packet AckPacketBuilder::build() && {
     builder.accountForCipherOverhead(maybeAead.value()->getCipherOverhead());
   }
   DCHECK(builder.canBuildPacket());
-  AckFrameMetaData ackData(
+  WriteAckState ackState;
-      *CHECK_NOTNULL(maybeAckBlocks.get_pointer()),
+  ackState.acks = *CHECK_NOTNULL(maybeAckBlocks.get_pointer());
-      *CHECK_NOTNULL(maybeAckDelay.get_pointer()),
+  AckFrameMetaData ackData = {
-      CHECK_NOTNULL(dstConn)->transportSettings.ackDelayExponent);
+      .ackState = ackState,
+      .ackDelay = *CHECK_NOTNULL(maybeAckDelay.get_pointer()),
+      .ackDelayExponent = static_cast<uint8_t>(
+          CHECK_NOTNULL(dstConn)->transportSettings.ackDelayExponent)};
   writeAckFrame(ackData, builder);
   return std::move(builder).buildPacket();
 }

quic/fizz/client/test/QuicClientTransportTest.cpp

@@ -4086,7 +4086,12 @@ TEST_F(QuicClientTransportVersionAndRetryTest, UnencryptedAckData) {
       kDefaultUDPSendPacketLen, std::move(header), 0 /* largestAcked */);
   builder.encodePacketHeader();
   DCHECK(builder.canBuildPacket());
-  AckFrameMetaData ackData(acks, 0us, 0);
+  // AckFrameMetaData ackData(acks, 0us, 0);
+  WriteAckState writeAckState = {.acks = acks};
+  AckFrameMetaData ackData = {
+      .ackState = writeAckState,
+      .ackDelay = 0us,
+      .ackDelayExponent = static_cast<uint8_t>(kDefaultAckDelayExponent)};
   writeAckFrame(ackData, builder);
   auto packet = packetToBufCleartext(
       std::move(builder).buildPacket(),

quic/handshake/TransportParameters.h

@@ -33,6 +33,9 @@ enum class TransportParameterId : uint64_t {
   retry_source_connection_id = 0x0010,
   max_datagram_frame_size = 0x0020,
   min_ack_delay = 0xff02de1a,
+  ack_receive_timestamps_enabled = 0xff0a001,
+  max_receive_timestamps_per_ack = 0xff0a002,
+  receive_timestamps_exponent = 0xff0a003
 };
 
 struct TransportParameter {

quic/logging/QLoggerConstants.cpp

@@ -90,6 +90,8 @@ folly::StringPiece toQlogString(FrameType frame) {
     case FrameType::GROUP_STREAM_OFF_LEN:
     case FrameType::GROUP_STREAM_OFF_LEN_FIN:
       return "group_stream";
+    case FrameType::ACK_RECEIVE_TIMESTAMPS:
+      return "ack_receive_timestamps";
   }
   folly::assume_unreachable();
 }

quic/server/state/ServerStateMachine.cpp

@@ -225,6 +225,15 @@ void processClientInitialParams(
       static_cast<TransportParameterId>(kStreamGroupsEnabledCustomParamId),
       clientParams.parameters);
 
+  auto isAckReceiveTimestampsEnabled = getIntegerParameter(
+      TransportParameterId::ack_receive_timestamps_enabled,
+      clientParams.parameters);
+  auto maxReceiveTimestampsPerAck = getIntegerParameter(
+      TransportParameterId::max_receive_timestamps_per_ack,
+      clientParams.parameters);
+  auto receiveTimestampsExponent = getIntegerParameter(
+      TransportParameterId::receive_timestamps_exponent,
+      clientParams.parameters);
   if (conn.version == QuicVersion::QUIC_DRAFT ||
       conn.version == QuicVersion::QUIC_V1 ||
       conn.version == QuicVersion::QUIC_V1_ALIAS) {
@@ -397,6 +406,19 @@ void processClientInitialParams(
   if (peerMaxStreamGroupsAdvertized) {
     conn.peerMaxStreamGroupsAdvertized = *peerMaxStreamGroupsAdvertized;
   }
+  if (isAckReceiveTimestampsEnabled.has_value() &&
+      isAckReceiveTimestampsEnabled.value() == 1) {
+    if (maxReceiveTimestampsPerAck.has_value() &&
+        receiveTimestampsExponent.has_value()) {
+      conn.maybePeerAckReceiveTimestampsConfig.assign(
+          {std::min(
+               static_cast<uint8_t>(maxReceiveTimestampsPerAck.value()),
+               kMaxReceivedPktsTimestampsStored),
+           std::max(
+               static_cast<uint8_t>(receiveTimestampsExponent.value()),
+               static_cast<uint8_t>(0))});
+    }
+  }
 }
 
 void updateHandshakeState(QuicServerConnectionState& conn) {
@@ -762,10 +784,10 @@ void onServerReadDataFromOpen(
       readData.networkData.data->computeChainDataLength() == 0) {
     return;
   }
-
   bool firstPacketFromPeer = false;
   if (!conn.readCodec) {
     firstPacketFromPeer = true;
+
     folly::io::Cursor cursor(readData.networkData.data.get());
     auto initialByte = cursor.readBE<uint8_t>();
     auto parsedLongHeader = parseLongHeaderInvariant(initialByte, cursor);
@@ -857,8 +879,10 @@ void onServerReadDataFromOpen(
       conn.qLogger->setScid(conn.serverConnectionId);
       conn.qLogger->setDcid(initialDestinationConnectionId);
     }
-    conn.readCodec->setCodecParameters(
+    conn.readCodec->setCodecParameters(CodecParameters(
-        CodecParameters(conn.peerAckDelayExponent, version));
+        conn.peerAckDelayExponent,
+        version,
+        conn.transportSettings.maybeAckReceiveTimestampsConfigSentToPeer));
     conn.initialWriteCipher = cryptoFactory.getServerInitialCipher(
         initialDestinationConnectionId, version);
 
@@ -1607,6 +1631,34 @@ std::vector<TransportParameter> setSupportedExtensionTransportParameters(
     }
   }
 
+  auto ackReceiveTimestampsEnabled =
+      std::make_unique<CustomIntegralTransportParameter>(
+          static_cast<uint64_t>(
+              TransportParameterId::ack_receive_timestamps_enabled),
+          conn.transportSettings.maybeAckReceiveTimestampsConfigSentToPeer
+                  .has_value()
+              ? 1
+              : 0);
+  customTransportParams.push_back(ackReceiveTimestampsEnabled->encode());
+  if (conn.transportSettings.maybeAckReceiveTimestampsConfigSentToPeer
+          .has_value()) {
+    auto maxReceiveTimestampsPerAck =
+        std::make_unique<CustomIntegralTransportParameter>(
+            static_cast<uint64_t>(
+                TransportParameterId::max_receive_timestamps_per_ack),
+            conn.transportSettings.maybeAckReceiveTimestampsConfigSentToPeer
+                .value()
+                .max_receive_timestamps_per_ack);
+    customTransportParams.push_back(maxReceiveTimestampsPerAck->encode());
+    auto receiveTimestampsExponent =
+        std::make_unique<CustomIntegralTransportParameter>(
+            static_cast<uint64_t>(
+                TransportParameterId::receive_timestamps_exponent),
+            conn.transportSettings.maybeAckReceiveTimestampsConfigSentToPeer
+                .value()
+                .receive_timestamps_exponent);
+    customTransportParams.push_back(receiveTimestampsExponent->encode());
+  }
   return customTransportParams;
 }
 } // namespace quic

quic/server/test/QuicServerTransportTest.cpp

@@ -3287,7 +3287,11 @@ TEST_F(QuicUnencryptedServerTransportTest, TestUnencryptedAck) {
       kDefaultUDPSendPacketLen, std::move(header), 0 /* largestAcked */);
   builder.encodePacketHeader();
   DCHECK(builder.canBuildPacket());
-  AckFrameMetaData ackData(acks, 0us, 0);
+  WriteAckState writeAckState = {.acks = acks};
+  AckFrameMetaData ackData = {
+      .ackState = writeAckState,
+      .ackDelay = 0us,
+      .ackDelayExponent = static_cast<uint8_t>(kDefaultAckDelayExponent)};
   writeAckFrame(ackData, builder);
   auto packet = packetToBufCleartext(
       std::move(builder).buildPacket(),

quic/state/AckStates.h

@@ -8,8 +8,8 @@
 #pragma once
 
 #include <quic/QuicConstants.h>
+#include <quic/codec/QuicWriteCodec.h>
 #include <quic/codec/Types.h>
-#include <quic/common/CircularDeque.h>
 #include <quic/common/IntervalSet.h>
 
 #include <folly/Random.h>
@@ -17,7 +17,7 @@
 namespace quic {
 
 // Ack and PacketNumber states. This is per-packet number space.
-struct AckState {
+struct AckState : WriteAckState {
   // Largest ack that has been written to a packet
   folly::Optional<PacketNum> largestAckScheduled;
   // Count of outstanding packets received with only non-retransmittable data.
@@ -26,26 +26,12 @@ struct AckState {
   folly::Optional<TimePoint> largestRecvdPacketTime;
   // Largest received packet numbers on the connection.
   folly::Optional<PacketNum> largestRecvdPacketNum;
-  // Receive timestamp and packet number for the largest received packet.
-  //
-  // Updated whenever we receive a packet with a larger packet number
-  // than all previously received packets in the packet number space
-  // tracked by this AckState.
-  folly::Optional<RecvdPacketInfo> largestRecvdPacketInfo;
-  // Receive timestamp and packet number for the last received packet.
-  //
-  // Will be different from the value stored in largestRecvdPacketInfo
-  // if the last packet was received out of order and thus had a packet
-  // number less than that of a previously received packet in the packet
-  // number space tracked by this AckState.
-  folly::Optional<RecvdPacketInfo> lastRecvdPacketInfo;
   // Latest packet number acked by peer
   folly::Optional<PacketNum> largestAckedByPeer;
   // Largest received packet number at the time we sent our last close message.
   folly::Optional<PacketNum> largestReceivedAtLastCloseSent;
   // Next PacketNum we will send for packet in this packet number space
   PacketNum nextPacketNum{0};
-  AckBlocks acks;
   uint64_t reorderThreshold{0};
   folly::Optional<uint64_t> tolerance;
   folly::Optional<uint64_t> ackFrequencySequenceNumber;
@@ -61,17 +47,6 @@ struct AckState {
   folly::Optional<TimePoint> latestRecvdPacketTime;
   // Packet number of the latest packet
   folly::Optional<PacketNum> latestReceivedPacketNum;
-  // Packet number and timestamp of recently received packets.
-  //
-  // The maximum number of packets stored in pktsReceivedTimestamps is
-  // controlled by kMaxReceivedPktsTimestampsStored.
-  //
-  // The packet number of entries in the deque is guarenteed to increase
-  // monotonically because an entry is only added for a received packet
-  // if the packet number is greater than the packet number of the last
-  // element in the deque (e.g., entries are not added for packets that
-  // arrive out of order relative to previously received packets).
-  CircularDeque<RecvdPacketInfo> recvdPacketInfos;
 };
 
 struct AckStates {

quic/state/StateData.h

@@ -730,6 +730,13 @@ struct QuicConnectionStateBase : public folly::DelayedDestruction {
 
   // GSO supported on conn.
   folly::Optional<bool> gsoSupported;
+
+  struct AckReceiveTimestampsConfig {
+    uint64_t maxReceiveTimestampsPerAck;
+    uint64_t receiveTimestampsExponent;
+  };
+  folly::Optional<AckReceiveTimestampsConfig>
+      maybePeerAckReceiveTimestampsConfig;
 };
 
 std::ostream& operator<<(std::ostream& os, const QuicConnectionStateBase& st);

quic/state/TransportSettings.h

@@ -11,6 +11,7 @@
 #include <quic/codec/QuicConnectionId.h>
 #include <quic/d6d/Types.h>
 #include <chrono>
+#include <cstdint>
 
 namespace quic {
 
@@ -133,6 +134,11 @@ struct DatagramConfig {
   uint32_t writeBufSize{kDefaultMaxDatagramsBuffered};
 };
 
+struct AckReceiveTimestampsConfig {
+  uint64_t max_receive_timestamps_per_ack{kMaxReceivedPktsTimestampsStored};
+  uint64_t receive_timestamps_exponent{kDefaultReceiveTimestampsExponent};
+};
+
 // JSON-serialized transport knobs
 struct SerializedKnob {
   uint64_t space;
@@ -325,6 +331,19 @@ struct TransportSettings {
   bool experimentalPacer{false};
   // experimental flag to close ingress SM when invoking stopSending
   bool dropIngressOnStopSending{false};
+
+  // Local configuration for ACK receive timestamps.
+  //
+  // Determines the ACK receive timestamp configuration sent to peer,
+  // which in turn determines the maximum number of timestamps and
+  // timestamp resolution included in ACK messages sent by the peer
+  // if the peer supports ACK receive timestamps.
+  //
+  // If structure is not initialized, ACK receive timestamps are
+  // not requested from peer regardless of whether the peer
+  // supports them.
+  folly::Optional<AckReceiveTimestampsConfig>
+      maybeAckReceiveTimestampsConfigSentToPeer;
 };
 
 } // namespace quic