Packets inflight, packets sent in OutstandingPacketMetadata

Summary:
- Adds counter of the number of ack-eliciting packets sent; useful for understanding lost / retransmission numbers
- Adds the following to `OutstandingPacketMetadata`
  - # of packets sent and # of ack-eliciting packets sent; this enables indexing of each packet when processed by an observer on loss / RTT event, including understanding its ordering in the flow of sent packets.
  - # of packets in flight; useful during loss analysis to understand if self-induced congestion could be culprit
- Fixes the inflightBytes counter in `OutstandingPacketMetadata`; it was previously _not_ including the packets own bytes, despite saying that it was.

Right now we are passing multiple integers into the `OutstandingPacket` constructor. I've switched to passing in `LossState` to avoid passing in two more integers, although I will need to come back and clean up the existing ones.

Differential Revision: D25861702

fbshipit-source-id: e34c0edcb136bc1a2a6aeb898ecbb4cf11d0aa2c

quic/api/QuicSocket.h

@@ -133,6 +133,7 @@ class QuicSocket {
     std::chrono::microseconds pacingInterval{0us};
     uint32_t packetsRetransmitted{0};
     uint32_t totalPacketsSent{0};
+    uint32_t totalAckElicitingPacketsSent{0};
     uint32_t totalPacketsMarkedLost{0};
     uint32_t totalPacketsMarkedLostByPto{0};
     uint32_t totalPacketsMarkedLostByReorderingThreshold{0};

quic/api/QuicTransportBase.cpp

@@ -583,6 +583,8 @@ QuicSocket::TransportInfo QuicTransportBase::getTransportInfo() const {
   transportInfo.pacingInterval = pacingInterval;
   transportInfo.packetsRetransmitted = conn_->lossState.rtxCount;
   transportInfo.totalPacketsSent = conn_->lossState.totalPacketsSent;
+  transportInfo.totalAckElicitingPacketsSent =
+      conn_->lossState.totalAckElicitingPacketsSent;
   transportInfo.totalPacketsMarkedLost =
       conn_->lossState.totalPacketsMarkedLost;
   transportInfo.totalPacketsMarkedLostByPto =

quic/api/QuicTransportFunctions.cpp

@@ -667,6 +667,7 @@ void updateConnection(
     DCHECK(!packetEvent);
     return;
   }
+  conn.lossState.totalAckElicitingPacketsSent++;
   auto packetIt =
       std::find_if(
           conn.outstandings.packets.rbegin(),
@@ -679,12 +680,18 @@ void updateConnection(
   auto& pkt = *conn.outstandings.packets.emplace(
       packetIt,
       std::move(packet),
-      std::move(sentTime),
+      sentTime,
       encodedSize,
       isHandshake,
       isD6DProbe,
+      // these numbers should all _include_ the current packet
+      // conn.lossState.inflightBytes isn't updated until below
+      // conn.outstandings.numOutstanding() + 1 since we're emplacing here
       conn.lossState.totalBytesSent,
-      conn.lossState.inflightBytes);
+      conn.lossState.inflightBytes + encodedSize,
+      conn.outstandings.numOutstanding() + 1,
+      conn.lossState);
+
   if (isD6DProbe) {
     ++conn.d6d.outstandingProbes;
     ++conn.d6d.meta.totalTxedProbes;

quic/api/test/QuicPacketSchedulerTest.cpp

@@ -40,7 +40,8 @@ PacketNum addInitialOutstandingPacket(QuicConnectionStateBase& conn) {
       nextPacketNum,
       QuicVersion::QUIC_DRAFT);
   RegularQuicWritePacket packet(std::move(header));
-  conn.outstandings.packets.emplace_back(packet, Clock::now(), 0, true, 0, 0);
+  conn.outstandings.packets.emplace_back(
+      packet, Clock::now(), 0, true, 0, 0, 0, LossState());
   conn.outstandings.handshakePacketsCount++;
   increaseNextPacketNum(conn, PacketNumberSpace::Handshake);
   return nextPacketNum;
@@ -58,7 +59,8 @@ PacketNum addHandshakeOutstandingPacket(QuicConnectionStateBase& conn) {
       nextPacketNum,
       QuicVersion::QUIC_DRAFT);
   RegularQuicWritePacket packet(std::move(header));
-  conn.outstandings.packets.emplace_back(packet, Clock::now(), 0, true, 0, 0);
+  conn.outstandings.packets.emplace_back(
+      packet, Clock::now(), 0, true, 0, 0, 0, LossState());
   conn.outstandings.handshakePacketsCount++;
   increaseNextPacketNum(conn, PacketNumberSpace::Handshake);
   return nextPacketNum;
@@ -71,7 +73,8 @@ PacketNum addOutstandingPacket(QuicConnectionStateBase& conn) {
       conn.clientConnectionId.value_or(quic::test::getTestConnectionId()),
       nextPacketNum);
   RegularQuicWritePacket packet(std::move(header));
-  conn.outstandings.packets.emplace_back(packet, Clock::now(), 0, false, 0, 0);
+  conn.outstandings.packets.emplace_back(
+      packet, Clock::now(), 0, false, 0, 0, 0, LossState());
   increaseNextPacketNum(conn, PacketNumberSpace::AppData);
   return nextPacketNum;
 }

quic/api/test/QuicTransportFunctionsTest.cpp

@@ -908,6 +908,9 @@ TEST_F(QuicTransportFunctionsTest, TestUpdateConnectionWithPureAck) {
       std::make_unique<NiceMock<MockCongestionController>>();
   auto rawController = mockCongestionController.get();
   conn->congestionController = std::move(mockCongestionController);
+  EXPECT_EQ(0, conn->lossState.totalPacketsSent);
+  EXPECT_EQ(0, conn->lossState.totalAckElicitingPacketsSent);
+  EXPECT_EQ(0, conn->outstandings.packets.size());
   ASSERT_EQ(0, conn->lossState.totalBytesAcked);
   WriteAckFrame ackFrame;
   ackFrame.ackBlocks.emplace_back(0, 10);
@@ -916,6 +919,8 @@ TEST_F(QuicTransportFunctionsTest, TestUpdateConnectionWithPureAck) {
   EXPECT_CALL(*rawPacer, onPacketSent()).Times(0);
   updateConnection(
       *conn, folly::none, packet.packet, TimePoint(), getEncodedSize(packet));
+  EXPECT_EQ(1, conn->lossState.totalPacketsSent);
+  EXPECT_EQ(0, conn->lossState.totalAckElicitingPacketsSent);
   EXPECT_EQ(0, conn->outstandings.packets.size());
   EXPECT_EQ(0, conn->lossState.totalBytesAcked);
   std::shared_ptr<quic::FileQLogger> qLogger =
@@ -947,13 +952,18 @@ TEST_F(QuicTransportFunctionsTest, TestUpdateConnectionWithBytesStats) {
   packet.packet.frames.push_back(std::move(writeStreamFrame));
   conn->lossState.totalBytesSent = 13579;
   conn->lossState.totalBytesAcked = 8642;
+  conn->lossState.inflightBytes = 16000;
   auto currentTime = Clock::now();
   conn->lossState.lastAckedTime = currentTime - 123s;
   conn->lossState.adjustedLastAckedTime = currentTime - 123s;
   conn->lossState.lastAckedPacketSentTime = currentTime - 234s;
   conn->lossState.totalBytesSentAtLastAck = 10000;
   conn->lossState.totalBytesAckedAtLastAck = 5000;
+  conn->lossState.totalPacketsSent = 20;
+  conn->lossState.totalAckElicitingPacketsSent = 15;
   updateConnection(*conn, folly::none, packet.packet, TimePoint(), 555);
+  EXPECT_EQ(21, conn->lossState.totalPacketsSent);
+  EXPECT_EQ(16, conn->lossState.totalAckElicitingPacketsSent);
 
   // verify QLogger contains correct packet information
   std::shared_ptr<quic::FileQLogger> qLogger =
@@ -972,6 +982,27 @@ TEST_F(QuicTransportFunctionsTest, TestUpdateConnectionWithBytesStats) {
       13579 + 555,
       getFirstOutstandingPacket(*conn, PacketNumberSpace::Handshake)
           ->metadata.totalBytesSent);
+  EXPECT_EQ(
+      16000 + 555,
+      getFirstOutstandingPacket(*conn, PacketNumberSpace::Handshake)
+          ->metadata.inflightBytes);
+  EXPECT_EQ(
+      1,
+      getFirstOutstandingPacket(*conn, PacketNumberSpace::Handshake)
+          ->metadata.packetsInflight);
+  EXPECT_EQ(
+      555,
+      getFirstOutstandingPacket(*conn, PacketNumberSpace::Handshake)
+          ->metadata.encodedSize);
+  EXPECT_EQ(
+      20 + 1,
+      getFirstOutstandingPacket(*conn, PacketNumberSpace::Handshake)
+          ->metadata.totalPacketsSent);
+  EXPECT_EQ(
+      15 + 1,
+      getFirstOutstandingPacket(*conn, PacketNumberSpace::Handshake)
+          ->metadata.totalAckElicitingPacketsSent);
+
   EXPECT_TRUE(getFirstOutstandingPacket(*conn, PacketNumberSpace::Handshake)
                   ->lastAckedPacketInfo.has_value());
   EXPECT_EQ(

quic/codec/test/QuicPacketRebuilderTest.cpp

@@ -29,7 +29,14 @@ OutstandingPacket makeDummyOutstandingPacket(
     const RegularQuicWritePacket& writePacket,
     uint64_t totalBytesSentOnConnection) {
   OutstandingPacket packet(
-      writePacket, Clock::now(), 1000, false, totalBytesSentOnConnection, 0);
+      writePacket,
+      Clock::now(),
+      1000,
+      false,
+      totalBytesSentOnConnection,
+      0,
+      0,
+      LossState());
   return packet;
 }
 

quic/common/test/TestUtils.cpp

@@ -571,7 +571,14 @@ OutstandingPacket makeTestingWritePacket(
       QuicVersion::MVFST);
   RegularQuicWritePacket packet(std::move(longHeader));
   return OutstandingPacket(
-      packet, sentTime, desiredSize, false, totalBytesSent, inflightBytes);
+      packet,
+      sentTime,
+      desiredSize,
+      false,
+      totalBytesSent,
+      inflightBytes,
+      0,
+      LossState());
 }
 
 CongestionController::AckEvent makeAck(

quic/congestion_control/test/Copa2Test.cpp

@@ -28,7 +28,14 @@ class Copa2Test : public Test {
     RegularQuicWritePacket packet(
         ShortHeader(ProtectionType::KeyPhaseZero, connId, packetNum));
     return OutstandingPacket(
-        std::move(packet), Clock::now(), size, false, totalSent, inflight);
+        std::move(packet),
+        Clock::now(),
+        size,
+        false,
+        totalSent,
+        inflight,
+        0,
+        LossState());
   }
 
   CongestionController::AckEvent createAckEvent(

quic/congestion_control/test/CopaTest.cpp

@@ -31,7 +31,14 @@ class CopaTest : public Test {
           ShortHeader(ProtectionType::KeyPhaseZero, connId, packetData.first));
       totalSentBytes += 10;
       loss.addLostPacket(OutstandingPacket(
-          std::move(packet), Clock::now(), 10, false, totalSentBytes, 0));
+          std::move(packet),
+          Clock::now(),
+          10,
+          false,
+          totalSentBytes,
+          0,
+          0,
+          LossState()));
       loss.lostBytes = packetData.second;
     }
     loss.lostPackets = lostPackets.size();
@@ -47,7 +54,14 @@ class CopaTest : public Test {
     RegularQuicWritePacket packet(
         ShortHeader(ProtectionType::KeyPhaseZero, connId, packetNum));
     return OutstandingPacket(
-        std::move(packet), Clock::now(), size, false, totalSent, inflight);
+        std::move(packet),
+        Clock::now(),
+        size,
+        false,
+        totalSent,
+        inflight,
+        0,
+        LossState());
   }
 
   CongestionController::AckEvent createAckEvent(

quic/congestion_control/test/NewRenoTest.cpp

@@ -26,8 +26,8 @@ CongestionController::LossEvent createLossEvent(
   for (auto packetData : lostPackets) {
     RegularQuicWritePacket packet(
         ShortHeader(ProtectionType::KeyPhaseZero, connId, packetData.first));
-    loss.addLostPacket(
+    loss.addLostPacket(OutstandingPacket(
-        OutstandingPacket(std::move(packet), Clock::now(), 10, false, 10, 0));
+        std::move(packet), Clock::now(), 10, false, 10, 0, 0, LossState()));
     loss.lostBytes = packetData.second;
   }
   loss.lostPackets = lostPackets.size();
@@ -46,7 +46,14 @@ CongestionController::AckEvent createAckEvent(
   ack.ackedBytes = ackedSize;
   ack.ackedPackets.push_back(
       makeAckPacketFromOutstandingPacket(OutstandingPacket(
-          std::move(packet), packetSentTime, ackedSize, false, ackedSize, 0)));
+          std::move(packet),
+          packetSentTime,
+          ackedSize,
+          false,
+          ackedSize,
+          0,
+          0,
+          LossState())));
   return ack;
 }
 
@@ -59,7 +66,7 @@ OutstandingPacket createPacket(
   RegularQuicWritePacket packet(
       ShortHeader(ProtectionType::KeyPhaseZero, connId, packetNum));
   return OutstandingPacket(
-      std::move(packet), sendTime, size, false, size, inflight);
+      std::move(packet), sendTime, size, false, size, inflight, 0, LossState());
 }
 
 TEST_F(NewRenoTest, TestLoss) {

quic/d6d/test/QuicD6DStateFunctionsTest.cpp

@@ -209,7 +209,9 @@ TEST_F(QuicD6DStateFunctionsTest, D6DProbeAckedInBase) {
       false,
       true,
       d6d.currentProbeSize,
-      d6d.currentProbeSize);
+      d6d.currentProbeSize,
+      0,
+      LossState());
   d6d.lastProbe = D6DProbePacket(
       pkt.packet.header.getPacketSequenceNum(), pkt.metadata.encodedSize);
   d6d.raiser = std::make_unique<MockProbeSizeRaiser>();
@@ -253,7 +255,9 @@ TEST_F(QuicD6DStateFunctionsTest, D6DProbeAckedInSearchingOne) {
       false,
       true,
       d6d.currentProbeSize,
-      d6d.currentProbeSize);
+      d6d.currentProbeSize,
+      0,
+      LossState());
   d6d.lastProbe = D6DProbePacket(
       pkt.packet.header.getPacketSequenceNum(), pkt.metadata.encodedSize);
   d6d.raiser = std::make_unique<MockProbeSizeRaiser>();
@@ -298,7 +302,9 @@ TEST_F(QuicD6DStateFunctionsTest, D6DProbeAckedInSearchingMax) {
       false,
       true,
       d6d.currentProbeSize,
-      d6d.currentProbeSize);
+      d6d.currentProbeSize,
+      0,
+      LossState());
   d6d.lastProbe = D6DProbePacket(
       pkt.packet.header.getPacketSequenceNum(), pkt.metadata.encodedSize);
   d6d.raiser = std::make_unique<MockProbeSizeRaiser>();
@@ -351,7 +357,9 @@ TEST_F(QuicD6DStateFunctionsTest, D6DProbeAckedInError) {
       false,
       true,
       d6d.currentProbeSize,
-      d6d.currentProbeSize);
+      d6d.currentProbeSize,
+      0,
+      LossState());
   d6d.lastProbe = D6DProbePacket(
       pkt.packet.header.getPacketSequenceNum(), pkt.metadata.encodedSize);
   d6d.raiser = std::make_unique<MockProbeSizeRaiser>();
@@ -394,7 +402,9 @@ TEST_F(QuicD6DStateFunctionsTest, BlackholeInSearching) {
       false,
       true,
       d6d.currentProbeSize,
-      d6d.currentProbeSize);
+      d6d.currentProbeSize,
+      0,
+      LossState());
   d6d.lastProbe = D6DProbePacket(
       pkt.packet.header.getPacketSequenceNum(), pkt.metadata.encodedSize);
 
@@ -404,7 +414,9 @@ TEST_F(QuicD6DStateFunctionsTest, BlackholeInSearching) {
       conn.udpSendPacketLen,
       false,
       conn.udpSendPacketLen + d6d.currentProbeSize,
-      conn.udpSendPacketLen + d6d.currentProbeSize);
+      conn.udpSendPacketLen + d6d.currentProbeSize,
+      0,
+      LossState());
 
   d6d.thresholdCounter = std::make_unique<WindowedCounter<uint64_t, uint64_t>>(
       std::chrono::microseconds(kDefaultD6DBlackholeDetectionWindow).count(),
@@ -459,7 +471,9 @@ TEST_F(QuicD6DStateFunctionsTest, BlackholeInSearchComplete) {
       false,
       true,
       d6d.currentProbeSize,
-      d6d.currentProbeSize);
+      d6d.currentProbeSize,
+      0,
+      LossState());
   d6d.lastProbe = D6DProbePacket(
       pkt.packet.header.getPacketSequenceNum(), pkt.metadata.encodedSize);
 
@@ -469,7 +483,9 @@ TEST_F(QuicD6DStateFunctionsTest, BlackholeInSearchComplete) {
       conn.udpSendPacketLen,
       false,
       conn.udpSendPacketLen + d6d.currentProbeSize,
-      conn.udpSendPacketLen + d6d.currentProbeSize);
+      conn.udpSendPacketLen + d6d.currentProbeSize,
+      0,
+      LossState());
 
   d6d.thresholdCounter = std::make_unique<WindowedCounter<uint64_t, uint64_t>>(
       std::chrono::microseconds(kDefaultD6DBlackholeDetectionWindow).count(),
@@ -527,7 +543,9 @@ TEST_F(QuicD6DStateFunctionsTest, ReachMaxPMTU) {
       false,
       true,
       d6d.currentProbeSize,
-      d6d.currentProbeSize);
+      d6d.currentProbeSize,
+      0,
+      LossState());
   d6d.lastProbe = D6DProbePacket(
       pkt.packet.header.getPacketSequenceNum(), pkt.metadata.encodedSize);
   d6d.raiser = std::make_unique<MockProbeSizeRaiser>();
@@ -564,7 +582,9 @@ TEST_F(
       false,
       true,
       d6d.currentProbeSize,
-      d6d.currentProbeSize);
+      d6d.currentProbeSize,
+      0,
+      LossState());
   d6d.lastProbe = D6DProbePacket(
       pkt.packet.header.getPacketSequenceNum(), pkt.metadata.encodedSize);
   d6d.raiser = std::make_unique<MockProbeSizeRaiser>();
@@ -583,7 +603,9 @@ TEST_F(
       false,
       false,
       d6d.currentProbeSize,
-      d6d.currentProbeSize);
+      d6d.currentProbeSize,
+      0,
+      LossState());
   // Generate a false positive blackhole signal
   detectPMTUBlackhole(conn, lostPacket);
   EXPECT_EQ(d6d.state, D6DMachineState::BASE);
@@ -619,7 +641,9 @@ TEST_F(QuicD6DStateFunctionsTest, UpperboundIsBase) {
       false,
       true,
       d6d.currentProbeSize,
-      d6d.currentProbeSize);
+      d6d.currentProbeSize,
+      0,
+      LossState());
   d6d.lastProbe = D6DProbePacket(
       pkt.packet.header.getPacketSequenceNum(), pkt.metadata.encodedSize);
   d6d.raiser = std::make_unique<MockProbeSizeRaiser>();

quic/loss/test/QuicLossFunctionsTest.cpp

@@ -249,7 +249,9 @@ PacketNum QuicLossFunctionsTest::sendPacket(
       isHandshake,
       isD6DProbe,
       encodedSize,
-      0);
+      0,
+      0,
+      LossState());
   outstandingPacket.associatedEvent = associatedEvent;
   conn.lossState.lastRetransmittablePacketSentTime = time;
   if (conn.congestionController) {
@@ -1066,8 +1068,8 @@ TEST_F(QuicLossFunctionsTest, TestHandleAckForLoss) {
       conn->version.value());
   RegularQuicWritePacket outstandingRegularPacket(std::move(longHeader));
   auto now = Clock::now();
-  conn->outstandings.packets.emplace_back(
+  conn->outstandings.packets.emplace_back(OutstandingPacket(
-      OutstandingPacket(outstandingRegularPacket, now, 0, false, 0, 0));
+      outstandingRegularPacket, now, 0, false, 0, 0, 0, LossState()));
 
   bool testLossMarkFuncCalled = false;
   auto testLossMarkFunc = [&](auto& /* conn */, auto&, bool) {

quic/state/LossState.h

@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ */
+
+#pragma once
+
+#include <quic/codec/Types.h>
+#include <quic/common/EnumArray.h>
+
+namespace quic {
+
+struct LossState {
+  enum class AlarmMethod { EarlyRetransmitOrReordering, PTO };
+  // Latest packet number sent
+  // TODO: this also needs to be 3 numbers now...
+  folly::Optional<PacketNum> largestSent;
+  // Timer for time reordering detection or early retransmit alarm.
+  EnumArray<PacketNumberSpace, folly::Optional<TimePoint>> lossTimes;
+  // Max ack delay received from peer
+  std::chrono::microseconds maxAckDelay{0us};
+  // minimum rtt. AckDelay isn't excluded from this.
+  std::chrono::microseconds mrtt{kDefaultMinRtt};
+  // Smooth rtt
+  std::chrono::microseconds srtt{0us};
+  // Latest rtt
+  std::chrono::microseconds lrtt{0us};
+  // Rtt var
+  std::chrono::microseconds rttvar{0us};
+  // The sent time of the latest acked packet
+  folly::Optional<TimePoint> lastAckedPacketSentTime;
+  // The latest time a packet is acked
+  folly::Optional<TimePoint> lastAckedTime;
+  // The latest time a packet is acked, minus ack delay
+  folly::Optional<TimePoint> adjustedLastAckedTime;
+  // The time when last retranmittable packet is sent for every packet number
+  // space
+  TimePoint lastRetransmittablePacketSentTime;
+  // Total number of bytes sent on this connection. This is after encoding.
+  uint64_t totalBytesSent{0};
+  // Total number of bytes received on this connection. This is before decoding.
+  uint64_t totalBytesRecvd{0};
+  // Total number of stream bytes retransmitted, excluding cloning.
+  uint64_t totalBytesRetransmitted{0};
+  // Total number of stream bytes cloned.
+  uint64_t totalStreamBytesCloned{0};
+  // Total number of bytes cloned.
+  uint64_t totalBytesCloned{0};
+  // Total number of bytes acked on this connection. If a packet is acked twice,
+  // it won't be count twice. Pure acks packets are NOT included.
+  uint64_t totalBytesAcked{0};
+  // The total number of bytes sent on this connection when the last time a
+  // packet is acked.
+  uint64_t totalBytesSentAtLastAck{0};
+  // The total number of bytes acked on this connection when the last time a
+  // packet is acked.
+  uint64_t totalBytesAckedAtLastAck{0};
+  // Total number of packets sent on this connection, including retransmissions.
+  uint32_t totalPacketsSent{0};
+  // Total number of ack-eliciting packets sent on this connection.
+  uint32_t totalAckElicitingPacketsSent{0};
+  // Total number of packets which were declared lost, including losses that
+  // we later detected were spurious (see totalPacketsSpuriouslyMarkedLost).
+  uint32_t totalPacketsMarkedLost{0};
+  // Total number of packets which were declared lost due to PTO; a packet can
+  // marked as lost by multiple detection mechanisms.
+  uint32_t totalPacketsMarkedLostByPto{0};
+  // Total number of packets which were declared lost based on the reordering
+  // threshold; a packet can marked as lost by multiple detection mechanisms.
+  uint32_t totalPacketsMarkedLostByReorderingThreshold{0};
+  // Total number of packets which were declared lost spuriously, i.e. we
+  // received an ACK for them later.
+  uint32_t totalPacketsSpuriouslyMarkedLost{0};
+  // Inflight bytes
+  uint64_t inflightBytes{0};
+  // Reordering threshold used
+  uint32_t reorderingThreshold{kReorderingThreshold};
+  // Number of packet loss timer fired before receiving an ack
+  uint32_t ptoCount{0};
+  // Total number of packet retransmitted on this connection, including packet
+  // clones, retransmitted clones, handshake and rejected zero rtt packets.
+  uint32_t rtxCount{0};
+  // Total number of retransmission due to PTO
+  uint32_t timeoutBasedRtxCount{0};
+  // Total number of PTO count
+  uint32_t totalPTOCount{0};
+  // Current method by which the loss detection alarm is set.
+  AlarmMethod currentAlarmMethod{AlarmMethod::EarlyRetransmitOrReordering};
+};
+} // namespace quic

quic/state/OutstandingPacket.h

@@ -9,6 +9,7 @@
 #pragma once
 
 #include <quic/codec/Types.h>
+#include <quic/state/LossState.h>
 #include <quic/state/PacketEvent.h>
 
 namespace quic {
@@ -28,6 +29,12 @@ struct OutstandingPacketMetadata {
   // Bytes in flight on this connection including this packet itself when this
   // packet is sent.
   uint64_t inflightBytes;
+  // Packets in flight on this connection including this packet itself.
+  uint64_t packetsInflight;
+  // Total number of packets sent on this connection.
+  uint32_t totalPacketsSent{0};
+  // Total number of ack-eliciting packets sent on this connection.
+  uint32_t totalAckElicitingPacketsSent{0};
 
   OutstandingPacketMetadata(
       TimePoint timeIn,
@@ -35,13 +42,19 @@ struct OutstandingPacketMetadata {
       bool isHandshakeIn,
       bool isD6DProbeIn,
       uint64_t totalBytesSentIn,
-      uint64_t inflightBytesIn)
+      uint64_t inflightBytesIn,
-      : time(std::move(timeIn)),
+      uint64_t packetsInflightIn,
+      const LossState& lossStateIn)
+      : time(timeIn),
         encodedSize(encodedSizeIn),
         isHandshake(isHandshakeIn),
         isD6DProbe(isD6DProbeIn),
         totalBytesSent(totalBytesSentIn),
-        inflightBytes(inflightBytesIn) {}
+        inflightBytes(inflightBytesIn),
+        packetsInflight(packetsInflightIn),
+        totalPacketsSent(lossStateIn.totalPacketsSent),
+        totalAckElicitingPacketsSent(lossStateIn.totalAckElicitingPacketsSent) {
+  }
 };
 
 // Data structure to represent outstanding retransmittable packets
@@ -104,15 +117,19 @@ struct OutstandingPacket {
       uint32_t encodedSizeIn,
       bool isHandshakeIn,
       uint64_t totalBytesSentIn,
-      uint64_t inflightBytesIn)
+      uint64_t inflightBytesIn,
+      uint64_t packetsInflightIn,
+      const LossState& lossStateIn)
       : packet(std::move(packetIn)),
         metadata(OutstandingPacketMetadata(
-            std::move(timeIn),
+            timeIn,
             encodedSizeIn,
             isHandshakeIn,
             false,
             totalBytesSentIn,
-            inflightBytesIn)) {}
+            inflightBytesIn,
+            packetsInflightIn,
+            lossStateIn)) {}
 
   OutstandingPacket(
       RegularQuicWritePacket packetIn,
@@ -121,14 +138,18 @@ struct OutstandingPacket {
       bool isHandshakeIn,
       bool isD6DProbeIn,
       uint64_t totalBytesSentIn,
-      uint64_t inflightBytesIn)
+      uint64_t inflightBytesIn,
+      uint64_t packetsInflightIn,
+      const LossState& lossStateIn)
       : packet(std::move(packetIn)),
         metadata(OutstandingPacketMetadata(
-            std::move(timeIn),
+            timeIn,
             encodedSizeIn,
             isHandshakeIn,
             isD6DProbeIn,
             totalBytesSentIn,
-            inflightBytesIn)) {}
+            inflightBytesIn,
+            packetsInflightIn,
+            lossStateIn)) {}
 };
 } // namespace quic

quic/state/StateData.h

@@ -15,12 +15,12 @@
 #include <quic/codec/QuicWriteCodec.h>
 #include <quic/codec/Types.h>
 #include <quic/common/BufAccessor.h>
-#include <quic/common/EnumArray.h>
 #include <quic/common/WindowedCounter.h>
 #include <quic/d6d/ProbeSizeRaiser.h>
 #include <quic/handshake/HandshakeLayer.h>
 #include <quic/logging/QLogger.h>
 #include <quic/state/AckStates.h>
+#include <quic/state/LossState.h>
 #include <quic/state/OutstandingPacket.h>
 #include <quic/state/PacketEvent.h>
 #include <quic/state/PendingPathRateLimiter.h>
@@ -407,82 +407,6 @@ using Resets = folly::F14FastMap<StreamId, RstStreamFrame>;
 
 using FrameList = std::vector<QuicSimpleFrame>;
 
-struct LossState {
-  enum class AlarmMethod { EarlyRetransmitOrReordering, PTO };
-  // Latest packet number sent
-  // TODO: this also needs to be 3 numbers now...
-  folly::Optional<PacketNum> largestSent;
-  // Timer for time reordering detection or early retransmit alarm.
-  EnumArray<PacketNumberSpace, folly::Optional<TimePoint>> lossTimes;
-  // Max ack delay received from peer
-  std::chrono::microseconds maxAckDelay{0us};
-  // minimum rtt. AckDelay isn't excluded from this.
-  std::chrono::microseconds mrtt{kDefaultMinRtt};
-  // Smooth rtt
-  std::chrono::microseconds srtt{0us};
-  // Latest rtt
-  std::chrono::microseconds lrtt{0us};
-  // Rtt var
-  std::chrono::microseconds rttvar{0us};
-  // The sent time of the latest acked packet
-  folly::Optional<TimePoint> lastAckedPacketSentTime;
-  // The latest time a packet is acked
-  folly::Optional<TimePoint> lastAckedTime;
-  // The latest time a packet is acked, minus ack delay
-  folly::Optional<TimePoint> adjustedLastAckedTime;
-  // The time when last retranmittable packet is sent for every packet number
-  // space
-  TimePoint lastRetransmittablePacketSentTime;
-  // Total number of bytes sent on this connection. This is after encoding.
-  uint64_t totalBytesSent{0};
-  // Total number of bytes received on this connection. This is before decoding.
-  uint64_t totalBytesRecvd{0};
-  // Total number of stream bytes retransmitted, excluding cloning.
-  uint64_t totalBytesRetransmitted{0};
-  // Total number of stream bytes cloned.
-  uint64_t totalStreamBytesCloned{0};
-  // Total number of bytes cloned.
-  uint64_t totalBytesCloned{0};
-  // Total number of bytes acked on this connection. If a packet is acked twice,
-  // it won't be count twice. Pure acks packets are NOT included.
-  uint64_t totalBytesAcked{0};
-  // The total number of bytes sent on this connection when the last time a
-  // packet is acked.
-  uint64_t totalBytesSentAtLastAck{0};
-  // The total number of bytes acked on this connection when the last time a
-  // packet is acked.
-  uint64_t totalBytesAckedAtLastAck{0};
-  // Total number of packets sent on this connection, including retransmissions.
-  uint32_t totalPacketsSent{0};
-  // Total number of packets which were declared lost, including losses that
-  // we later detected were spurious (see totalPacketsSpuriouslyMarkedLost).
-  uint32_t totalPacketsMarkedLost{0};
-  // Total number of packets which were declared lost due to PTO; a packet can
-  // marked as lost by multiple detection mechanisms.
-  uint32_t totalPacketsMarkedLostByPto{0};
-  // Total number of packets which were declared lost based on the reordering
-  // threshold; a packet can marked as lost by multiple detection mechanisms.
-  uint32_t totalPacketsMarkedLostByReorderingThreshold{0};
-  // Total number of packets which were declared lost spuriously, i.e. we
-  // received an ACK for them later.
-  uint32_t totalPacketsSpuriouslyMarkedLost{0};
-  // Inflight bytes
-  uint64_t inflightBytes{0};
-  // Reordering threshold used
-  uint32_t reorderingThreshold{kReorderingThreshold};
-  // Number of packet loss timer fired before receiving an ack
-  uint32_t ptoCount{0};
-  // Total number of packet retransmitted on this connection, including packet
-  // clones, retransmitted clones, handshake and rejected zero rtt packets.
-  uint32_t rtxCount{0};
-  // Total number of retransmission due to PTO
-  uint32_t timeoutBasedRtxCount{0};
-  // Total number of PTO count
-  uint32_t totalPTOCount{0};
-  // Current method by which the loss detection alarm is set.
-  AlarmMethod currentAlarmMethod{AlarmMethod::EarlyRetransmitOrReordering};
-};
-
 class Logger;
 class CongestionControllerFactory;
 class LoopDetectorCallback;

quic/state/test/AckHandlersTest.cpp

@@ -57,7 +57,9 @@ auto emplacePackets(
         1,
         false /* handshake */,
         packetNum,
-        packetNum + 1);
+        packetNum + 1,
+        packetNum + 1,
+        quic::LossState());
     conn.outstandings.packets.emplace_back(sentPacket);
     packetNum++;
   }
@@ -81,7 +83,14 @@ TEST_P(AckHandlersTest, TestAckMultipleSequentialBlocks) {
     WriteStreamFrame frame(currentStreamId++, 0, 0, true);
     regularPacket.frames.emplace_back(std::move(frame));
     conn.outstandings.packets.emplace_back(OutstandingPacket(
-        std::move(regularPacket), sentTime, 1, false, packetNum, 0));
+        std::move(regularPacket),
+        sentTime,
+        1,
+        false,
+        packetNum,
+        0,
+        0,
+        LossState()));
   }
   ReadAckFrame ackFrame;
   ackFrame.largestAcked = 101;
@@ -152,7 +161,14 @@ TEST_P(AckHandlersTest, TestAckMultipleSequentialBlocksLoss) {
     WriteStreamFrame frame(currentStreamId++, 0, 0, true);
     regularPacket.frames.emplace_back(std::move(frame));
     conn.outstandings.packets.emplace_back(OutstandingPacket(
-        std::move(regularPacket), sentTime, 1, false, packetNum, 0));
+        std::move(regularPacket),
+        sentTime,
+        1,
+        false,
+        packetNum,
+        0,
+        0,
+        LossState()));
   }
   ReadAckFrame ackFrame;
   ackFrame.largestAcked = 101;
@@ -287,7 +303,14 @@ TEST_P(AckHandlersTest, TestAckBlocksWithGaps) {
     WriteStreamFrame frame(currentStreamId++, 0, 0, true);
     regularPacket.frames.emplace_back(std::move(frame));
     conn.outstandings.packets.emplace_back(OutstandingPacket(
-        std::move(regularPacket), Clock::now(), 1, false, packetNum, 0));
+        std::move(regularPacket),
+        Clock::now(),
+        1,
+        false,
+        packetNum,
+        0,
+        0,
+        LossState()));
   }
 
   ReadAckFrame ackFrame;
@@ -385,7 +408,14 @@ TEST_P(AckHandlersTest, TestNonSequentialPacketNumbers) {
     WriteStreamFrame frame(current++, 0, 0, true);
     regularPacket.frames.emplace_back(std::move(frame));
     conn.outstandings.packets.emplace_back(OutstandingPacket(
-        std::move(regularPacket), Clock::now(), 1, false, packetNum, 0));
+        std::move(regularPacket),
+        Clock::now(),
+        1,
+        false,
+        packetNum,
+        0,
+        0,
+        LossState()));
   }
 
   for (PacketNum packetNum = 20; packetNum < 40; packetNum += 3) {
@@ -394,7 +424,14 @@ TEST_P(AckHandlersTest, TestNonSequentialPacketNumbers) {
     current += 3;
     regularPacket.frames.emplace_back(std::move(frame));
     conn.outstandings.packets.emplace_back(OutstandingPacket(
-        std::move(regularPacket), Clock::now(), 1, false, packetNum, 0));
+        std::move(regularPacket),
+        Clock::now(),
+        1,
+        false,
+        packetNum,
+        0,
+        0,
+        LossState()));
   }
 
   ReadAckFrame ackFrame;
@@ -474,8 +511,8 @@ TEST_P(AckHandlersTest, AckVisitorForAckTest) {
   conn.ackStates.appDataAckState.acks.insert(900, 1000);
   conn.ackStates.appDataAckState.acks.insert(500, 700);
   firstPacket.frames.emplace_back(std::move(firstAckFrame));
-  conn.outstandings.packets.emplace_back(
+  conn.outstandings.packets.emplace_back(OutstandingPacket(
-      OutstandingPacket(std::move(firstPacket), Clock::now(), 0, false, 0, 0));
+      std::move(firstPacket), Clock::now(), 0, false, 0, 0, 0, LossState()));
 
   auto secondPacket = createNewPacket(101 /* packetNum */, GetParam());
   WriteAckFrame secondAckFrame;
@@ -484,8 +521,8 @@ TEST_P(AckHandlersTest, AckVisitorForAckTest) {
   conn.ackStates.appDataAckState.acks.insert(1100, 2000);
   conn.ackStates.appDataAckState.acks.insert(1002, 1090);
   secondPacket.frames.emplace_back(std::move(secondAckFrame));
-  conn.outstandings.packets.emplace_back(
+  conn.outstandings.packets.emplace_back(OutstandingPacket(
-      OutstandingPacket(std::move(secondPacket), Clock::now(), 0, false, 0, 0));
+      std::move(secondPacket), Clock::now(), 0, false, 0, 0, 0, LossState()));
 
   ReadAckFrame firstReceivedAck;
   firstReceivedAck.largestAcked = 100;
@@ -546,7 +583,7 @@ TEST_P(AckHandlersTest, NoNewAckedPacket) {
   PacketNum packetAfterRtoNum = 10;
   auto packetAfterRto = createNewPacket(packetAfterRtoNum, GetParam());
   conn.outstandings.packets.emplace_back(OutstandingPacket(
-      std::move(packetAfterRto), Clock::now(), 0, false, 0, 0));
+      std::move(packetAfterRto), Clock::now(), 0, false, 0, 0, 0, LossState()));
 
   ReadAckFrame ackFrame;
   ackFrame.largestAcked = 5;
@@ -592,12 +629,12 @@ TEST_P(AckHandlersTest, AckPacketNumDoesNotExist) {
   PacketNum packetNum1 = 9;
   auto regularPacket1 = createNewPacket(packetNum1, GetParam());
   conn.outstandings.packets.emplace_back(
-      std::move(regularPacket1), Clock::now(), 0, false, 0, 0);
+      std::move(regularPacket1), Clock::now(), 0, false, 0, 0, 0, LossState());
 
   PacketNum packetNum2 = 10;
   auto regularPacket2 = createNewPacket(packetNum2, GetParam());
   conn.outstandings.packets.emplace_back(
-      std::move(regularPacket2), Clock::now(), 0, false, 0, 0);
+      std::move(regularPacket2), Clock::now(), 0, false, 0, 0, 0, LossState());
 
   // Ack a packet one higher than the packet so that we don't trigger reordering
   // threshold.
@@ -631,7 +668,9 @@ TEST_P(AckHandlersTest, TestHandshakeCounterUpdate) {
         0,
         packetNum % 2 && GetParam() != PacketNumberSpace::AppData,
         packetNum / 2,
-        0);
+        0,
+        0,
+        LossState());
     if (GetParam() == PacketNumberSpace::Initial) {
       conn.outstandings.initialPacketsCount += packetNum % 2;
     } else if (GetParam() == PacketNumberSpace::Handshake) {
@@ -714,7 +753,7 @@ TEST_P(AckHandlersTest, NoSkipAckVisitor) {
   WriteStreamFrame frame(0, 0, 0, true);
   regularPacket.frames.emplace_back(std::move(frame));
   conn.outstandings.packets.emplace_back(OutstandingPacket(
-      std::move(regularPacket), Clock::now(), 1, false, 1, 0));
+      std::move(regularPacket), Clock::now(), 1, false, 1, 0, 0, LossState()));
   ReadAckFrame ackFrame;
   ackFrame.largestAcked = 0;
   ackFrame.ackBlocks.emplace_back(0, 0);
@@ -756,7 +795,7 @@ TEST_P(AckHandlersTest, SkipAckVisitor) {
   WriteStreamFrame frame(0, 0, 0, true);
   regularPacket.frames.emplace_back(std::move(frame));
   OutstandingPacket outstandingPacket(
-      std::move(regularPacket), Clock::now(), 1, false, 1, 0);
+      std::move(regularPacket), Clock::now(), 1, false, 1, 0, 0, LossState());
   // Give this outstandingPacket an associatedEvent that's not in
   // outstandings.packetEvents
   outstandingPacket.associatedEvent.emplace(PacketNumberSpace::AppData, 0);
@@ -797,12 +836,12 @@ TEST_P(AckHandlersTest, NoDoubleProcess) {
   regularPacket2.frames.push_back(frame);
 
   OutstandingPacket outstandingPacket1(
-      std::move(regularPacket1), Clock::now(), 1, false, 1, 0);
+      std::move(regularPacket1), Clock::now(), 1, false, 1, 0, 0, LossState());
   outstandingPacket1.associatedEvent.emplace(
       PacketNumberSpace::AppData, packetNum1);
 
   OutstandingPacket outstandingPacket2(
-      std::move(regularPacket2), Clock::now(), 1, false, 1, 0);
+      std::move(regularPacket2), Clock::now(), 1, false, 1, 0, 0, LossState());
   // The seconds packet has the same PacketEvent
   outstandingPacket2.associatedEvent.emplace(
       PacketNumberSpace::AppData, packetNum1);
@@ -859,7 +898,7 @@ TEST_P(AckHandlersTest, ClonedPacketsCounter) {
   auto regularPacket1 = createNewPacket(packetNum1, GetParam());
   regularPacket1.frames.push_back(frame);
   OutstandingPacket outstandingPacket1(
-      std::move(regularPacket1), Clock::now(), 1, false, 1, 0);
+      std::move(regularPacket1), Clock::now(), 1, false, 1, 0, 0, LossState());
   outstandingPacket1.associatedEvent.emplace(
       PacketNumberSpace::AppData, packetNum1);
 
@@ -868,7 +907,7 @@ TEST_P(AckHandlersTest, ClonedPacketsCounter) {
   auto regularPacket2 = createNewPacket(packetNum2, GetParam());
   regularPacket2.frames.push_back(frame);
   OutstandingPacket outstandingPacket2(
-      std::move(regularPacket2), Clock::now(), 1, false, 1, 0);
+      std::move(regularPacket2), Clock::now(), 1, false, 1, 0, 0, LossState());
 
   conn.outstandings.packets.push_back(std::move(outstandingPacket1));
   conn.outstandings.packets.push_back(std::move(outstandingPacket2));
@@ -906,8 +945,8 @@ TEST_P(AckHandlersTest, UpdateMaxAckDelay) {
   PacketNum packetNum = 0;
   auto regularPacket = createNewPacket(packetNum, GetParam());
   auto sentTime = Clock::now();
-  conn.outstandings.packets.emplace_back(
+  conn.outstandings.packets.emplace_back(OutstandingPacket(
-      OutstandingPacket(std::move(regularPacket), sentTime, 1, false, 1, 0));
+      std::move(regularPacket), sentTime, 1, false, 1, 0, 0, LossState()));
 
   ReadAckFrame ackFrame;
   // ackDelay has no effect on mrtt
@@ -964,7 +1003,9 @@ TEST_P(AckHandlersTest, AckNotOutstandingButLoss) {
       1,
       false,
       1,
-      0);
+      0,
+      0,
+      LossState());
   conn.outstandings.packets.push_back(std::move(outstandingPacket));
   conn.outstandings.clonedPacketsCount++;
 
@@ -1008,7 +1049,9 @@ TEST_P(AckHandlersTest, UpdatePendingAckStates) {
       111,
       false,
       conn.lossState.totalBytesSent + 111,
-      0));
+      0,
+      0,
+      LossState()));
   conn.lossState.totalBytesSent += 111;
 
   ReadAckFrame ackFrame;
@@ -1052,7 +1095,9 @@ TEST_P(AckHandlersTest, AckEventCreation) {
         1,
         false /* handshake */,
         packetNum,
-        0);
+        0,
+        0,
+        LossState());
     sentPacket.isAppLimited = (packetNum % 2);
     conn.outstandings.packets.emplace_back(sentPacket);
     packetNum++;
@@ -1107,7 +1152,9 @@ TEST_P(AckHandlersTest, ImplictAckEventCreation) {
         1,
         false /* handshake */,
         packetNum,
-        packetNum + 1);
+        packetNum + 1,
+        0,
+        LossState());
     sentPacket.isAppLimited = (packetNum % 2);
     conn.outstandings.packets.emplace_back(sentPacket);
     packetNum++;
@@ -1172,7 +1219,9 @@ TEST_P(AckHandlersTest, TestRTTPacketObserverCallback) {
         1,
         false /* handshake */,
         packetNum,
-        packetNum + 1);
+        packetNum + 1,
+        0,
+        LossState());
     sentPacket.isAppLimited = false;
     conn.outstandings.packets.emplace_back(sentPacket);
     packetNum++;

quic/state/test/QuicStateFunctionsTest.cpp

@@ -554,25 +554,31 @@ TEST_F(QuicStateFunctionsTest, GetOutstandingPackets) {
       135,
       false,
       0,
-      0);
+      0,
+      0,
+      LossState());
   conn.outstandings.packets.emplace_back(
       makeTestLongPacket(LongHeader::Types::Handshake),
       Clock::now(),
       1217,
       false,
       0,
-      0);
+      0,
+      0,
+      LossState());
   conn.outstandings.packets.emplace_back(
-      makeTestShortPacket(), Clock::now(), 5556, false, 0, 0);
+      makeTestShortPacket(), Clock::now(), 5556, false, 0, 0, 0, LossState());
   conn.outstandings.packets.emplace_back(
       makeTestLongPacket(LongHeader::Types::Initial),
       Clock::now(),
       56,
       false,
       0,
-      0);
+      0,
+      0,
+      LossState());
   conn.outstandings.packets.emplace_back(
-      makeTestShortPacket(), Clock::now(), 6665, false, 0, 0);
+      makeTestShortPacket(), Clock::now(), 6665, false, 0, 0, 0, LossState());
   EXPECT_EQ(
       135,
       getFirstOutstandingPacket(conn, PacketNumberSpace::Initial)

quic/state/test/StateDataTest.cpp

@@ -11,6 +11,7 @@
 
 #include <quic/common/test/TestUtils.h>
 #include <quic/state/StateData.h>
+#include "quic/state/LossState.h"
 
 using namespace quic;
 using namespace testing;
@@ -36,7 +37,7 @@ TEST_F(StateDataTest, SingleLostPacketEvent) {
       100,
       kVersion));
   OutstandingPacket outstandingPacket(
-      packet, Clock::now(), 1234, false, 1234, 0);
+      packet, Clock::now(), 1234, false, 1234, 0, 0, LossState());
   CongestionController::LossEvent loss;
   loss.addLostPacket(outstandingPacket);
   EXPECT_EQ(1234, loss.lostBytes);
@@ -51,7 +52,7 @@ TEST_F(StateDataTest, MultipleLostPacketsEvent) {
       100,
       kVersion));
   OutstandingPacket outstandingPacket1(
-      packet1, Clock::now(), 1234, false, 1234, 0);
+      packet1, Clock::now(), 1234, false, 1234, 0, 0, LossState());
 
   RegularQuicWritePacket packet2(LongHeader(
       LongHeader::Types::Initial,
@@ -60,7 +61,7 @@ TEST_F(StateDataTest, MultipleLostPacketsEvent) {
       110,
       kVersion));
   OutstandingPacket outstandingPacket2(
-      packet2, Clock::now(), 1357, false, 1357, 0);
+      packet2, Clock::now(), 1357, false, 1357, 0, 0, LossState());
 
   CongestionController::LossEvent loss;
   loss.addLostPacket(outstandingPacket1);