Record stream details in packet metadata

Summary:
Introruced a new DetailsPerStream struct inside the outstanding packet metadata
so we can easily track all streams carrying app data within a packet and each
stream's bytes sent (both new and total).

With this, consumers can easily deduce the number of retransmitted bytes per
stream using the StreamDetails members:
newStreamBytesSent - streamBytesSent.

Reviewed By: bschlinker

Differential Revision: D28063916

fbshipit-source-id: d915514f85f24f4889cfac2f7a66bfd2d6b0a4af

quic/api/QuicTransportFunctions.cpp

@@ -602,6 +602,7 @@ void updateConnection(
   uint32_t ackFrameCounter = 0;
   uint32_t streamBytesSent = 0;
   uint32_t newStreamBytesSent = 0;
+  OutstandingPacket::Metadata::DetailsPerStream detailsPerStream;
   auto packetNumberSpace = packet.header.getPacketNumberSpace();
   bool isD6DProbe = packetNumberSpace == PacketNumberSpace::AppData &&
       conn.d6d.lastProbe.hasValue() &&
@@ -651,6 +652,7 @@ void updateConnection(
         conn.streamManager->updateWritableStreams(*stream);
         conn.streamManager->updateLossStreams(*stream);
         streamBytesSent += writeStreamFrame.len;
+        detailsPerStream.addFrame(writeStreamFrame, newStreamDataWritten);
         break;
       }
       case QuicWriteFrame::Type::WriteCryptoFrame: {
@@ -833,7 +835,8 @@ void updateConnection(
       conn.lossState.inflightBytes + encodedSize,
       conn.outstandings.numOutstanding() + 1,
       conn.lossState,
-      conn.writeCount);
+      conn.writeCount,
+      std::move(detailsPerStream));
 
   if (isD6DProbe) {
     ++conn.d6d.outstandingProbes;

quic/api/test/QuicPacketSchedulerTest.cpp

@@ -114,7 +114,7 @@ PacketNum addOutstandingPacket(QuicConnectionStateBase& conn) {
       nextPacketNum);
   RegularQuicWritePacket packet(std::move(header));
   conn.outstandings.packets.emplace_back(
-      packet, Clock::now(), 0, 0, false, 0, 0, 0, 0, LossState());
+      packet, Clock::now(), 0, 0, false, 0, 0, 0, 0, LossState(), 0);
   increaseNextPacketNum(conn, PacketNumberSpace::AppData);
   return nextPacketNum;
 }

quic/api/test/QuicTransportFunctionsTest.cpp

@@ -1647,6 +1647,276 @@ TEST_F(QuicTransportFunctionsTest, TestUpdateConnectionConnWindowUpdate) {
   EXPECT_EQ(frame->maximumData, conn->flowControlState.advertisedMaxOffset);
 }
 
+TEST_F(QuicTransportFunctionsTest, TestStreamDetailsEmptyPacket) {
+  auto conn = createConn();
+  auto packet = buildEmptyPacket(*conn, PacketNumberSpace::AppData);
+  updateConnection(
+      *conn,
+      folly::none,
+      packet.packet,
+      TimePoint(),
+      getEncodedSize(packet),
+      getEncodedBodySize(packet),
+      false /* isDSRPacket */);
+  // Since there is no ACK eliciting frame in this packet, it is not included as
+  // an outstanding packet
+  EXPECT_EQ(0, conn->outstandings.packets.size());
+}
+
+TEST_F(QuicTransportFunctionsTest, TestStreamDetailsControlPacket) {
+  auto conn = createConn();
+  auto packet = buildEmptyPacket(*conn, PacketNumberSpace::AppData);
+  auto stream = conn->streamManager->createNextBidirectionalStream().value();
+  StreamDataBlockedFrame blockedFrame(stream->id, 1000);
+  packet.packet.frames.push_back(blockedFrame);
+  packet.packet.frames.push_back(PingFrame());
+  updateConnection(
+      *conn,
+      folly::none,
+      packet.packet,
+      TimePoint(),
+      getEncodedSize(packet),
+      getEncodedBodySize(packet),
+      false /* isDSRPacket */);
+  // If we have only control frames sent, there should be no stream data in the
+  // outstanding packet.
+  ASSERT_EQ(1, conn->outstandings.packets.size());
+  auto detailsPerStream =
+      getFirstOutstandingPacket(*conn, PacketNumberSpace::AppData)
+          ->metadata.maybeDetailsPerStream;
+  EXPECT_EQ(true, detailsPerStream.has_value());
+  EXPECT_EQ(0, detailsPerStream->getDetails().size());
+}
+
+TEST_F(QuicTransportFunctionsTest, TestStreamDetailsAppDataPacketSingleStream) {
+  auto conn = createConn();
+  auto packet = buildEmptyPacket(*conn, PacketNumberSpace::AppData);
+  auto stream = conn->streamManager->createNextBidirectionalStream().value();
+  writeDataToQuicStream(*stream, folly::IOBuf::copyBuffer("abcdefghij"), true);
+  WriteStreamFrame writeStreamFrame(
+      stream->id, 0 /* offset */, 10 /* length */, false /* fin */);
+  packet.packet.frames.push_back(writeStreamFrame);
+  updateConnection(
+      *conn,
+      folly::none,
+      packet.packet,
+      TimePoint(),
+      getEncodedSize(packet),
+      getEncodedBodySize(packet),
+      false /* isDSRPacket */);
+
+  ASSERT_EQ(1, conn->outstandings.packets.size());
+  auto detailsPerStream =
+      getFirstOutstandingPacket(*conn, PacketNumberSpace::AppData)
+          ->metadata.maybeDetailsPerStream->getDetails();
+  EXPECT_EQ(1, detailsPerStream.size());
+  auto streamDetail = detailsPerStream[stream->id];
+  EXPECT_EQ(false, streamDetail.finObserved);
+  EXPECT_EQ(10, streamDetail.streamBytesSent);
+  EXPECT_EQ(10, streamDetail.newStreamBytesSent);
+  EXPECT_EQ(0, streamDetail.maybeFirstNewStreamByteOffset.value());
+}
+
+TEST_F(
+    QuicTransportFunctionsTest,
+    TestStreamDetailsAppDataPacketSingleStreamMultipleFrames) {
+  auto conn = createConn();
+  auto packet = buildEmptyPacket(*conn, PacketNumberSpace::AppData);
+  auto stream = conn->streamManager->createNextBidirectionalStream().value();
+  writeDataToQuicStream(
+      *stream, folly::IOBuf::copyBuffer("abcdefghijklmno"), true);
+  WriteStreamFrame writeStreamFrame1(
+      stream->id, 0 /* offset */, 10 /* length */, false /* fin */);
+  WriteStreamFrame writeStreamFrame2(
+      stream->id, 10 /* offset */, 5 /* length */, true /* fin */);
+  packet.packet.frames.push_back(writeStreamFrame1);
+  packet.packet.frames.push_back(writeStreamFrame2);
+  updateConnection(
+      *conn,
+      folly::none,
+      packet.packet,
+      TimePoint(),
+      getEncodedSize(packet),
+      getEncodedBodySize(packet),
+      false /* isDSRPacket */);
+
+  ASSERT_EQ(1, conn->outstandings.packets.size());
+  auto detailsPerStream =
+      getFirstOutstandingPacket(*conn, PacketNumberSpace::AppData)
+          ->metadata.maybeDetailsPerStream->getDetails();
+  EXPECT_EQ(1, detailsPerStream.size());
+  auto streamDetail = detailsPerStream[stream->id];
+  EXPECT_EQ(true, streamDetail.finObserved);
+  EXPECT_EQ(15, streamDetail.streamBytesSent);
+  EXPECT_EQ(15, streamDetail.newStreamBytesSent);
+  EXPECT_EQ(0, streamDetail.maybeFirstNewStreamByteOffset.value());
+}
+
+TEST_F(
+    QuicTransportFunctionsTest,
+    TestStreamDetailsAppDataPacketSingleStreamRetransmit) {
+  auto conn = createConn();
+  auto stream = conn->streamManager->createNextBidirectionalStream().value();
+
+  writeDataToQuicStream(*stream, folly::IOBuf::copyBuffer("abcdefghij"), true);
+  uint64_t frame1Offset = 0;
+  uint64_t frame1Len = 10;
+  auto packet = buildEmptyPacket(*conn, PacketNumberSpace::AppData);
+  WriteStreamFrame frame1(stream->id, frame1Offset, frame1Len, false /* fin */);
+  packet.packet.frames.push_back(frame1);
+  updateConnection(
+      *conn,
+      folly::none,
+      packet.packet,
+      TimePoint(),
+      getEncodedSize(packet),
+      getEncodedBodySize(packet),
+      false /* isDSRPacket */);
+
+  ASSERT_EQ(1, conn->outstandings.packets.size());
+
+  // The first outstanding packet is the one with new data
+  auto detailsPerStream =
+      getFirstOutstandingPacket(*conn, PacketNumberSpace::AppData)
+          ->metadata.maybeDetailsPerStream->getDetails();
+  EXPECT_EQ(1, detailsPerStream.size());
+  auto streamDetail = detailsPerStream[stream->id];
+  EXPECT_EQ(false, streamDetail.finObserved);
+  EXPECT_EQ(frame1Len, streamDetail.streamBytesSent);
+  EXPECT_EQ(frame1Len, streamDetail.newStreamBytesSent);
+  EXPECT_EQ(frame1Offset, streamDetail.maybeFirstNewStreamByteOffset.value());
+
+  // retransmit the same frame1 again.
+  packet = buildEmptyPacket(*conn, PacketNumberSpace::AppData);
+  packet.packet.frames.push_back(frame1);
+  updateConnection(
+      *conn,
+      folly::none,
+      packet.packet,
+      TimePoint(),
+      getEncodedSize(packet),
+      getEncodedBodySize(packet),
+      false /* isDSRPacket */);
+
+  ASSERT_EQ(2, conn->outstandings.packets.size());
+
+  // The second outstanding packet is the one with retransmit data
+  detailsPerStream = getLastOutstandingPacket(*conn, PacketNumberSpace::AppData)
+                         ->metadata.maybeDetailsPerStream->getDetails();
+  EXPECT_EQ(1, detailsPerStream.size());
+  streamDetail = detailsPerStream[stream->id];
+  EXPECT_EQ(false, streamDetail.finObserved);
+  EXPECT_EQ(frame1Len, streamDetail.streamBytesSent);
+  EXPECT_EQ(0, streamDetail.newStreamBytesSent);
+  EXPECT_FALSE(streamDetail.maybeFirstNewStreamByteOffset.has_value());
+
+  // Retransmit frame1 and send new data in frame2.
+  writeDataToQuicStream(
+      *stream, folly::IOBuf::copyBuffer("klmnopqrstuvwxy"), true);
+  uint64_t frame2Offset = 10;
+  uint64_t frame2Len = 15;
+  packet = buildEmptyPacket(*conn, PacketNumberSpace::AppData);
+  WriteStreamFrame frame2(stream->id, frame2Offset, frame2Len, false /* fin */);
+  packet.packet.frames.push_back(frame1);
+  packet.packet.frames.push_back(frame2);
+  updateConnection(
+      *conn,
+      folly::none,
+      packet.packet,
+      TimePoint(),
+      getEncodedSize(packet),
+      getEncodedBodySize(packet),
+      false /* isDSRPacket */);
+
+  ASSERT_EQ(3, conn->outstandings.packets.size());
+
+  // The third outstanding packet will have both new and retransmitted data.
+  detailsPerStream = getLastOutstandingPacket(*conn, PacketNumberSpace::AppData)
+                         ->metadata.maybeDetailsPerStream->getDetails();
+  EXPECT_EQ(1, detailsPerStream.size());
+  streamDetail = detailsPerStream[stream->id];
+  EXPECT_EQ(false, streamDetail.finObserved);
+  EXPECT_EQ(frame1Len + frame2Len, streamDetail.streamBytesSent);
+  EXPECT_EQ(frame2Len, streamDetail.newStreamBytesSent);
+  EXPECT_EQ(frame2Offset, streamDetail.maybeFirstNewStreamByteOffset.value());
+
+  // Retransmit frame1 aand frame2.
+  packet = buildEmptyPacket(*conn, PacketNumberSpace::AppData);
+  packet.packet.frames.push_back(frame1);
+  packet.packet.frames.push_back(frame2);
+  updateConnection(
+      *conn,
+      folly::none,
+      packet.packet,
+      TimePoint(),
+      getEncodedSize(packet),
+      getEncodedBodySize(packet),
+      false /* isDSRPacket */);
+
+  ASSERT_EQ(4, conn->outstandings.packets.size());
+
+  // The forth outstanding packet will have only retransmit data.
+  detailsPerStream = getLastOutstandingPacket(*conn, PacketNumberSpace::AppData)
+                         ->metadata.maybeDetailsPerStream->getDetails();
+  EXPECT_EQ(1, detailsPerStream.size());
+  streamDetail = detailsPerStream[stream->id];
+  EXPECT_EQ(false, streamDetail.finObserved);
+  EXPECT_EQ(frame1Len + frame2Len, streamDetail.streamBytesSent);
+  EXPECT_EQ(0, streamDetail.newStreamBytesSent);
+  EXPECT_FALSE(streamDetail.maybeFirstNewStreamByteOffset.has_value());
+}
+
+TEST_F(
+    QuicTransportFunctionsTest,
+    TestStreamDetailsAppDataPacketMultipleStreams) {
+  auto conn = createConn();
+  auto packet = buildEmptyPacket(*conn, PacketNumberSpace::AppData);
+  auto stream1Id =
+      conn->streamManager->createNextBidirectionalStream().value()->id;
+  auto stream2Id =
+      conn->streamManager->createNextBidirectionalStream().value()->id;
+  auto stream1 = conn->streamManager->findStream(stream1Id);
+  auto stream2 = conn->streamManager->findStream(stream2Id);
+  EXPECT_NE(nullptr, stream1);
+  EXPECT_NE(nullptr, stream2);
+
+  auto buf = IOBuf::copyBuffer("hey whats up");
+  writeDataToQuicStream(*stream1, buf->clone(), true);
+  writeDataToQuicStream(*stream2, buf->clone(), true);
+
+  WriteStreamFrame writeStreamFrame1(stream1->id, 0, 5, false),
+      writeStreamFrame2(stream2->id, 0, 12, true);
+  packet.packet.frames.push_back(writeStreamFrame1);
+  packet.packet.frames.push_back(writeStreamFrame2);
+
+  updateConnection(
+      *conn,
+      folly::none,
+      packet.packet,
+      TimePoint(),
+      getEncodedSize(packet),
+      getEncodedBodySize(packet),
+      false /* isDSRPacket */);
+
+  ASSERT_EQ(1, conn->outstandings.packets.size());
+  auto detailsPerStream =
+      getFirstOutstandingPacket(*conn, PacketNumberSpace::AppData)
+          ->metadata.maybeDetailsPerStream->getDetails();
+  EXPECT_EQ(2, detailsPerStream.size());
+  auto stream1Detail = detailsPerStream[stream1Id];
+  auto stream2Detail = detailsPerStream[stream2Id];
+
+  EXPECT_EQ(false, stream1Detail.finObserved);
+  EXPECT_EQ(5, stream1Detail.streamBytesSent);
+  EXPECT_EQ(5, stream1Detail.newStreamBytesSent);
+  EXPECT_EQ(0, stream1Detail.maybeFirstNewStreamByteOffset.value());
+
+  EXPECT_EQ(true, stream2Detail.finObserved);
+  EXPECT_EQ(12, stream2Detail.streamBytesSent);
+  EXPECT_EQ(12, stream2Detail.newStreamBytesSent);
+  EXPECT_EQ(0, stream2Detail.maybeFirstNewStreamByteOffset.value());
+}
+
 TEST_F(QuicTransportFunctionsTest, WriteQuicDataToSocketWithCC) {
   auto conn = createConn();
   conn->udpSendPacketLen = 30;
@@ -2101,8 +2371,8 @@ TEST_F(QuicTransportFunctionsTest, WriteProbingOldData) {
       1,
       writeProbingDataToSocketForTest(
           *rawSocket, *conn, 1, *aead, *headerCipher, getVersion(*conn)));
-  // Now we have no new data, let's probe again, and verify the same old data is
+  // Now we have no new data, let's probe again, and verify the same old data
-  // sent.
+  // is sent.
   folly::IOBuf secondBodyCaptured;
   EXPECT_CALL(*capturingAead, _inplaceEncrypt(_, _, _))
       .WillRepeatedly(Invoke([&](auto& buf, auto, auto) {
@@ -2458,7 +2728,8 @@ TEST_F(QuicTransportFunctionsTest, ShouldWriteDataTestDuringPathValidation) {
   auto buf = IOBuf::copyBuffer("0123456789");
   writeDataToQuicStream(*stream1, buf->clone(), false);
 
-  // Only case that we allow the write; both CC / PathLimiter have writablebytes
+  // Only case that we allow the write; both CC / PathLimiter have
+  // writablebytes
   EXPECT_CALL(*rawCongestionController, getWritableBytes()).WillOnce(Return(1));
   EXPECT_CALL(*rawLimiter, currentCredit(_, _)).WillOnce(Return(1));
 

quic/codec/test/QuicPacketRebuilderTest.cpp

@@ -38,7 +38,8 @@ OutstandingPacket makeDummyOutstandingPacket(
       0,
       0,
       0,
-      LossState());
+      LossState(),
+      0);
   return packet;
 }
 

quic/congestion_control/test/CopaTest.cpp

@@ -40,7 +40,8 @@ class CopaTest : public Test {
           0,
           0,
           0,
-          LossState()));
+          LossState(),
+          0));
       loss.lostBytes = packetData.second;
     }
     loss.lostPackets = lostPackets.size();
@@ -65,7 +66,8 @@ class CopaTest : public Test {
         0,
         inflight,
         0,
-        LossState());
+        LossState(),
+        0);
   }
 
   CongestionController::AckEvent createAckEvent(

quic/congestion_control/test/NewRenoTest.cpp

@@ -36,7 +36,8 @@ CongestionController::LossEvent createLossEvent(
         0,
         0,
         0,
-        LossState()));
+        LossState(),
+        0));
     loss.lostBytes = packetData.second;
   }
   loss.lostPackets = lostPackets.size();
@@ -64,7 +65,8 @@ CongestionController::AckEvent createAckEvent(
           0,
           0,
           0,
-          LossState())));
+          LossState(),
+          0)));
   return ack;
 }
 
@@ -86,7 +88,8 @@ OutstandingPacket createPacket(
       0,
       inflight,
       0,
-      LossState());
+      LossState(),
+      0);
 }
 
 TEST_F(NewRenoTest, TestLoss) {

quic/d6d/test/QuicD6DStateFunctionsTest.cpp

@@ -212,7 +212,8 @@ TEST_F(QuicD6DStateFunctionsTest, D6DProbeAckedInBase) {
       d6d.currentProbeSize,
       d6d.currentProbeSize,
       0,
-      LossState());
+      LossState(),
+      0);
   d6d.lastProbe = D6DProbePacket(
       pkt.packet.header.getPacketSequenceNum(), pkt.metadata.encodedSize);
   d6d.raiser = std::make_unique<MockProbeSizeRaiser>();
@@ -259,7 +260,8 @@ TEST_F(QuicD6DStateFunctionsTest, D6DProbeAckedInSearchingOne) {
       d6d.currentProbeSize,
       d6d.currentProbeSize,
       0,
-      LossState());
+      LossState(),
+      0);
   d6d.lastProbe = D6DProbePacket(
       pkt.packet.header.getPacketSequenceNum(), pkt.metadata.encodedSize);
   d6d.raiser = std::make_unique<MockProbeSizeRaiser>();
@@ -307,7 +309,8 @@ TEST_F(QuicD6DStateFunctionsTest, D6DProbeAckedInSearchingMax) {
       d6d.currentProbeSize,
       d6d.currentProbeSize,
       0,
-      LossState());
+      LossState(),
+      0);
   d6d.lastProbe = D6DProbePacket(
       pkt.packet.header.getPacketSequenceNum(), pkt.metadata.encodedSize);
   d6d.raiser = std::make_unique<MockProbeSizeRaiser>();
@@ -363,7 +366,8 @@ TEST_F(QuicD6DStateFunctionsTest, D6DProbeAckedInError) {
       d6d.currentProbeSize,
       d6d.currentProbeSize,
       0,
-      LossState());
+      LossState(),
+      0);
   d6d.lastProbe = D6DProbePacket(
       pkt.packet.header.getPacketSequenceNum(), pkt.metadata.encodedSize);
   d6d.raiser = std::make_unique<MockProbeSizeRaiser>();
@@ -409,7 +413,8 @@ TEST_F(QuicD6DStateFunctionsTest, BlackholeInSearching) {
       d6d.currentProbeSize,
       d6d.currentProbeSize,
       0,
-      LossState());
+      LossState(),
+      0);
   d6d.lastProbe = D6DProbePacket(
       pkt.packet.header.getPacketSequenceNum(), pkt.metadata.encodedSize);
 
@@ -423,7 +428,8 @@ TEST_F(QuicD6DStateFunctionsTest, BlackholeInSearching) {
       0,
       conn.udpSendPacketLen + d6d.currentProbeSize,
       0,
-      LossState());
+      LossState(),
+      0);
 
   d6d.thresholdCounter = std::make_unique<WindowedCounter<uint64_t, uint64_t>>(
       std::chrono::microseconds(kDefaultD6DBlackholeDetectionWindow).count(),
@@ -481,7 +487,8 @@ TEST_F(QuicD6DStateFunctionsTest, BlackholeInSearchComplete) {
       d6d.currentProbeSize,
       d6d.currentProbeSize,
       0,
-      LossState());
+      LossState(),
+      0);
   d6d.lastProbe = D6DProbePacket(
       pkt.packet.header.getPacketSequenceNum(), pkt.metadata.encodedSize);
 
@@ -495,7 +502,8 @@ TEST_F(QuicD6DStateFunctionsTest, BlackholeInSearchComplete) {
       0,
       conn.udpSendPacketLen + d6d.currentProbeSize,
       0,
-      LossState());
+      LossState(),
+      0);
 
   d6d.thresholdCounter = std::make_unique<WindowedCounter<uint64_t, uint64_t>>(
       std::chrono::microseconds(kDefaultD6DBlackholeDetectionWindow).count(),
@@ -556,7 +564,8 @@ TEST_F(QuicD6DStateFunctionsTest, ReachMaxPMTU) {
       d6d.currentProbeSize,
       d6d.currentProbeSize,
       0,
-      LossState());
+      LossState(),
+      0);
   d6d.lastProbe = D6DProbePacket(
       pkt.packet.header.getPacketSequenceNum(), pkt.metadata.encodedSize);
   d6d.raiser = std::make_unique<MockProbeSizeRaiser>();
@@ -596,7 +605,8 @@ TEST_F(
       d6d.currentProbeSize,
       d6d.currentProbeSize,
       0,
-      LossState());
+      LossState(),
+      0);
   d6d.lastProbe = D6DProbePacket(
       pkt.packet.header.getPacketSequenceNum(), pkt.metadata.encodedSize);
   d6d.raiser = std::make_unique<MockProbeSizeRaiser>();
@@ -618,7 +628,8 @@ TEST_F(
       d6d.currentProbeSize,
       d6d.currentProbeSize,
       0,
-      LossState());
+      LossState(),
+      0);
   // Generate a false positive blackhole signal
   detectPMTUBlackhole(conn, lostPacket);
   EXPECT_EQ(d6d.state, D6DMachineState::BASE);
@@ -657,7 +668,8 @@ TEST_F(QuicD6DStateFunctionsTest, UpperboundIsBase) {
       d6d.currentProbeSize,
       d6d.currentProbeSize,
       0,
-      LossState());
+      LossState(),
+      0);
   d6d.lastProbe = D6DProbePacket(
       pkt.packet.header.getPacketSequenceNum(), pkt.metadata.encodedSize);
   d6d.raiser = std::make_unique<MockProbeSizeRaiser>();

quic/loss/test/QuicLossFunctionsTest.cpp

@@ -255,7 +255,8 @@ PacketNum QuicLossFunctionsTest::sendPacket(
       encodedBodySize,
       0,
       0,
-      LossState());
+      LossState(),
+      0);
   outstandingPacket.associatedEvent = associatedEvent;
   conn.lossState.lastRetransmittablePacketSentTime = time;
   if (conn.congestionController) {
@@ -1007,7 +1008,7 @@ TEST_F(QuicLossFunctionsTest, TestHandleAckForLoss) {
   RegularQuicWritePacket outstandingRegularPacket(std::move(longHeader));
   auto now = Clock::now();
   conn->outstandings.packets.emplace_back(OutstandingPacket(
-      outstandingRegularPacket, now, 0, 0, false, 0, 0, 0, 0, LossState()));
+      outstandingRegularPacket, now, 0, 0, false, 0, 0, 0, 0, LossState(), 0));
   conn->outstandings.packetCount[PacketNumberSpace::Handshake]++;
 
   bool testLossMarkFuncCalled = false;

quic/state/OutstandingPacket.h

@@ -11,6 +11,7 @@
 #include <quic/codec/Types.h>
 #include <quic/state/LossState.h>
 #include <quic/state/PacketEvent.h>
+#include "folly/container/F14Map.h"
 
 namespace quic {
 
@@ -44,6 +45,50 @@ struct OutstandingPacketMetadata {
   // tracks the number of writes on this socket.
   uint64_t writeCount{0};
 
+  // Structure used to hold information about each stream with frames in packet
+  class DetailsPerStream {
+   public:
+    struct StreamDetails {
+      bool finObserved{false};
+      uint64_t streamBytesSent{0};
+      uint64_t newStreamBytesSent{0};
+      folly::Optional<uint64_t> maybeFirstNewStreamByteOffset;
+    };
+
+    void addFrame(const WriteStreamFrame& frame, const bool newData) {
+      auto ret = detailsPerStream.emplace(
+          std::piecewise_construct,
+          std::make_tuple(frame.streamId),
+          std::make_tuple());
+
+      auto& streamDetails = ret.first->second;
+      streamDetails.streamBytesSent += frame.len;
+      if (frame.fin) {
+        streamDetails.finObserved = true;
+      }
+      if (newData) {
+        streamDetails.newStreamBytesSent += frame.len;
+        if (streamDetails.maybeFirstNewStreamByteOffset) {
+          streamDetails.maybeFirstNewStreamByteOffset = std::min(
+              frame.offset, *streamDetails.maybeFirstNewStreamByteOffset);
+        } else {
+          streamDetails.maybeFirstNewStreamByteOffset = frame.offset;
+        }
+      }
+    }
+
+    FOLLY_NODISCARD const folly::F14FastMap<StreamId, StreamDetails>
+    getDetails() const {
+      return detailsPerStream;
+    }
+
+   private:
+    folly::F14FastMap<StreamId, StreamDetails> detailsPerStream;
+  };
+
+  // When enabled, details about each stream with frames in this packet
+  folly::Optional<DetailsPerStream> maybeDetailsPerStream;
+
   OutstandingPacketMetadata(
       TimePoint timeIn,
       uint32_t encodedSizeIn,
@@ -55,7 +100,8 @@ struct OutstandingPacketMetadata {
       uint64_t inflightBytesIn,
       uint64_t packetsInflightIn,
       const LossState& lossStateIn,
-      uint64_t writeCount)
+      uint64_t writeCount,
+      folly::Optional<DetailsPerStream> maybeDetailsPerStream)
       : time(timeIn),
         encodedSize(encodedSizeIn),
         encodedBodySize(encodedBodySizeIn),
@@ -67,11 +113,14 @@ struct OutstandingPacketMetadata {
         packetsInflight(packetsInflightIn),
         totalPacketsSent(lossStateIn.totalPacketsSent),
         totalAckElicitingPacketsSent(lossStateIn.totalAckElicitingPacketsSent),
-        writeCount(writeCount) {}
+        writeCount(writeCount),
+        maybeDetailsPerStream(std::move(maybeDetailsPerStream)) {}
 };
 
 // Data structure to represent outstanding retransmittable packets
 struct OutstandingPacket {
+  using Metadata = OutstandingPacketMetadata;
+
   // Structure representing the frames that are outstanding including the header
   // that was sent.
   RegularQuicWritePacket packet;
@@ -138,7 +187,9 @@ struct OutstandingPacket {
       uint64_t inflightBytesIn,
       uint64_t packetsInflightIn,
       const LossState& lossStateIn,
-      uint64_t writeCount = 0)
+      uint64_t writeCount,
+      folly::Optional<Metadata::DetailsPerStream> maybeDetailsPerStream =
+          folly::none)
       : packet(std::move(packetIn)),
         metadata(OutstandingPacketMetadata(
             timeIn,
@@ -151,7 +202,8 @@ struct OutstandingPacket {
             inflightBytesIn,
             packetsInflightIn,
             lossStateIn,
-            writeCount)) {}
+            writeCount,
+            std::move(maybeDetailsPerStream))) {}
 
   OutstandingPacket(
       RegularQuicWritePacket packetIn,
@@ -165,7 +217,9 @@ struct OutstandingPacket {
       uint64_t inflightBytesIn,
       uint64_t packetsInflightIn,
       const LossState& lossStateIn,
-      uint64_t writeCount = 0)
+      uint64_t writeCount,
+      folly::Optional<Metadata::DetailsPerStream> maybeDetailsPerStream =
+          folly::none)
       : packet(std::move(packetIn)),
         metadata(OutstandingPacketMetadata(
             timeIn,
@@ -178,6 +232,7 @@ struct OutstandingPacket {
             inflightBytesIn,
             packetsInflightIn,
             lossStateIn,
-            writeCount)) {}
+            writeCount,
+            std::move(maybeDetailsPerStream))) {}
 };
 } // namespace quic

quic/state/test/AckHandlersTest.cpp

@@ -63,7 +63,8 @@ auto emplacePackets(
         0,
         packetNum + 1,
         packetNum + 1,
-        quic::LossState());
+        quic::LossState(),
+        0);
     conn.outstandings.packets.emplace_back(sentPacket);
     packetNum++;
   }
@@ -98,7 +99,8 @@ TEST_P(AckHandlersTest, TestAckMultipleSequentialBlocks) {
         0,
         0,
         0,
-        LossState()));
+        LossState(),
+        0));
   }
   ReadAckFrame ackFrame;
   ackFrame.largestAcked = 101;
@@ -180,7 +182,8 @@ TEST_P(AckHandlersTest, TestAckMultipleSequentialBlocksLoss) {
         0,
         0,
         0,
-        LossState()));
+        LossState(),
+        0));
   }
   ReadAckFrame ackFrame;
   ackFrame.largestAcked = 101;
@@ -326,7 +329,8 @@ TEST_P(AckHandlersTest, TestAckBlocksWithGaps) {
         0,
         0,
         0,
-        LossState()));
+        LossState(),
+        0));
   }
 
   ReadAckFrame ackFrame;
@@ -435,7 +439,8 @@ TEST_P(AckHandlersTest, TestNonSequentialPacketNumbers) {
         0,
         0,
         0,
-        LossState()));
+        LossState(),
+        0));
   }
 
   for (PacketNum packetNum = 20; packetNum < 40; packetNum += 3) {
@@ -455,7 +460,8 @@ TEST_P(AckHandlersTest, TestNonSequentialPacketNumbers) {
         0,
         0,
         0,
-        LossState()));
+        LossState(),
+        0));
   }
 
   ReadAckFrame ackFrame;
@@ -546,7 +552,8 @@ TEST_P(AckHandlersTest, AckVisitorForAckTest) {
       0,
       0,
       0,
-      LossState()));
+      LossState(),
+      0));
 
   auto secondPacket = createNewPacket(101 /* packetNum */, GetParam());
   WriteAckFrame secondAckFrame;
@@ -566,7 +573,8 @@ TEST_P(AckHandlersTest, AckVisitorForAckTest) {
       0,
       0,
       0,
-      LossState()));
+      LossState(),
+      0));
 
   ReadAckFrame firstReceivedAck;
   firstReceivedAck.largestAcked = 100;
@@ -637,7 +645,8 @@ TEST_P(AckHandlersTest, NoNewAckedPacket) {
       0,
       0,
       0,
-      LossState()));
+      LossState(),
+      0));
 
   ReadAckFrame ackFrame;
   ackFrame.largestAcked = 5;
@@ -693,7 +702,8 @@ TEST_P(AckHandlersTest, AckPacketNumDoesNotExist) {
       0,
       0,
       0,
-      LossState());
+      LossState(),
+      0);
 
   PacketNum packetNum2 = 10;
   auto regularPacket2 = createNewPacket(packetNum2, GetParam());
@@ -708,7 +718,8 @@ TEST_P(AckHandlersTest, AckPacketNumDoesNotExist) {
       0,
       0,
       0,
-      LossState());
+      LossState(),
+      0);
 
   // Ack a packet one higher than the packet so that we don't trigger reordering
   // threshold.
@@ -748,7 +759,8 @@ TEST_P(AckHandlersTest, TestHandshakeCounterUpdate) {
         0,
         0,
         0,
-        LossState());
+        LossState(),
+        0);
   }
 
   ReadAckFrame ackFrame;
@@ -836,7 +848,8 @@ TEST_P(AckHandlersTest, NoSkipAckVisitor) {
       0,
       0,
       0,
-      LossState()));
+      LossState(),
+      0));
   ReadAckFrame ackFrame;
   ackFrame.largestAcked = 0;
   ackFrame.ackBlocks.emplace_back(0, 0);
@@ -887,7 +900,8 @@ TEST_P(AckHandlersTest, SkipAckVisitor) {
       0,
       0,
       0,
-      LossState());
+      LossState(),
+      0);
   // Give this outstandingPacket an associatedEvent that's not in
   // outstandings.packetEvents
   outstandingPacket.associatedEvent.emplace(GetParam(), 0);
@@ -937,7 +951,8 @@ TEST_P(AckHandlersTest, NoDoubleProcess) {
       0,
       0,
       0,
-      LossState());
+      LossState(),
+      0);
   outstandingPacket1.associatedEvent.emplace(GetParam(), packetNum1);
 
   OutstandingPacket outstandingPacket2(
@@ -950,7 +965,8 @@ TEST_P(AckHandlersTest, NoDoubleProcess) {
       0,
       0,
       0,
-      LossState());
+      LossState(),
+      0);
   // The seconds packet has the same PacketEvent
   outstandingPacket2.associatedEvent.emplace(GetParam(), packetNum1);
 
@@ -1015,7 +1031,8 @@ TEST_P(AckHandlersTest, ClonedPacketsCounter) {
       0,
       0,
       0,
-      LossState());
+      LossState(),
+      0);
   outstandingPacket1.associatedEvent.emplace(GetParam(), packetNum1);
 
   conn.ackStates.appDataAckState.nextPacketNum++;
@@ -1032,7 +1049,8 @@ TEST_P(AckHandlersTest, ClonedPacketsCounter) {
       0,
       0,
       0,
-      LossState());
+      LossState(),
+      0);
 
   conn.outstandings
       .packetCount[outstandingPacket1.packet.header.getPacketNumberSpace()]++;
@@ -1084,7 +1102,8 @@ TEST_P(AckHandlersTest, UpdateMaxAckDelay) {
       0,
       0,
       0,
-      LossState()));
+      LossState(),
+      0));
 
   ReadAckFrame ackFrame;
   // ackDelay has no effect on mrtt
@@ -1151,7 +1170,8 @@ TEST_P(AckHandlersTest, AckNotOutstandingButLoss) {
       0,
       0,
       0,
-      LossState());
+      LossState(),
+      0);
   conn.outstandings.packets.push_back(std::move(outstandingPacket));
   conn.outstandings.packetCount[GetParam()]++;
 
@@ -1202,7 +1222,8 @@ TEST_P(AckHandlersTest, UpdatePendingAckStates) {
       conn.lossState.totalBodyBytesSent + 100,
       0,
       0,
-      LossState()));
+      LossState(),
+      0));
   conn.lossState.totalBytesSent += 111;
   conn.lossState.totalBodyBytesSent += 100;
 
@@ -1254,7 +1275,8 @@ TEST_P(AckHandlersTest, AckEventCreation) {
         0,
         0,
         0,
-        LossState());
+        LossState(),
+        0);
     sentPacket.isAppLimited = (packetNum % 2);
     conn.outstandings.packets.emplace_back(sentPacket);
     packetNum++;
@@ -1315,7 +1337,8 @@ TEST_P(AckHandlersTest, ImplictAckEventCreation) {
         0,
         packetNum + 1,
         0,
-        LossState());
+        LossState(),
+        0);
     sentPacket.isAppLimited = (packetNum % 2);
     conn.outstandings.packets.emplace_back(sentPacket);
     packetNum++;
@@ -1386,7 +1409,8 @@ TEST_P(AckHandlersTest, TestRTTPacketObserverCallback) {
         0,
         packetNum + 1,
         0,
-        LossState());
+        LossState(),
+        0);
     sentPacket.isAppLimited = false;
     conn.outstandings.packets.emplace_back(sentPacket);
     packetNum++;

quic/state/test/QuicStateFunctionsTest.cpp

@@ -683,7 +683,8 @@ TEST_F(QuicStateFunctionsTest, GetOutstandingPackets) {
       0,
       0,
       0,
-      LossState());
+      LossState(),
+      0);
   conn.outstandings.packets.emplace_back(
       makeTestLongPacket(LongHeader::Types::Handshake),
       Clock::now(),
@@ -694,7 +695,8 @@ TEST_F(QuicStateFunctionsTest, GetOutstandingPackets) {
       0,
       0,
       0,
-      LossState());
+      LossState(),
+      0);
   conn.outstandings.packets.emplace_back(
       makeTestShortPacket(),
       Clock::now(),
@@ -705,7 +707,8 @@ TEST_F(QuicStateFunctionsTest, GetOutstandingPackets) {
       0,
       0,
       0,
-      LossState());
+      LossState(),
+      0);
   conn.outstandings.packets.emplace_back(
       makeTestLongPacket(LongHeader::Types::Initial),
       Clock::now(),
@@ -716,7 +719,8 @@ TEST_F(QuicStateFunctionsTest, GetOutstandingPackets) {
       0,
       0,
       0,
-      LossState());
+      LossState(),
+      0);
   conn.outstandings.packets.emplace_back(
       makeTestShortPacket(),
       Clock::now(),
@@ -727,7 +731,8 @@ TEST_F(QuicStateFunctionsTest, GetOutstandingPackets) {
       0,
       0,
       0,
-      LossState());
+      LossState(),
+      0);
   EXPECT_EQ(
       135,
       getFirstOutstandingPacket(conn, PacketNumberSpace::Initial)

quic/state/test/StateDataTest.cpp

@@ -37,7 +37,7 @@ TEST_F(StateDataTest, SingleLostPacketEvent) {
       100,
       kVersion));
   OutstandingPacket outstandingPacket(
-      packet, Clock::now(), 1234, 0, false, 1234, 0, 0, 0, LossState());
+      packet, Clock::now(), 1234, 0, false, 1234, 0, 0, 0, LossState(), 0);
   CongestionController::LossEvent loss;
   loss.addLostPacket(outstandingPacket);
   EXPECT_EQ(1234, loss.lostBytes);
@@ -52,7 +52,7 @@ TEST_F(StateDataTest, MultipleLostPacketsEvent) {
       100,
       kVersion));
   OutstandingPacket outstandingPacket1(
-      packet1, Clock::now(), 1234, 0, false, 1234, 0, 0, 0, LossState());
+      packet1, Clock::now(), 1234, 0, false, 1234, 0, 0, 0, LossState(), 0);
 
   RegularQuicWritePacket packet2(LongHeader(
       LongHeader::Types::Initial,
@@ -61,7 +61,7 @@ TEST_F(StateDataTest, MultipleLostPacketsEvent) {
       110,
       kVersion));
   OutstandingPacket outstandingPacket2(
-      packet2, Clock::now(), 1357, 0, false, 1357, 0, 0, 0, LossState());
+      packet2, Clock::now(), 1357, 0, false, 1357, 0, 0, 0, LossState(), 0);
 
   CongestionController::LossEvent loss;
   loss.addLostPacket(outstandingPacket1);