Remove cmsgs from the outstanding packet.

Summary:
Remove cmsgs from each outstanding packet. Storing this as a map is excessive. Instead store it as a packed enum. This supports one kind of mark per packet.

If we ned to we could in the future support multiple types of marks per packet by using a bitset-style flag, e.g. each bit index representing a different mark. For now just use an enum.

Reviewed By: jbeshay

Differential Revision: D57397865

fbshipit-source-id: 6d34215c9d7e39537c44c6c304a8ce3a5883541e

quic/api/QuicTransportFunctions.cpp

@@ -884,7 +884,7 @@ void updateConnection(
       conn.appLimitedTracker.getTotalAppLimitedTime(),
       std::move(packetDestroyFn));
 
-  pkt.metadata.cmsgs = conn.socketCmsgsState.additionalCmsgs;
+  maybeAddPacketMark(conn, pkt);
 
   pkt.isAppLimited = conn.congestionController
       ? conn.congestionController->isAppLimited()
@@ -2032,4 +2032,34 @@ void maybeVerifyPendingKeyUpdate(
   }
 }
 
+// Unfortunate, we should make this more portable.
+#if !defined IPV6_HOPLIMIT
+#define IPV6_HOPLIMIT -1
+#endif
+#if !defined IP_TTL
+#define IP_TTL -1
+#endif
+// Add a packet mark to the outstanding packet. Currently only supports
+// TTLD marking.
+void maybeAddPacketMark(
+    QuicConnectionStateBase& conn,
+    OutstandingPacketWrapper& op) {
+  static constexpr folly::SocketOptionKey kHopLimitOptionKey = {
+      IPPROTO_IPV6, IPV6_HOPLIMIT};
+  static constexpr folly::SocketOptionKey kTTLOptionKey = {IPPROTO_IP, IP_TTL};
+  if (!conn.socketCmsgsState.additionalCmsgs.has_value()) {
+    return;
+  }
+  const auto& cmsgs = conn.socketCmsgsState.additionalCmsgs;
+  auto it = cmsgs->find(kHopLimitOptionKey);
+  if (it != cmsgs->end() && it->second == 255) {
+    op.metadata.mark = OutstandingPacketMark::TTLD;
+    return;
+  }
+  it = cmsgs->find(kTTLOptionKey);
+  if (it != cmsgs->end() && it->second == 255) {
+    op.metadata.mark = OutstandingPacketMark::TTLD;
+  }
+}
+
 } // namespace quic

quic/api/QuicTransportFunctions.h

@@ -340,5 +340,8 @@ void maybeVerifyPendingKeyUpdate(
     QuicConnectionStateBase& conn,
     const OutstandingPacketWrapper& outstandingPacket,
     const RegularQuicPacket& ackPacket);
+void maybeAddPacketMark(
+    QuicConnectionStateBase& conn,
+    OutstandingPacketWrapper& op);
 
 } // namespace quic

quic/api/test/QuicTypedTransportTest.cpp

@@ -965,14 +965,8 @@ TYPED_TEST(
   this->loopForWrites();
   ASSERT_FALSE(this->getConn().outstandings.packets.empty());
 
-  folly::SocketCmsgMap expectedCmsgs;
-  expectedCmsgs[{IPPROTO_IPV6, IPV6_HOPLIMIT}] = 255;
-  expectedCmsgs[{IPPROTO_IPV6, IPV6_DONTFRAG}] = 1;
   auto pkt = this->getConn().outstandings.packets.rbegin();
-  ASSERT_TRUE(pkt->metadata.cmsgs);
+  EXPECT_EQ(pkt->metadata.mark, OutstandingPacketMark::TTLD);
-  EXPECT_THAT(
-      pkt->metadata.cmsgs.value(), UnorderedElementsAreArray(expectedCmsgs));
-
   this->destroyTransport();
 }
 
@@ -1009,15 +1003,10 @@ TYPED_TEST(
     ASSERT_EQ(this->getConn().outstandings.packets.size(), packetsSent);
 
     // Verify both packets are marked
-    folly::SocketCmsgMap expectedCmsgs;
-    expectedCmsgs[{IPPROTO_IPV6, IPV6_HOPLIMIT}] = 255;
     auto pkt = this->getConn().outstandings.packets.rbegin();
-    ASSERT_TRUE(pkt->metadata.cmsgs);
+    EXPECT_EQ(pkt->metadata.mark, OutstandingPacketMark::TTLD);
-    EXPECT_THAT(
+    pkt++;
-        pkt->metadata.cmsgs.value(), UnorderedElementsAreArray(expectedCmsgs));
+    EXPECT_EQ(pkt->metadata.mark, OutstandingPacketMark::TTLD);
-    ASSERT_TRUE((++pkt)->metadata.cmsgs);
-    EXPECT_THAT(
-        pkt->metadata.cmsgs.value(), UnorderedElementsAreArray(expectedCmsgs));
   }
 
   {
@@ -1035,15 +1024,16 @@ TYPED_TEST(
 
     // Verify the last two packets have no marks
     auto pkt = this->getConn().outstandings.packets.rbegin();
-    EXPECT_FALSE(pkt->metadata.cmsgs);
+    EXPECT_EQ(pkt->metadata.mark, OutstandingPacketMark::NONE);
-    EXPECT_FALSE((++pkt)->metadata.cmsgs);
+    pkt++;
+    EXPECT_EQ(pkt->metadata.mark, OutstandingPacketMark::NONE);
   }
 
   {
     // Send two packets with the same marking
     EXPECT_CALL(*rawPacketProcessor, prewrite()).Times(1).WillOnce([]() {
       PacketProcessor::PrewriteRequest req;
-      req.cmsgs.assign({{{IPPROTO_IPV6, IPV6_DONTFRAG}, 1}});
+      req.cmsgs.assign({{{IPPROTO_IPV6, IPV6_HOPLIMIT}, 255}});
       return req;
     });
     auto streamId = this->getTransport()->createBidirectionalStream().value();
@@ -1055,15 +1045,10 @@ TYPED_TEST(
     ASSERT_EQ(this->getConn().outstandings.packets.size(), packetsSent);
 
     // Verify the last two packets are marked
-    folly::SocketCmsgMap expectedCmsgs;
-    expectedCmsgs[{IPPROTO_IPV6, IPV6_DONTFRAG}] = 1;
     auto pkt = this->getConn().outstandings.packets.rbegin();
-    ASSERT_TRUE(pkt->metadata.cmsgs);
+    EXPECT_EQ(pkt->metadata.mark, OutstandingPacketMark::TTLD);
-    EXPECT_THAT(
+    pkt++;
-        pkt->metadata.cmsgs.value(), UnorderedElementsAreArray(expectedCmsgs));
+    EXPECT_EQ(pkt->metadata.mark, OutstandingPacketMark::TTLD);
-    ASSERT_TRUE((++pkt)->metadata.cmsgs);
-    EXPECT_THAT(
-        pkt->metadata.cmsgs.value(), UnorderedElementsAreArray(expectedCmsgs));
   }
 
   this->destroyTransport();

quic/state/OutstandingPacket.h

@@ -15,6 +15,13 @@
 
 namespace quic {
 
+// Currently only used for TTLD marking, which is a mark indicating
+// retransmitted data.
+enum class OutstandingPacketMark : uint8_t {
+  NONE = 0, // No marking.
+  TTLD = 1, // Marked for TTL-D retransmission.
+};
+
 struct OutstandingPacketMetadata {
   // Time that the packet was sent.
   TimePoint time;
@@ -26,8 +33,6 @@ struct OutstandingPacketMetadata {
   // Write Count is the value of the monotonically increasing counter which
   // tracks the number of writes on this socket.
   uint64_t writeCount{0};
-  // Cmsgs added by the QuicSocket when this packet was written
-  folly::Optional<folly::SocketCmsgMap> cmsgs;
 
   // Has value if the packet is lost by timeout. The value is the loss timeout
   // dividend that was used to declare this packet.
@@ -107,7 +112,9 @@ struct OutstandingPacketMetadata {
   // Size of only the body within the packet sent on the wire.
   uint16_t encodedBodySize;
 
-  bool scheduledForDestruction{false};
+  bool scheduledForDestruction : 1;
+
+  OutstandingPacketMark mark : 7;
 
   OutstandingPacketMetadata(
       TimePoint timeIn,
@@ -127,7 +134,9 @@ struct OutstandingPacketMetadata {
         totalAppLimitedTimeUsecs(totalAppLimitedTimeUsecsIn),
         inflightBytes(inflightBytesIn),
         encodedSize(encodedSizeIn),
-        encodedBodySize(encodedBodySizeIn) {}
+        encodedBodySize(encodedBodySizeIn),
+        scheduledForDestruction(false),
+        mark(OutstandingPacketMark::NONE) {}
 };
 
 // Data structure to represent outstanding retransmittable packets