Change Implementation of WritableBytesLimit

Summary: - updating usage of WritableBytesLimit

Reviewed By: mjoras

Differential Revision: D33079816

fbshipit-source-id: 1854f40a7b00526afb2167764aeddf55edb1771f

quic/QuicConstants.h

@@ -287,6 +287,7 @@ enum class QuicVersion : uint32_t {
   MVFST_INVALID = 0xfaceb00f,
   MVFST_EXPERIMENTAL2 = 0xfaceb011, // Experimental alias for MVFST
   MVFST_ALIAS2 = 0xfaceb012,
+  MVFST_EXPERIMENTAL3 = 0xfaceb013, // Experimental alias for MVFST
 };
 
 using QuicVersionType = std::underlying_type<QuicVersion>::type;
@@ -395,10 +396,10 @@ constexpr uint64_t kDefaultMaxCwndInMss = 2000;
 // Max cwnd limit for perf test purpose
 constexpr uint64_t kLargeMaxCwndInMss = 860000;
 
-// When server receives early data attempt without valid source address token,
+// When server receives initial data without valid source address token,
 // server will limit bytes in flight to avoid amplification attack until CFIN
 // is received which proves sender owns the address.
-constexpr uint64_t kLimitedCwndInMss = 3;
+constexpr uint64_t kLimitedCwndInMss = 5;
 
 /* Hybrid slow start: */
 // The first kAckSampling Acks within a RTT round will be used to sample delays

quic/api/QuicTransportFunctions.cpp

@@ -915,6 +915,7 @@ uint64_t congestionControlWritableBytes(const QuicConnectionStateBase& conn) {
         conn.lossState.srtt == 0us ? kDefaultInitialRtt : conn.lossState.srtt);
   } else if (conn.writableBytesLimit) {
     if (*conn.writableBytesLimit <= conn.lossState.totalBytesSent) {
+      QUIC_STATS(conn.statsCallback, onConnectionWritableBytesLimited);
       return 0;
     }
     writableBytes = *conn.writableBytesLimit - conn.lossState.totalBytesSent;
@@ -1468,7 +1469,7 @@ uint64_t writeProbingDataToSocket(
                      builder,
                      pnSpace,
                      cloningScheduler,
-                     unlimitedWritableBytes,
+                     congestionControlWritableBytes,
                      probesToSend,
                      aead,
                      headerCipher,
@@ -1492,7 +1493,7 @@ uint64_t writeProbingDataToSocket(
                    builder,
                    pnSpace,
                    pingScheduler,
-                   unlimitedWritableBytes,
+                   congestionControlWritableBytes,
                    probesToSend - written,
                    aead,
                    headerCipher,

quic/api/test/QuicTransportFunctionsTest.cpp

@@ -2970,6 +2970,8 @@ TEST_F(QuicTransportFunctionsTest, WriteProbingNewData) {
   auto currentPacketSeqNum = conn->ackStates.appDataAckState.nextPacketNum;
   auto mockCongestionController =
       std::make_unique<NiceMock<MockCongestionController>>();
+  EXPECT_CALL(*mockCongestionController, getWritableBytes())
+      .WillRepeatedly(Return(2000));
   auto rawCongestionController = mockCongestionController.get();
   conn->congestionController = std::move(mockCongestionController);
   EventBase evb;
@@ -3059,6 +3061,8 @@ TEST_F(QuicTransportFunctionsTest, WriteProbingCryptoData) {
   // Replace real congestionController with MockCongestionController:
   auto mockCongestionController =
       std::make_unique<NiceMock<MockCongestionController>>();
+  EXPECT_CALL(*mockCongestionController, getWritableBytes())
+      .WillRepeatedly(Return(2000));
   auto rawCongestionController = mockCongestionController.get();
   conn.congestionController = std::move(mockCongestionController);
   EventBase evb;

quic/codec/Types.cpp

@@ -455,6 +455,8 @@ std::string toString(QuicVersion version) {
       return "MVFST_EXPERIMENTAL2";
     case QuicVersion::MVFST_ALIAS2:
       return "MVFST_ALIAS2";
+    case QuicVersion::MVFST_EXPERIMENTAL3:
+      return "MVFST_EXPERIMENTAL3";
   }
   LOG(WARNING) << "toString has unhandled version type";
   return "UNKNOWN";

quic/common/test/TestUtils.h

@@ -403,36 +403,50 @@ class FakeServerHandshake : public FizzServerHandshake {
 
   MOCK_METHOD(void, writeNewSessionTicket, (const AppToken&));
 
-  void doHandshake(std::unique_ptr<folly::IOBuf> data, EncryptionLevel)
+  void onClientHello(bool chloWithCert = false) {
-      override {
-    folly::IOBufEqualTo eq;
-    auto chlo = folly::IOBuf::copyBuffer("CHLO");
-    auto clientFinished = folly::IOBuf::copyBuffer("FINISHED");
-    if (eq(data, chlo)) {
-      if (chloSync_) {
     // Do NOT invoke onCryptoEventAvailable callback
     // Fall through and let the ServerStateMachine to process the event
     writeDataToQuicStream(
         *getCryptoStream(*conn_.cryptoState, EncryptionLevel::Initial),
         folly::IOBuf::copyBuffer("SHLO"));
+    if (chloWithCert) {
+      /* write 4000 bytes of data to the handshake crypto stream */
+      writeDataToQuicStream(
+          *getCryptoStream(*conn_.cryptoState, EncryptionLevel::Handshake),
+          folly::IOBuf::copyBuffer(std::string(4000, '.')));
+    }
+
     if (allowZeroRttKeys_) {
       validateAndUpdateSourceToken(conn_, sourceAddrs_);
       phase_ = Phase::KeysDerived;
       setEarlyKeys();
     }
     setHandshakeKeys();
+  }
+
+  void onClientFin() {
+    // Do NOT invoke onCryptoEventAvailable callback
+    // Fall through and let the ServerStateMachine to process the event
+    setOneRttKeys();
+    phase_ = Phase::Established;
+    handshakeDone_ = true;
+  }
+
+  void doHandshake(std::unique_ptr<folly::IOBuf> data, EncryptionLevel)
+      override {
+    folly::IOBufEqualTo eq;
+    auto chlo = folly::IOBuf::copyBuffer("CHLO");
+    auto chloWithCert = folly::IOBuf::copyBuffer("CHLO_CERT");
+    auto clientFinished = folly::IOBuf::copyBuffer("FINISHED");
+    bool sendHandshakeBytes = false;
+
+    if (eq(data, chlo) || (sendHandshakeBytes = eq(data, chloWithCert))) {
+      if (chloSync_) {
+        onClientHello(sendHandshakeBytes);
       } else {
         // Asynchronously schedule the callback
-        executor_->add([&] {
+        executor_->add([sendHandshakeBytes, this] {
-          writeDataToQuicStream(
+          onClientHello(sendHandshakeBytes);
-              *getCryptoStream(*conn_.cryptoState, EncryptionLevel::Initial),
-              folly::IOBuf::copyBuffer("SHLO"));
-          if (allowZeroRttKeys_) {
-            validateAndUpdateSourceToken(conn_, sourceAddrs_);
-            phase_ = Phase::KeysDerived;
-            setEarlyKeys();
-          }
-          setHandshakeKeys();
           if (callback_) {
             callback_->onCryptoEventAvailable();
           }
@@ -440,17 +454,11 @@ class FakeServerHandshake : public FizzServerHandshake {
       }
     } else if (eq(data, clientFinished)) {
       if (cfinSync_) {
-        // Do NOT invoke onCryptoEventAvailable callback
+        onClientFin();
-        // Fall through and let the ServerStateMachine to process the event
-        setOneRttKeys();
-        phase_ = Phase::Established;
-        handshakeDone_ = true;
       } else {
         // Asynchronously schedule the callback
         executor_->add([&] {
-          setOneRttKeys();
+          onClientFin();
-          phase_ = Phase::Established;
-          handshakeDone_ = true;
           if (callback_) {
             callback_->onCryptoEventAvailable();
           }

quic/server/QuicServer.h

@@ -399,6 +399,7 @@ class QuicServer : public QuicServerWorker::WorkerCallback,
       {QuicVersion::MVFST,
        QuicVersion::MVFST_EXPERIMENTAL,
        QuicVersion::MVFST_EXPERIMENTAL2,
+       QuicVersion::MVFST_EXPERIMENTAL3,
        QuicVersion::MVFST_ALIAS,
        QuicVersion::QUIC_V1,
        QuicVersion::QUIC_DRAFT,

quic/server/QuicServerTransport.cpp

@@ -676,6 +676,13 @@ void QuicServerTransport::onTransportKnobs(Buf knobBlob) {
   }
 }
 
+void QuicServerTransport::verifiedClientAddress() {
+  if (serverConn_) {
+    serverConn_->isClientAddrVerified = true;
+    conn_->writableBytesLimit = folly::none;
+  }
+}
+
 void QuicServerTransport::registerAllTransportKnobParamHandlers() {
   registerTransportKnobParamHandler(
       static_cast<uint64_t>(

quic/server/QuicServerTransport.h

@@ -125,6 +125,8 @@ class QuicServerTransport
 
   void setClientChosenDestConnectionId(const ConnectionId& serverCid);
 
+  void verifiedClientAddress();
+
   // From QuicTransportBase
   void onReadData(
       const folly::SocketAddress& peer,

quic/server/QuicServerWorker.cpp

@@ -696,6 +696,9 @@ void QuicServerWorker::dispatchPacketData(
           trans->setHandshakeFinishedCallback(this);
           trans->setSupportedVersions(supportedVersions_);
           trans->setOriginalPeerAddress(client);
+          if (isValidNewToken) {
+            trans->verifiedClientAddress();
+          }
 #ifdef CCP_ENABLED
           trans->setCcpDatapath(getCcpReader()->getDatapath());
 #endif

quic/server/state/ServerStateMachine.cpp

@@ -90,7 +90,8 @@ void recoverOrResetCongestionAndRttState(
   }
 }
 
-void setExperimentalSettings(QuicServerConnectionState& conn) {
+void maybeSetExperimentalSettings(QuicServerConnectionState& conn) {
+  if (conn.version == QuicVersion::MVFST_EXPERIMENTAL) {
     // MVFST_EXPERIMENTAL currently enables experimental congestion control
     // and experimental pacer. (here and in the client transport)
     if (conn.congestionController) {
@@ -99,6 +100,9 @@ void setExperimentalSettings(QuicServerConnectionState& conn) {
     if (conn.pacer) {
       conn.pacer->setExperimental(true);
     }
+  } else if (conn.version == QuicVersion::MVFST_EXPERIMENTAL3) {
+    conn.enableWritableBytesLimit = true;
+  }
 }
 } // namespace
 
@@ -386,6 +390,7 @@ void updateHandshakeState(QuicServerConnectionState& conn) {
       conn.qLogger->addTransportStateUpdate(kDerivedOneRttReadCipher);
     }
     // Clear limit because CFIN is received at this point
+    conn.isClientAddrVerified = true;
     conn.writableBytesLimit = folly::none;
     conn.readCodec->setOneRttReadCipher(std::move(oneRttReadCipher));
   }
@@ -478,6 +483,9 @@ void updateWritableByteLimitOnRecvPacket(QuicServerConnectionState& conn) {
   if (conn.writableBytesLimit) {
     conn.writableBytesLimit = *conn.writableBytesLimit +
         conn.transportSettings.limitedCwndInMss * conn.udpSendPacketLen;
+  } else if (!conn.isClientAddrVerified && conn.enableWritableBytesLimit) {
+    conn.writableBytesLimit =
+        conn.transportSettings.limitedCwndInMss * conn.udpSendPacketLen;
   }
 }
 
@@ -895,9 +903,7 @@ void onServerReadDataFromOpen(
             "Invalid packet type", TransportErrorCode::PROTOCOL_VIOLATION);
       }
       conn.version = longHeader->getVersion();
-      if (conn.version == QuicVersion::MVFST_EXPERIMENTAL) {
+      maybeSetExperimentalSettings(conn);
-        setExperimentalSettings(conn);
-      }
     }
 
     if (conn.peerAddress != readData.peer) {

quic/server/state/ServerStateMachine.h

@@ -134,6 +134,13 @@ struct QuicServerConnectionState : public QuicConnectionStateBase {
   // Number of bytes the server has written during the handshake.
   uint64_t numHandshakeBytesSent{0};
 
+  // Whether or not the client has verified their address (thru CFIN or
+  // NewToken).
+  bool isClientAddrVerified{false};
+
+  // Whether or not to enable WritableBytes limit
+  bool enableWritableBytesLimit{false};
+
 #ifdef CCP_ENABLED
   // Pointer to struct that maintains state needed for interacting with libccp.
   // Once instance of this struct is created for each instance of
@@ -158,6 +165,7 @@ struct QuicServerConnectionState : public QuicConnectionStateBase {
         {QuicVersion::MVFST,
          QuicVersion::MVFST_EXPERIMENTAL,
          QuicVersion::MVFST_EXPERIMENTAL2,
+         QuicVersion::MVFST_EXPERIMENTAL3,
          QuicVersion::MVFST_ALIAS,
          QuicVersion::QUIC_V1,
          QuicVersion::QUIC_DRAFT,

quic/server/test/QuicServerTransportTest.cpp

@@ -3261,6 +3261,212 @@ TEST_F(QuicUnencryptedServerTransportTest, TestNoAckOnlyCryptoInitial) {
   }
 }
 
+TEST_F(
+    QuicUnencryptedServerTransportTest,
+    TestHandshakeNotWritableBytesLimited) {
+  /**
+   * Set the WritableBytes limit to 5x (~ 5 * 1200 = 6,000). This will be enough
+   * for the handshake to fit (1200 initial + 3000 handshake = 4,200 < 6,000).
+   */
+  auto transportSettings = server->getTransportSettings();
+  transportSettings.limitedCwndInMss = 5;
+  server->setTransportSettings(transportSettings);
+  server->getNonConstConn().enableWritableBytesLimit = true;
+  EXPECT_CALL(*quicStats_, onConnectionWritableBytesLimited()).Times(0);
+
+  recvClientHello(true, QuicVersion::MVFST, "CHLO_CERT");
+
+  EXPECT_GE(serverWrites.size(), 3);
+
+  AckStates ackStates;
+
+  auto clientCodec = makeClientEncryptedCodec(true);
+  bool hasCryptoInitialFrame = false;
+  bool hasCryptoHandshakeFrame = false;
+  bool hasAckFrame = false;
+
+  /**
+   * Verify that we've written some cypto frames (initial, handshake packet
+   * spaces) and some acks.
+   */
+  for (auto& write : serverWrites) {
+    auto packetQueue = bufToQueue(write->clone());
+    auto result = clientCodec->parsePacket(packetQueue, ackStates);
+    auto& regularPacket = *result.regularPacket();
+    // EXPECT_TRUE(regularPacket);
+    ProtectionType protectionType = regularPacket.header.getProtectionType();
+    EXPECT_GE(regularPacket.frames.size(), 1);
+    bool hasCryptoFrame = false;
+    for (auto& frame : regularPacket.frames) {
+      hasCryptoFrame |= frame.asReadCryptoFrame() != nullptr;
+      hasAckFrame |= frame.asReadAckFrame() != nullptr;
+    }
+
+    hasCryptoInitialFrame |=
+        (protectionType == ProtectionType::Initial && hasCryptoFrame);
+    hasCryptoHandshakeFrame |=
+        (protectionType == ProtectionType::Handshake && hasCryptoFrame);
+  }
+
+  EXPECT_TRUE(hasCryptoInitialFrame);
+  EXPECT_TRUE(hasCryptoHandshakeFrame);
+  // skipping ack-only initial should not kick in here since we also have crypto
+  // data to write.
+  EXPECT_TRUE(hasAckFrame);
+}
+
+TEST_F(
+    QuicUnencryptedServerTransportTest,
+    TestHandshakeWritableBytesLimitedWithCFin) {
+  EXPECT_CALL(*quicStats_, onConnectionWritableBytesLimited())
+      .Times(AtLeast(1));
+  /**
+   * Set the WritableBytes limit to 3x (~ 3 * 1200 = 3,600). This will not be
+   * enough for the handshake to fit (1200 initial + 4000 handshake = 5,200 >
+   * 3,600). We expect to be WritableBytes limited. After receiving an ack/cfin
+   * from the client, the limit should increase and we're now unblocked.
+   */
+  auto transportSettings = server->getTransportSettings();
+  transportSettings.limitedCwndInMss = 3;
+  server->setTransportSettings(transportSettings);
+  server->getNonConstConn().enableWritableBytesLimit = true;
+
+  recvClientHello(true, QuicVersion::MVFST, "CHLO_CERT");
+
+  // basically the maximum we can write is three packets before we hit the limit
+  EXPECT_EQ(serverWrites.size(), 3);
+
+  AckStates ackStates;
+
+  auto clientCodec = makeClientEncryptedCodec(true);
+  bool hasCryptoInitialFrame, hasCryptoHandshakeFrame, hasAckFrame;
+  hasCryptoInitialFrame = hasCryptoHandshakeFrame = hasAckFrame = false;
+
+  for (auto& write : serverWrites) {
+    auto packetQueue = bufToQueue(write->clone());
+    auto result = clientCodec->parsePacket(packetQueue, ackStates);
+    auto& regularPacket = *result.regularPacket();
+    // EXPECT_TRUE(regularPacket);
+    ProtectionType protectionType = regularPacket.header.getProtectionType();
+    EXPECT_GE(regularPacket.frames.size(), 1);
+    bool hasCryptoFrame = false;
+    for (auto& frame : regularPacket.frames) {
+      hasCryptoFrame |= frame.asReadCryptoFrame() != nullptr;
+      hasAckFrame |= frame.asReadAckFrame() != nullptr;
+    }
+
+    hasCryptoInitialFrame |=
+        (protectionType == ProtectionType::Initial && hasCryptoFrame);
+    hasCryptoHandshakeFrame |=
+        (protectionType == ProtectionType::Handshake && hasCryptoFrame);
+  }
+
+  EXPECT_TRUE(hasCryptoInitialFrame);
+  EXPECT_TRUE(hasCryptoHandshakeFrame);
+  EXPECT_TRUE(hasAckFrame);
+  /**
+   * Let's now send an ack/cfin to the server which will unblock and let us
+   * finish the handshake. The packets written by the server at this point are
+   * expected to have crypto data and acks only in the handshake pn space, not
+   * initial.
+   */
+  EXPECT_CALL(*quicStats_, onConnectionWritableBytesLimited()).Times(0);
+  serverWrites.clear();
+  recvClientFinished();
+  EXPECT_TRUE(server->getConn().isClientAddrVerified);
+  EXPECT_FALSE(server->getConn().writableBytesLimit);
+  EXPECT_GT(serverWrites.size(), 0);
+
+  hasCryptoInitialFrame = hasCryptoHandshakeFrame = hasAckFrame = false;
+
+  for (auto& write : serverWrites) {
+    auto packetQueue = bufToQueue(write->clone());
+    auto result = clientCodec->parsePacket(packetQueue, ackStates);
+    auto& regularPacket = *result.regularPacket();
+    ProtectionType protectionType = regularPacket.header.getProtectionType();
+    EXPECT_GE(regularPacket.frames.size(), 1);
+    bool hasCryptoFrame = false;
+    for (auto& frame : regularPacket.frames) {
+      hasCryptoFrame |= frame.asReadCryptoFrame() != nullptr;
+      hasAckFrame |= frame.asReadAckFrame() != nullptr;
+    }
+
+    hasCryptoHandshakeFrame |=
+        (protectionType == ProtectionType::Handshake && hasCryptoFrame);
+  }
+
+  // We don't expect crypto frame in initial pnspace since we're done
+  EXPECT_FALSE(hasCryptoInitialFrame);
+  EXPECT_TRUE(hasCryptoHandshakeFrame);
+  EXPECT_TRUE(hasAckFrame);
+}
+
+TEST_F(
+    QuicUnencryptedServerTransportTest,
+    TestHandshakeWritableBytesLimitedPartialAck) {
+  /**
+   * Set the WritableBytes limit to 3x (~ 3 * 1200 = 3,600). This will not be
+   * enough for the handshake to fit (1200 initial + 4000 handshake = 5,200 >
+   * 3,600). We expect to be WritableBytes limited. After receiving an ack
+   * from the client acking only the initial crypto data, the pto should fire
+   * immediately to resend the handshake crypto data.
+   */
+  auto transportSettings = server->getTransportSettings();
+  transportSettings.limitedCwndInMss = 3;
+  server->setTransportSettings(transportSettings);
+  server->getNonConstConn().enableWritableBytesLimit = true;
+  EXPECT_CALL(*quicStats_, onConnectionWritableBytesLimited())
+      .Times(AtLeast(1));
+
+  recvClientHello(true, QuicVersion::MVFST, "CHLO_CERT");
+
+  // basically the maximum we can write is three packets before we hit the limit
+  EXPECT_EQ(serverWrites.size(), 3);
+
+  AckStates ackStates;
+
+  auto clientCodec = makeClientEncryptedCodec(true);
+
+  for (auto& write : serverWrites) {
+    auto packetQueue = bufToQueue(write->clone());
+    auto result = clientCodec->parsePacket(packetQueue, ackStates);
+    EXPECT_TRUE(result.regularPacket());
+  }
+
+  /**
+   * Let's now send an partial ack to the server, acking only the initial pn
+   * space, which will unblock and let us finish the handshake. Since we've
+   * already sent the handshake data, we expect a pto to fire immediately and
+   */
+
+  serverWrites.clear();
+  auto nextPacketNum = clientNextInitialPacketNum++;
+  auto aead = getInitialCipher();
+  auto headerCipher = getInitialHeaderCipher();
+  AckBlocks acks;
+  auto start = getFirstOutstandingPacket(
+                   server->getNonConstConn(), PacketNumberSpace::Initial)
+                   ->packet.header.getPacketSequenceNum();
+  auto end = getLastOutstandingPacket(
+                 server->getNonConstConn(), PacketNumberSpace::Initial)
+                 ->packet.header.getPacketSequenceNum();
+  acks.insert(start, end);
+  EXPECT_CALL(*quicStats_, onConnectionWritableBytesLimited()).Times(0);
+  deliverData(packetToBufCleartext(
+      createAckPacket(
+          server->getNonConstConn(),
+          nextPacketNum,
+          acks,
+          PacketNumberSpace::Initial,
+          aead.get()),
+      *aead,
+      *headerCipher,
+      nextPacketNum));
+
+  // The server is unblocked and should now be able to finish the handshake
+  EXPECT_GE(serverWrites.size(), 1);
+}
+
 TEST_F(QuicUnencryptedServerTransportTest, TestCorruptedDstCidInitialTest) {
   auto chlo = folly::IOBuf::copyBuffer("CHLO");
   auto nextPacketNum = clientNextInitialPacketNum++;
@@ -3387,6 +3593,7 @@ TEST_F(
 }
 
 TEST_F(QuicUnencryptedServerTransportTest, TestSendHandshakeDone) {
+  EXPECT_CALL(*quicStats_, onConnectionWritableBytesLimited()).Times(0);
   EXPECT_CALL(handshakeFinishedCallback, onHandshakeFinished());
   getFakeHandshakeLayer()->allowZeroRttKeys();
   setupClientReadCodec();
@@ -3430,6 +3637,7 @@ std::pair<int, std::vector<const NewTokenFrame*>> getNewTokenFrame(
 }
 
 TEST_F(QuicUnencryptedServerTransportTest, TestSendHandshakeDoneNewTokenFrame) {
+  EXPECT_CALL(*quicStats_, onConnectionWritableBytesLimited()).Times(0);
   std::array<uint8_t, kRetryTokenSecretLength> secret;
   folly::Random::secureRandom(secret.data(), secret.size());
   server->getNonConstConn().transportSettings.retryTokenSecret = secret;

quic/server/test/QuicServerTransportTestUtil.h

@@ -353,6 +353,9 @@ class QuicServerTransportTestBase : public virtual testing::Test {
   virtual void setupConnection() {
     EXPECT_EQ(server->getConn().readCodec, nullptr);
     EXPECT_EQ(server->getConn().statsCallback, quicStats_.get());
+    // None of these connections should cause the server to get WritableBytes
+    // limited.
+    EXPECT_CALL(*quicStats_, onConnectionWritableBytesLimited()).Times(0);
     // Not all connections are successful, in which case we don't call
     // onConnectionClose. The best we can test here is that onConnectionClose
     // doesn't get invoked more than once

quic/state/StateData.h

@@ -333,7 +333,7 @@ struct QuicConnectionStateBase : public folly::DelayedDestruction {
   // When server receives early data attempt without valid source address token,
   // server will limit bytes in flight to avoid amplification attack.
   // This limit should be cleared and set back to max after CFIN is received.
-  folly::Optional<uint32_t> writableBytesLimit;
+  folly::Optional<uint64_t> writableBytesLimit;
 
   std::unique_ptr<PendingPathRateLimiter> pathValidationLimiter;