diff --git a/quic/api/QuicPacketScheduler.cpp b/quic/api/QuicPacketScheduler.cpp
index f17942e19..113e3dcd6 100644
--- a/quic/api/QuicPacketScheduler.cpp
+++ b/quic/api/QuicPacketScheduler.cpp
@@ -476,11 +476,13 @@ bool CryptoStreamScheduler::writeCryptoData(PacketBuilderInterface& builder) {
       cryptoDataWritten = true;
     }
   }
-  if (cryptoDataWritten && conn_.nodeType == QuicNodeType::Client) {
+  if (cryptoDataWritten) {
     const LongHeader* longHeader = builder.getPacketHeader().asLong();
     bool initialPacket =
         longHeader && longHeader->getHeaderType() == LongHeader::Types::Initial;
-    if (initialPacket) {
+    if (initialPacket &&
+        (conn_.nodeType == QuicNodeType::Client ||
+         longHeader->getPacketSequenceNum() == 0)) {
       // This is the initial packet, we need to fill er up.
       while (builder.remainingSpaceInPkt() > 0) {
         writeFrame(PaddingFrame(), builder);
diff --git a/quic/api/test/QuicPacketSchedulerTest.cpp b/quic/api/test/QuicPacketSchedulerTest.cpp
index 84825d4fb..50392acdd 100644
--- a/quic/api/test/QuicPacketSchedulerTest.cpp
+++ b/quic/api/test/QuicPacketSchedulerTest.cpp
@@ -140,7 +140,7 @@ TEST_F(QuicPacketSchedulerTest, CryptoPaddingInitialPacket) {
   EXPECT_GT(builder2.remainingSpaceInPkt(), 0);
 }
 
-TEST_F(QuicPacketSchedulerTest, CryptoServerInitialNotPadded) {
+TEST_F(QuicPacketSchedulerTest, CryptoServerInitialPadded) {
   QuicServerConnectionState conn;
   auto connId = getTestConnectionId();
   PacketNum nextPacketNum = getNextPacketNum(conn, PacketNumberSpace::Initial);
@@ -159,7 +159,22 @@ TEST_F(QuicPacketSchedulerTest, CryptoServerInitialNotPadded) {
   writeDataToQuicStream(
       conn.cryptoState->initialStream, folly::IOBuf::copyBuffer("shlo"));
   scheduler.writeCryptoData(builder1);
-  EXPECT_GT(builder1.remainingSpaceInPkt(), 0);
+  EXPECT_EQ(builder1.remainingSpaceInPkt(), 0);
+  nextPacketNum++;
+  LongHeader longHeader2(
+      LongHeader::Types::Initial,
+      getTestConnectionId(1),
+      connId,
+      nextPacketNum,
+      QuicVersion::MVFST);
+  RegularQuicPacketBuilder builder2(
+      conn.udpSendPacketLen,
+      std::move(longHeader2),
+      conn.ackStates.initialAckState.largestAckedByPeer);
+  writeDataToQuicStream(
+      conn.cryptoState->initialStream, folly::IOBuf::copyBuffer("shlo"));
+  scheduler.writeCryptoData(builder2);
+  EXPECT_GT(builder2.remainingSpaceInPkt(), 0);
 }
 
 TEST_F(QuicPacketSchedulerTest, CryptoPaddingRetransmissionClientInitial) {