diff --git a/quic/QuicConstants.cpp b/quic/QuicConstants.cpp index 7efa626fb..ef5e2d4e1 100644 --- a/quic/QuicConstants.cpp +++ b/quic/QuicConstants.cpp @@ -18,6 +18,8 @@ std::string_view congestionControlTypeToString(CongestionControlType type) { return kCongestionControlCubicStr; case CongestionControlType::BBR: return kCongestionControlBbrStr; + case CongestionControlType::BBR2: + return kCongestionControlBbr2Str; case CongestionControlType::BBRTesting: return kCongestionControlBbrTestingStr; case CongestionControlType::Copa: @@ -41,6 +43,8 @@ std::optional congestionControlStrToType( std::string_view str) { if (str == kCongestionControlCubicStr) { return quic::CongestionControlType::Cubic; + } else if (str == kCongestionControlBbr2Str) { + return quic::CongestionControlType::BBR2; } else if (str == kCongestionControlBbrStr) { return quic::CongestionControlType::BBR; } else if (str == kCongestionControlBbrTestingStr) { diff --git a/quic/QuicConstants.h b/quic/QuicConstants.h index 42efb3bda..fd4dc1ce7 100644 --- a/quic/QuicConstants.h +++ b/quic/QuicConstants.h @@ -282,6 +282,7 @@ enum class LocalErrorCode : uint32_t { CWND_OVERFLOW = 0x40000007, INFLIGHT_BYTES_OVERFLOW = 0x40000008, LOST_BYTES_OVERFLOW = 0x40000009, + CONGESTION_CONTROL_ERROR = 0x4000000A, // This is a retryable error. When encountering this error, // the user should retry the request. NEW_VERSION_NEGOTIATED = 0x4000000A, @@ -391,6 +392,7 @@ constexpr DurationRep kDefaultWriteLimitRttFraction = 25; // Congestion control: constexpr std::string_view kCongestionControlCubicStr = "cubic"; constexpr std::string_view kCongestionControlBbrStr = "bbr"; +constexpr std::string_view kCongestionControlBbr2Str = "bbr2"; constexpr std::string_view kCongestionControlBbrTestingStr = "bbr_testing"; constexpr std::string_view kCongestionControlCopaStr = "copa"; constexpr std::string_view kCongestionControlCopa2Str = "copa2"; @@ -405,6 +407,7 @@ enum class CongestionControlType : uint8_t { Copa, Copa2, BBR, + BBR2, BBRTesting, StaticCwnd, None, diff --git a/quic/api/QuicTransportBase.cpp b/quic/api/QuicTransportBase.cpp index ffc119b7e..d5266dddf 100644 --- a/quic/api/QuicTransportBase.cpp +++ b/quic/api/QuicTransportBase.cpp @@ -3246,12 +3246,24 @@ void QuicTransportBase::setTransportSettings( (conn_->transportSettings.defaultCongestionController == CongestionControlType::BBR || conn_->transportSettings.defaultCongestionController == - CongestionControlType::BBRTesting); + CongestionControlType::BBRTesting || + conn_->transportSettings.defaultCongestionController == + CongestionControlType::BBR2); auto minCwnd = usingBbr ? kMinCwndInMssForBbr : conn_->transportSettings.minCwndInMss; conn_->pacer = std::make_unique(*conn_, minCwnd); conn_->pacer->setExperimental(conn_->transportSettings.experimentalPacer); conn_->canBePaced = conn_->transportSettings.pacingEnabledFirstFlight; + if (conn_->transportSettings.defaultCongestionController == + CongestionControlType::BBR2) { + // We need to have the pacer rate be as accurate as possible for BBR2. + // The current BBR behavior is dependent on the existing pacing behavior + // so the override is only for BBR2. + // TODO: This should be removed once the pacer changes are adopted as + // the defaults or the pacer is fixed in another way. + conn_->pacer->setExperimental(true); + writeLooper_->setFireLoopEarly(true); + } } else { LOG(ERROR) << "Pacing cannot be enabled without a timer"; conn_->transportSettings.pacingEnabled = false; @@ -3355,7 +3367,8 @@ void QuicTransportBase::validateCongestionAndPacing( CongestionControlType& type) { // Fallback to Cubic if Pacing isn't enabled with BBR together if ((type == CongestionControlType::BBR || - type == CongestionControlType::BBRTesting) && + type == CongestionControlType::BBRTesting || + type == CongestionControlType::BBR2) && (!conn_->transportSettings.pacingEnabled || !writeLooper_->hasPacingTimer())) { LOG(ERROR) << "Unpaced BBR isn't supported"; diff --git a/quic/congestion_control/Bandwidth.h b/quic/congestion_control/Bandwidth.h index d707b5966..8b8370ee7 100644 --- a/quic/congestion_control/Bandwidth.h +++ b/quic/congestion_control/Bandwidth.h @@ -24,22 +24,28 @@ struct Bandwidth { uint64_t units{0}; std::chrono::microseconds interval{0us}; UnitType unitType{UnitType::BYTES}; + bool isAppLimited{false}; explicit Bandwidth() : units(0), interval(std::chrono::microseconds::zero()) {} explicit Bandwidth( uint64_t unitsDelievered, - std::chrono::microseconds deliveryInterval) - : units(unitsDelievered), interval(deliveryInterval) {} + std::chrono::microseconds deliveryInterval, + bool appLimited = false) + : units(unitsDelievered), + interval(deliveryInterval), + isAppLimited(appLimited) {} explicit Bandwidth( uint64_t unitsDelievered, std::chrono::microseconds deliveryInterval, - UnitType unitTypeIn) + UnitType unitTypeIn, + bool appLimited = false) : units(unitsDelievered), interval(deliveryInterval), - unitType(unitTypeIn) {} + unitType(unitTypeIn), + isAppLimited(appLimited) {} explicit operator bool() const noexcept { return units != 0 && interval != 0us; @@ -95,6 +101,9 @@ bool operator>(const Bandwidth& lhs, const Bandwidth& rhs); bool operator>=(const Bandwidth& lhs, const Bandwidth& rhs); bool operator==(const Bandwidth& lhs, const Bandwidth& rhs); +template ::value>> +Bandwidth operator*(T t, const Bandwidth& bandwidth) noexcept; + uint64_t operator*(std::chrono::microseconds delay, const Bandwidth& bandwidth); std::ostream& operator<<(std::ostream& os, const Bandwidth& bandwidth); } // namespace quic diff --git a/quic/congestion_control/Bbr2.cpp b/quic/congestion_control/Bbr2.cpp new file mode 100644 index 000000000..349c189e7 --- /dev/null +++ b/quic/congestion_control/Bbr2.cpp @@ -0,0 +1,927 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include +#include +#include +#include +#include +#include + +namespace quic { + +constexpr uint64_t kMaxBwFilterLen = 2; // Measured in number of ProbeBW cycles +constexpr std::chrono::microseconds kMinRttFilterLen = 10s; +constexpr std::chrono::microseconds kProbeRTTInterval = 5s; +constexpr std::chrono::microseconds kProbeRttDuration = 200ms; +constexpr uint64_t kMaxExtraAckedFilterLen = + 10; // Measured in packet-timed round trips + +constexpr float kStartupPacingGain = 2.77; // 4 * ln(2) +constexpr float kDrainPacingGain = 1 / kStartupPacingGain; +constexpr float kProbeBwDownPacingGain = 0.9; +constexpr float kProbeBwCruisePacingGain = 1.0; +constexpr float kProbeBwRefillPacingGain = 1.0; +constexpr float kProbeBwUpPacingGain = 1.25; +constexpr float kProbeRttPacingGain = 1.0; + +constexpr float kStartupCwndGain = 2.0; +constexpr float kProbeBwCwndGain = 2.0; +constexpr float kProbeRttCwndGain = 0.5; + +constexpr float kBeta = 0.7; + +constexpr float kLossThreshold = 0.02; +constexpr float kHeadroomFactor = 0.15; + +quic::Bandwidth kMinPacingRateForSendQuantum{1200 * 1000, 1s}; +constexpr uint8_t kPacingMarginPercent = 1; + +Bbr2CongestionController::Bbr2CongestionController( + QuicConnectionStateBase& conn) + : conn_(conn), + maxBwFilter_(kMaxBwFilterLen, Bandwidth(), 0), + probeRttMinTimestamp_(Clock::now()), + maxExtraAckedFilter_(kMaxExtraAckedFilterLen, 0, 0), + cwndBytes_( + conn_.udpSendPacketLen * conn_.transportSettings.initCwndInMss) { + resetCongestionSignals(); + resetLowerBounds(); + // Start with the default pacing settings. + enterStartup(); +} + +// Congestion Controller Interface + +void Bbr2CongestionController::onRemoveBytesFromInflight( + uint64_t bytesToRemove) { + subtractAndCheckUnderflow(conn_.lossState.inflightBytes, bytesToRemove); +} + +void Bbr2CongestionController::onPacketSent( + const OutstandingPacketWrapper& packet) { + // Handle restart from idle + if (conn_.lossState.inflightBytes == 0 && isAppLimited()) { + idleRestart_ = true; + extraAckedStartTimestamp_ = Clock::now(); + + if (isProbeBwState(state_)) { + setPacing(); + } else if (state_ == State::ProbeRTT) { + checkProbeRttDone(); + } + } + + addAndCheckOverflow( + conn_.lossState.inflightBytes, packet.metadata.encodedSize); + + // Maintain cwndLimited flag + if (conn_.lossState.inflightBytes >= cwndBytes_) { + cwndLimitedInRound_ |= true; + } +} + +void Bbr2CongestionController::onPacketAckOrLoss( + const AckEvent* FOLLY_NULLABLE ackEvent, + const LossEvent* FOLLY_NULLABLE lossEvent) { + if (conn_.qLogger) { + conn_.qLogger->addCongestionMetricUpdate( + conn_.lossState.inflightBytes, + getCongestionWindow(), + kCongestionPacketAck, + bbr2StateToString(state_)); + } + if (ackEvent) { + subtractAndCheckUnderflow( + conn_.lossState.inflightBytes, ackEvent->ackedBytes); + } + if (lossEvent) { + subtractAndCheckUnderflow( + conn_.lossState.inflightBytes, lossEvent->lostBytes); + } + SCOPE_EXIT { + VLOG(6) << fmt::format( + "State={} inflight={} cwnd={} (gain={})", + bbr2StateToString(state_), + conn_.lossState.inflightBytes, + getCongestionWindow(), + cwndGain_); + }; + + if (lossEvent && lossEvent->lostPackets > 0) { + // The pseudo code in BBRHandleLostPacket is included in + // updateProbeBwCyclePhase. No need to repeat it here. + + // We don't expose the loss type here, so always use fast recovery for + // non-persistent congestion + saveCwnd(); + inPacketConservation_ = true; + packetConservationStartTime_ = Clock::now(); + if (lossEvent->persistentCongestion) { + cwndBytes_ = kMinCwndInMssForBbr * conn_.udpSendPacketLen; + } else { + auto newlyAcked = ackEvent ? ackEvent->ackedBytes : 0; + cwndBytes_ = conn_.lossState.inflightBytes + + std::max(newlyAcked, conn_.udpSendPacketLen); + } + } + + if (ackEvent) { + if (appLimited_ && + appLimitedLastSendTime_ <= ackEvent->largestNewlyAckedPacketSentTime) { + appLimited_ = false; + } + + if (inPacketConservation_ && + packetConservationStartTime_ <= + ackEvent->largestNewlyAckedPacketSentTime) { + inPacketConservation_ = false; + restoreCwnd(); + } + + // UpdateModelAndState + updateLatestDeliverySignals(*ackEvent); + updateRound(*ackEvent); + + // Update cwndLimited state + if (roundStart_) { + cwndLimitedInRound_ = false; + } + + updateCongestionSignals(lossEvent); + updateAckAggregation(*ackEvent); + checkStartupDone(); + checkDrain(); + + auto inflightBytesAtLargestAckedPacket = + ackEvent->getLargestNewlyAckedPacket() + ? ackEvent->getLargestNewlyAckedPacket() + ->outstandingPacketMetadata.inflightBytes + : conn_.lossState.inflightBytes; + auto lostBytes = lossEvent ? lossEvent->lostBytes : 0; + updateProbeBwCyclePhase( + ackEvent->ackedBytes, inflightBytesAtLargestAckedPacket, lostBytes); + updateMinRtt(); + checkProbeRtt(ackEvent->ackedBytes); + advanceLatestDeliverySignals(*ackEvent); + boundBwForModel(); + + // UpdateControlParameters + setPacing(); + setSendQuantum(); + setCwnd(ackEvent->ackedBytes, lostBytes); + } +} + +uint64_t Bbr2CongestionController::getWritableBytes() const noexcept { + return getCongestionWindow() > conn_.lossState.inflightBytes + ? getCongestionWindow() - conn_.lossState.inflightBytes + : 0; +} + +uint64_t Bbr2CongestionController::getCongestionWindow() const noexcept { + return cwndBytes_; +} + +CongestionControlType Bbr2CongestionController::type() const noexcept { + return CongestionControlType::BBR2; +} + +bool Bbr2CongestionController::isInBackgroundMode() const { + return false; +} + +bool Bbr2CongestionController::isAppLimited() const { + return appLimited_; +} + +void Bbr2CongestionController::setAppLimited() noexcept { + appLimited_ = true; + appLimitedLastSendTime_ = Clock::now(); +} + +// Internals +void Bbr2CongestionController::resetCongestionSignals() { + lossBytesInRound_ = 0; + lossEventsInRound_ = 0; + bandwidthLatest_ = Bandwidth(); + inflightLatest_ = 0; +} + +void Bbr2CongestionController::resetLowerBounds() { + bandwidthLo_ = Bandwidth(std::numeric_limits::max(), 1us); + inflightLo_ = std::numeric_limits::max(); +} +void Bbr2CongestionController::enterStartup() { + state_ = State::Startup; + pacingGain_ = kStartupPacingGain; + cwndGain_ = kStartupCwndGain; +} + +void Bbr2CongestionController::setPacing() { + auto rate = bandwidth_ * pacingGain_ * (100 - kPacingMarginPercent) / 100; + VLOG(6) << fmt::format( + "Setting pacing to {}. From bandwdith_={} pacingGain_={} kPacingMarginPercent={} units={} interval={}", + rate.normalizedDescribe(), + bandwidth_.normalizedDescribe(), + pacingGain_, + kPacingMarginPercent, + rate.units, + rate.interval.count()); + conn_.pacer->refreshPacingRate(rate.units, rate.interval); +} + +void Bbr2CongestionController::setSendQuantum() { + auto rate = bandwidth_ * pacingGain_ * (100 - kPacingMarginPercent) / 100; + auto floor = 2 * conn_.udpSendPacketLen; + if (rate < kMinPacingRateForSendQuantum) { + floor = conn_.udpSendPacketLen; + } + auto rateIn1Ms = rate * 1ms; + sendQuantum_ = std::min(rateIn1Ms, decltype(rateIn1Ms)(64 * 1024)); + sendQuantum_ = std::max(sendQuantum_, floor); +} + +void Bbr2CongestionController::setCwnd( + uint64_t ackedBytes, + uint64_t lostBytes) { + // BBRUpdateMaxInflight() + inflightMax_ = addQuantizationBudget( + getBDPWithGain(cwndGain_) + maxExtraAckedFilter_.GetBest()); + // BBRModulateCwndForRecovery() + if (lostBytes > 0) { + cwndBytes_ = std::max( + cwndBytes_ - std::min(lostBytes, cwndBytes_), + kMinCwndInMssForBbr * conn_.udpSendPacketLen); + } + if (inPacketConservation_) { + cwndBytes_ = + std::max(cwndBytes_, conn_.lossState.inflightBytes + ackedBytes); + } + + if (!inPacketConservation_) { + if (filledPipe_) { + cwndBytes_ = std::min(cwndBytes_ + ackedBytes, inflightMax_); + } else if ( + cwndBytes_ < inflightMax_ || + conn_.lossState.totalBytesAcked < + conn_.transportSettings.initCwndInMss * conn_.udpSendPacketLen) { + cwndBytes_ += ackedBytes; + } + cwndBytes_ = + std::max(cwndBytes_, kMinCwndInMssForBbr * conn_.udpSendPacketLen); + } + + // BBRBoundCwndForProbeRTT() + if (state_ == State::ProbeRTT) { + cwndBytes_ = std::min(cwndBytes_, getProbeRTTCwnd()); + } + + // BBRBoundCwndForModel() + auto cap = std::numeric_limits::max(); + if (isProbeBwState(state_) && state_ != State::ProbeBw_Cruise) { + cap = inflightHi_; + } else if (state_ == State::ProbeRTT || state_ == State::ProbeBw_Cruise) { + cap = getTargetInflightWithHeadroom(); + } + cap = std::min(cap, inflightLo_); + cap = std::max(cap, kMinCwndInMssForBbr * conn_.udpSendPacketLen); + cwndBytes_ = std::min(cwndBytes_, cap); +} + +void Bbr2CongestionController::checkProbeRttDone() { + auto timeNow = Clock::now(); + if (probeRttDoneTimestamp_ && timeNow > *probeRttDoneTimestamp_) { + // Schedule the next ProbeRTT + probeRttMinTimestamp_ = timeNow; + restoreCwnd(); + exitProbeRtt(); + } +} + +void Bbr2CongestionController::restoreCwnd() { + cwndBytes_ = std::max(cwndBytes_, previousCwndBytes_); + VLOG(6) << "Restored cwnd: " << cwndBytes_; +} +void Bbr2CongestionController::exitProbeRtt() { + resetLowerBounds(); + cwndGain_ = kProbeBwCwndGain; + if (filledPipe_) { + startProbeBwDown(); + startProbeBwCruise(); + } else { + enterStartup(); + } +} + +void Bbr2CongestionController::updateLatestDeliverySignals( + const AckEvent& ackEvent) { + lossRoundStart_ = false; + + bandwidthLatest_ = + std::max(bandwidthLatest_, getBandwidthSampleFromAck(ackEvent)); + VLOG(6) << fmt::format( + "Bandwidth latest= {} AppLimited={}", + bandwidthLatest_.normalizedDescribe(), + bandwidthLatest_.isAppLimited); + inflightLatest_ = std::max(inflightLatest_, bandwidthLatest_.units); + + auto pkt = ackEvent.getLargestNewlyAckedPacket(); + if (pkt && + pkt->outstandingPacketMetadata.totalBytesSent > lossRoundEndBytesSent_) { + // Uses bytes sent instead of ACKed in the spec. This doesn't affect the + // round counting + lossPctInLastRound_ = static_cast(lossBytesInRound_) / + static_cast(conn_.lossState.totalBytesSent - + lossRoundEndBytesSent_); + lossEventsInLastRound_ = lossEventsInRound_; + lossRoundEndBytesSent_ = conn_.lossState.totalBytesSent; + lossRoundStart_ = true; + } +} + +void Bbr2CongestionController::updateCongestionSignals( + const LossEvent* FOLLY_NULLABLE lossEvent) { + // Update max bandwidth + if (bandwidthLatest_ > maxBwFilter_.GetBest() || + !bandwidthLatest_.isAppLimited) { + VLOG(6) << fmt::format( + "Updating bandwidth filter with sample: {}", + bandwidthLatest_.normalizedDescribe()); + maxBwFilter_.Update(bandwidthLatest_, cycleCount_); + } + + // Update loss signal + if (lossEvent && lossEvent->lostBytes > 0) { + lossBytesInRound_ += lossEvent->lostBytes; + lossEventsInRound_ += 1; + } + + if (!lossRoundStart_) { + return; // we're still within the same round + } + // AdaptLowerBoundsFromCongestion - once per round-trip + if (state_ == State::ProbeBw_Up) { + return; + } + if (lossBytesInRound_ > 0) { + // InitLowerBounds + if (!bandwidthLo_) { + bandwidthLo_ = maxBwFilter_.GetBest(); + } + if (!inflightLo_) { + inflightLo_ = cwndBytes_; + } + + // LossLowerBounds + bandwidthLo_ = std::max(bandwidthLatest_, bandwidthLo_ * kBeta); + inflightLo_ = + std::max(inflightLatest_, static_cast(inflightLo_ * kBeta)); + } + + lossBytesInRound_ = 0; + lossEventsInRound_ = 0; +} + +void Bbr2CongestionController::updateAckAggregation(const AckEvent& ackEvent) { + /* Find excess ACKed beyond expected amount over this interval */ + auto interval = + Clock::now() - extraAckedStartTimestamp_.value_or(TimePoint()); + auto expectedDelivered = bandwidth_ * + std::chrono::duration_cast(interval); + /* Reset interval if ACK rate is below expected rate: */ + if (extraAckedDelivered_ < expectedDelivered) { + extraAckedDelivered_ = 0; + extraAckedStartTimestamp_ = Clock::now(); + expectedDelivered = 0; + } + extraAckedDelivered_ += ackEvent.ackedBytes; + auto extra = extraAckedDelivered_ - expectedDelivered; + extra = std::min(extra, cwndBytes_); + maxExtraAckedFilter_.Update(extra, roundCount_); +} +void Bbr2CongestionController::checkStartupDone() { + checkStartupFullBandwidth(); + checkStartupHighLoss(); + + if (state_ == State::Startup && filledPipe_) { + enterDrain(); + } +} + +void Bbr2CongestionController::checkStartupFullBandwidth() { + if (filledPipe_ || !roundStart_ || isAppLimited()) { + return; /* no need to check for a full pipe now */ + } + if (maxBwFilter_.GetBest() >= + filledPipeBandwidth_ * 1.25) { /* still growing? */ + filledPipeBandwidth_ = + maxBwFilter_.GetBest(); /* record new baseline level */ + filledPipeCount_ = 0; + return; + } + filledPipeCount_++; /* another round w/o much growth */ + if (filledPipeCount_ >= 3) { + filledPipe_ = true; + } +} + +void Bbr2CongestionController::checkStartupHighLoss() { + /* + Our implementation differs from the spec a bit here. The conditions in the + spec are: + 1. The connection has been in fast recovery for at least one full round trip. + 2. The loss rate over the time scale of a single full round trip exceeds + BBRLossThresh (2%). + 3. There are at least BBRStartupFullLossCnt=3 noncontiguous sequence ranges + lost in that round trip. + + For 1,2 we use the loss pct from the last loss round which means we could exit + before a full RTT. For 3, we check we received three separate loss events + which servers a similar purpose to discontiguous ranges but it's not exactly + the same. + */ + if (filledPipe_ || !roundStart_ || isAppLimited()) { + // TODO: the appLimited condition means we could tolerate losses in startup + // if we haven't found the full bandwidth. This may need to be revisited. + + return; /* no need to check for a the loss exit condition now */ + } + if (lossPctInLastRound_ > kLossThreshold && lossEventsInLastRound_ >= 3) { + filledPipe_ = true; + } +} + +void Bbr2CongestionController::enterDrain() { + state_ = State::Drain; + pacingGain_ = kDrainPacingGain; // Slow down pacing + cwndGain_ = kStartupCwndGain; // maintain cwnd +} + +void Bbr2CongestionController::checkDrain() { + if (state_ == State::Drain) { + VLOG(6) << fmt::format( + "Current inflight {} target inflight {}", + conn_.lossState.inflightBytes, + getTargetInflightWithGain(1.0)); + } + if (state_ == State::Drain && + conn_.lossState.inflightBytes <= getTargetInflightWithGain(1.0)) { + enterProbeBW(); /* BBR estimates the queue was drained */ + } +} +void Bbr2CongestionController::updateProbeBwCyclePhase( + uint64_t ackedBytes, + uint64_t inflightBytesAtLargestAckedPacket, + uint64_t lostBytes) { + /* The core state machine logic for ProbeBW: */ + if (!filledPipe_) { + return; /* only handling steady-state behavior here */ + } + adaptUpperBounds(ackedBytes, inflightBytesAtLargestAckedPacket, lostBytes); + if (!isProbeBwState(state_)) { + return; /* only handling ProbeBW states here: */ + } + switch (state_) { + case State::ProbeBw_Down: + + if (checkTimeToProbeBW()) { + return; /* already decided state transition */ + } + if (checkTimeToCruise()) { + startProbeBwCruise(); + } + + break; + + case State::ProbeBw_Cruise: + if (checkTimeToProbeBW()) { + return; /* already decided state transition */ + } + break; + case State::ProbeBw_Refill: + /* After one round of REFILL, start UP */ + if (roundStart_) { + bwProbeSamples_ = 1; + startProbeBwUp(); + } + break; + case State::ProbeBw_Up: + + if (hasElapsedInPhase(minRtt_) && + inflightLatest_ > getTargetInflightWithGain(1.25)) { + startProbeBwDown(); + } + break; + default: + throw QuicInternalException( + "BBR2: Unexpected state in ProbeBW phase: " + + bbr2StateToString(state_), + LocalErrorCode::CONGESTION_CONTROL_ERROR); + } +} + +void Bbr2CongestionController::adaptUpperBounds( + uint64_t ackedBytes, + uint64_t inflightBytesAtLargestAckedPacket, + uint64_t lostBytes) { + /* Track ACK state and update BBR.max_bw window and + * BBR.inflight_hi and BBR.bw_hi. */ + if (ackPhase_ == AckPhase::ProbeStarting && roundStart_) { + /* starting to get bw probing samples */ + ackPhase_ = AckPhase::ProbeFeedback; + } + if (ackPhase_ == AckPhase::ProbeStopping && roundStart_) { + /* end of samples from bw probing phase */ + if (isProbeBwState(state_) && !isAppLimited()) { + cycleCount_++; + } + } + + if (!checkInflightTooHigh(inflightBytesAtLargestAckedPacket, lostBytes)) { + /* Loss rate is safe. Adjust upper bounds + upward. */ + if (inflightHi_ == std::numeric_limits::max() || + bandwidthHi_.units == std::numeric_limits::max()) { + return; /* no upper bounds to raise */ + } + + if (inflightBytesAtLargestAckedPacket > inflightHi_) { + inflightHi_ = inflightBytesAtLargestAckedPacket; + } + if (bandwidthLatest_ > bandwidthHi_) { + bandwidthHi_ = bandwidthLatest_; + } + if (state_ == State::ProbeBw_Up) { + probeInflightHiUpward(ackedBytes); + } + } +} + +bool Bbr2CongestionController::checkTimeToProbeBW() { + if (hasElapsedInPhase(bwProbeWait_) || isRenoCoexistenceProbeTime()) { + startProbeBwRefill(); + return true; + } else { + return false; + } +} + +bool Bbr2CongestionController::checkTimeToCruise() { + if (conn_.lossState.inflightBytes > getTargetInflightWithHeadroom()) { + return false; /* not enough headroom */ + } else if (conn_.lossState.inflightBytes <= getTargetInflightWithGain()) { + return true; /* inflight <= estimated BDP */ + } + // Neither conditions met. Do not cruise yet. + return false; +} + +bool Bbr2CongestionController::hasElapsedInPhase( + std::chrono::microseconds interval) { + return Clock::now() > probeBWCycleStart_ + interval; +} + +// Was the loss percent too hight for the last ack received? +bool Bbr2CongestionController::checkInflightTooHigh( + uint64_t inflightBytesAtLargestAckedPacket, + uint64_t lostBytes) { + if (isInflightTooHigh(inflightBytesAtLargestAckedPacket, lostBytes)) { + if (bwProbeSamples_) { + handleInFlightTooHigh(inflightBytesAtLargestAckedPacket); + } + return true; + } else { + return false; + } +} + +bool Bbr2CongestionController::isInflightTooHigh( + uint64_t inflightBytesAtLargestAckedPacket, + uint64_t lostBytes) { + return static_cast(lostBytes) > + static_cast(inflightBytesAtLargestAckedPacket) * kLossThreshold; +} + +void Bbr2CongestionController::handleInFlightTooHigh( + uint64_t inflightBytesAtLargestAckedPacket) { + bwProbeSamples_ = 0; + // TODO: Should this be the app limited state of the largest acknowledged + // packet? + if (!isAppLimited()) { + inflightHi_ = std::max( + inflightBytesAtLargestAckedPacket, + static_cast( + static_cast(getTargetInflightWithGain()) * kBeta)); + } + if (state_ == State::ProbeBw_Up) { + startProbeBwDown(); + } +} + +uint64_t Bbr2CongestionController::getTargetInflightWithHeadroom() const { + /* Return a volume of data that tries to leave free + * headroom in the bottleneck buffer or link for + * other flows, for fairness convergence and lower + * RTTs and loss */ + if (inflightHi_ == std::numeric_limits::max()) { + return inflightHi_; + } + auto headroom = static_cast( + std::max(1.0f, kHeadroomFactor * static_cast(inflightHi_))); + return std::max( + inflightHi_ - headroom, + quic::kMinCwndInMssForBbr * conn_.udpSendPacketLen); +} + +void Bbr2CongestionController::probeInflightHiUpward(uint64_t ackedBytes) { + if (!cwndLimitedInRound_ || cwndBytes_ < inflightHi_) { + return; /* not fully using inflight_hi, so don't grow it */ + } + probeUpAcks_ += ackedBytes; + if (probeUpAcks_ >= probeUpCount_) { + auto delta = probeUpAcks_ / probeUpCount_; + probeUpAcks_ -= delta * probeUpCount_; + inflightHi_ += delta; + } + if (roundStart_) { + raiseInflightHiSlope(); + } +} + +void Bbr2CongestionController::updateMinRtt() { + probeRttExpired_ = probeRttMinTimestamp_ + ? Clock::now() > (probeRttMinTimestamp_.value() + kProbeRTTInterval) + : true; + auto& lrtt = conn_.lossState.lrtt; + if (lrtt > 0us && (lrtt < probeRttMinValue_ || probeRttExpired_)) { + probeRttMinValue_ = lrtt; + probeRttMinTimestamp_ = Clock::now(); + } + + auto minRttExpired = minRttTimestamp_ + ? Clock::now() > (minRttTimestamp_.value() + kMinRttFilterLen) + : true; + if (probeRttMinValue_ < minRtt_ || minRttExpired) { + minRtt_ = probeRttMinValue_; + minRttTimestamp_ = probeRttMinTimestamp_; + } +} + +void Bbr2CongestionController::checkProbeRtt(uint64_t ackedBytes) { + if (state_ != State::ProbeRTT && probeRttExpired_ && !idleRestart_) { + enterProbeRtt(); + saveCwnd(); + probeRttDoneTimestamp_.reset(); + ackPhase_ = AckPhase::ProbeStopping; + startRound(); + } + if (state_ == State::ProbeRTT) { + handleProbeRtt(); + } + if (ackedBytes > 0) { + idleRestart_ = false; + } +} + +void Bbr2CongestionController::enterProbeRtt() { + state_ = State::ProbeRTT; + pacingGain_ = kProbeRttPacingGain; + cwndGain_ = kProbeRttCwndGain; +} + +void Bbr2CongestionController::handleProbeRtt() { + /* Ignore low rate samples during ProbeRTT: */ + // TODO: I don't understand the logic in the spec in + // MarkConnectionAppLimited() but just setting app limited is reasonable + setAppLimited(); + + if (!probeRttDoneTimestamp_ && + conn_.lossState.inflightBytes <= getProbeRTTCwnd()) { + /* Wait for at least ProbeRTTDuration to elapse: */ + probeRttDoneTimestamp_ = Clock::now() + kProbeRttDuration; + /* Wait for at least one round to elapse: */ + // Is this needed? BBR.probe_rtt_round_done = false + startRound(); + } else if (probeRttDoneTimestamp_) { + if (roundStart_) { + checkProbeRttDone(); + } + } +} + +void Bbr2CongestionController::advanceLatestDeliverySignals( + const AckEvent& ackEvent) { + if (lossRoundStart_) { + bandwidthLatest_ = getBandwidthSampleFromAck(ackEvent); + inflightLatest_ = bandwidthLatest_.units; + } +} + +uint64_t Bbr2CongestionController::getProbeRTTCwnd() { + return std::max( + getBDPWithGain(kProbeRttCwndGain), + quic::kMinCwndInMssForBbr * conn_.udpSendPacketLen); +} +void Bbr2CongestionController::boundCwndForProbeRTT() { + if (state_ == State::ProbeRTT) { + cwndBytes_ = std::min(cwndBytes_, getProbeRTTCwnd()); + } +} + +void Bbr2CongestionController::boundBwForModel() { + if (state_ == State::Startup) { + bandwidth_ = maxBwFilter_.GetBest(); + } else { + bandwidth_ = + std::min(std::min(maxBwFilter_.GetBest(), bandwidthLo_), bandwidthHi_); + } + if (conn_.qLogger) { + conn_.qLogger->addBandwidthEstUpdate(bandwidth_.units, bandwidth_.interval); + } +} + +uint64_t Bbr2CongestionController::addQuantizationBudget(uint64_t input) const { + // BBRUpdateOffloadBudget() + auto offloadBudget = 3 * sendQuantum_; + input = std::max(input, offloadBudget); + input = std::max(input, quic::kMinCwndInMssForBbr * conn_.udpSendPacketLen); + if (state_ == State::ProbeBw_Up) { + // This number is arbitrary from the spec. It's probably to guarantee that + // probing up is more aggressive (?) + input += 2 * conn_.udpSendPacketLen; + } + return input; +} + +void Bbr2CongestionController::saveCwnd() { + if (!inLossRecovery_ && state_ != State::ProbeRTT) { + previousCwndBytes_ = cwndBytes_; + } else { + previousCwndBytes_ = std::max(cwndBytes_, previousCwndBytes_); + } + VLOG(6) << "Saved cwnd: " << previousCwndBytes_; +} + +uint64_t Bbr2CongestionController::getTargetInflightWithGain(float gain) const { + return addQuantizationBudget(getBDPWithGain(gain)); +} + +uint64_t Bbr2CongestionController::getBDPWithGain(float gain) const { + if (minRtt_ == kDefaultMinRtt) { + return uint64_t( + gain * conn_.transportSettings.initCwndInMss * conn_.udpSendPacketLen); + } else { + return uint64_t(gain * (minRtt_ * bandwidth_)); + } +} + +void Bbr2CongestionController::enterProbeBW() { + cwndGain_ = kProbeBwCwndGain; + startProbeBwDown(); +} + +void Bbr2CongestionController::startRound() { + nextRoundDelivered_ = conn_.lossState.totalBytesAcked; +} +void Bbr2CongestionController::updateRound(const AckEvent& ackEvent) { + auto pkt = ackEvent.getLargestNewlyAckedPacket(); + if (pkt && pkt->lastAckedPacketInfo && + pkt->lastAckedPacketInfo->totalBytesAcked >= nextRoundDelivered_) { + startRound(); + roundCount_++; + roundsSinceBwProbe_++; + roundStart_ = true; + } else { + roundStart_ = false; + } +} + +void Bbr2CongestionController::startProbeBwDown() { + resetCongestionSignals(); + probeUpCount_ = + std::numeric_limits::max(); /* not growing inflight_hi */ + /* Decide random round-trip bound for wait: */ + roundsSinceBwProbe_ = folly::Random::rand32() % 2; + /* Decide the random wall clock bound for wait: between 2-3 seconds */ + bwProbeWait_ = std::chrono::milliseconds(2 + folly::Random::rand32() % 1000); + + probeBWCycleStart_ = Clock::now(); + ackPhase_ = AckPhase::ProbeStopping; + state_ = State::ProbeBw_Down; + pacingGain_ = kProbeBwDownPacingGain; + startRound(); +} +void Bbr2CongestionController::startProbeBwCruise() { + state_ = State::ProbeBw_Cruise; + pacingGain_ = kProbeBwCruisePacingGain; +} + +void Bbr2CongestionController::startProbeBwRefill() { + resetLowerBounds(); + probeUpRounds_ = 0; + probeUpAcks_ = 0; + ackPhase_ = AckPhase::ProbeRefilling; + state_ = State::ProbeBw_Refill; + pacingGain_ = kProbeBwRefillPacingGain; + startRound(); +} +void Bbr2CongestionController::startProbeBwUp() { + ackPhase_ = AckPhase::ProbeStarting; + probeBWCycleStart_ = Clock::now(); + state_ = State::ProbeBw_Up; + pacingGain_ = kProbeBwUpPacingGain; + startRound(); + raiseInflightHiSlope(); +} + +void Bbr2CongestionController::raiseInflightHiSlope() { + auto growthThisRound = conn_.udpSendPacketLen << probeUpRounds_; + probeUpRounds_ = std::min(probeUpRounds_ + 1, decltype(probeUpRounds_)(30)); + probeUpCount_ = + std::max(cwndBytes_ / growthThisRound, decltype(cwndBytes_)(1)); +} + +// Utilities +bool Bbr2CongestionController::isProbeBwState( + const Bbr2CongestionController::State state) { + return ( + state == Bbr2CongestionController::State::ProbeBw_Down || + state == Bbr2CongestionController::State::ProbeBw_Cruise || + state == Bbr2CongestionController::State::ProbeBw_Refill || + state == Bbr2CongestionController::State::ProbeBw_Up); +} + +Bandwidth Bbr2CongestionController::getBandwidthSampleFromAck( + const AckEvent& ackEvent) { + auto ackTime = ackEvent.adjustedAckTime; + auto pkt = ackEvent.getLargestNewlyAckedPacket(); + if (!pkt) { + return Bandwidth(); + } + auto& lastAckedPacket = pkt->lastAckedPacketInfo; + auto lastSentTime = + lastAckedPacket ? lastAckedPacket->sentTime : conn_.connectionTime; + + auto sendElapsed = pkt->outstandingPacketMetadata.time - lastSentTime; + + auto lastAckTime = + lastAckedPacket ? lastAckedPacket->adjustedAckTime : conn_.connectionTime; + auto ackElapsed = ackTime - lastAckTime; + auto interval = std::max(ackElapsed, sendElapsed); + if (interval == 0us) { + return Bandwidth(); + } + auto lastBytesDelivered = + lastAckedPacket ? lastAckedPacket->totalBytesAcked : 0; + auto bytesDelivered = ackEvent.totalBytesAcked - lastBytesDelivered; + Bandwidth bwSample( + bytesDelivered, + std::chrono::duration_cast(interval), + pkt->isAppLimited || lastSentTime < appLimitedLastSendTime_); + return bwSample; +} + +bool Bbr2CongestionController::isRenoCoexistenceProbeTime() { + auto renoBdpInPackets = std::min(getTargetInflightWithGain(), cwndBytes_) / + conn_.udpSendPacketLen; + auto roundsBeforeRenoProbe = + std::min(renoBdpInPackets, decltype(renoBdpInPackets)(63)); + return roundsSinceBwProbe_ >= roundsBeforeRenoProbe; +} + +Bbr2CongestionController::State Bbr2CongestionController::getState() + const noexcept { + return state_; +} + +void Bbr2CongestionController::getStats( + CongestionControllerStats& stats) const { + stats.bbr2Stats.state = uint8_t(state_); +} + +std::string bbr2StateToString(Bbr2CongestionController::State state) { + switch (state) { + case Bbr2CongestionController::State::Startup: + return "Startup"; + case Bbr2CongestionController::State::Drain: + return "Drain"; + case Bbr2CongestionController::State::ProbeBw_Down: + return "ProbeBw_Down"; + case Bbr2CongestionController::State::ProbeBw_Cruise: + return "ProbeBw_Cruise"; + case Bbr2CongestionController::State::ProbeBw_Refill: + return "ProbeBw_Refill"; + case Bbr2CongestionController::State::ProbeBw_Up: + return "ProbeBw_Up"; + case Bbr2CongestionController::State::ProbeRTT: + return "ProbeRTT"; + } +} +} // namespace quic diff --git a/quic/congestion_control/Bbr2.h b/quic/congestion_control/Bbr2.h new file mode 100644 index 000000000..9af884495 --- /dev/null +++ b/quic/congestion_control/Bbr2.h @@ -0,0 +1,211 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace quic { +class Bbr2CongestionController : public CongestionController { + public: + enum class State : uint8_t { + Startup = 0, + Drain = 1, + ProbeBw_Down = 2, + ProbeBw_Cruise = 3, + ProbeBw_Refill = 4, + ProbeBw_Up = 5, + ProbeRTT = 6 + }; + + explicit Bbr2CongestionController(QuicConnectionStateBase& conn); + void onRemoveBytesFromInflight(uint64_t bytesToRemove) override; + + void onPacketSent(const OutstandingPacketWrapper& packet) override; + + void onPacketAckOrLoss( + const AckEvent* FOLLY_NULLABLE ackEvent, + const LossEvent* FOLLY_NULLABLE lossEvent) override; + + FOLLY_NODISCARD uint64_t getWritableBytes() const noexcept override; + + FOLLY_NODISCARD uint64_t getCongestionWindow() const noexcept override; + + FOLLY_NODISCARD CongestionControlType type() const noexcept override; + + FOLLY_NODISCARD bool isInBackgroundMode() const override; + + FOLLY_NODISCARD bool isAppLimited() const override; + + void setAppLimited() noexcept override; + + void getStats(CongestionControllerStats& /*stats*/) const override; + + void setAppIdle(bool, TimePoint) noexcept override {} + + void setBandwidthUtilizationFactor(float) noexcept override {} + + [[nodiscard]] State getState() const noexcept; + + private: + void resetCongestionSignals(); + void resetLowerBounds(); + void updateLatestDeliverySignals(const AckEvent& ackEvent); + void updateCongestionSignals(const LossEvent* FOLLY_NULLABLE lossEvent); + void updateAckAggregation(const AckEvent& ackEvent); + void advanceLatestDeliverySignals(const AckEvent& ackEvent); + void boundBwForModel(); + void adaptUpperBounds( + uint64_t ackedBytes, + uint64_t inflightBytesAtLargestAckedPacket, + uint64_t lostBytes); + + void startRound(); + void updateRound(const AckEvent& ackEvent); + + void setPacing(); + void setCwnd(uint64_t ackedBytes, uint64_t lostBytes); + void saveCwnd(); + void restoreCwnd(); + void setSendQuantum(); + + void enterStartup(); + void checkStartupDone(); + void checkStartupFullBandwidth(); + void checkStartupHighLoss(); + + void enterDrain(); + void checkDrain(); + + void enterProbeRtt(); + void handleProbeRtt(); + void checkProbeRtt(uint64_t ackedBytes); + void checkProbeRttDone(); + void exitProbeRtt(); + void updateMinRtt(); + uint64_t getProbeRTTCwnd(); + void boundCwndForProbeRTT(); + + void enterProbeBW(); + void startProbeBwDown(); + void startProbeBwCruise(); + void updateProbeBwCyclePhase( + uint64_t ackedBytes, + uint64_t inflightBytesAtLargestAckedPacket, + uint64_t lostBytes); + void startProbeBwRefill(); + void startProbeBwUp(); + bool checkTimeToProbeBW(); + bool checkTimeToCruise(); + bool hasElapsedInPhase(std::chrono::microseconds interval); + bool checkInflightTooHigh( + uint64_t inflightBytesAtLargestAckedPacket, + uint64_t lostBytes); + bool isInflightTooHigh( + uint64_t inflightBytesAtLargestAckedPacket, + uint64_t lostBytes); + void handleInFlightTooHigh(uint64_t inflightBytesAtLargestAckedPacket); + void raiseInflightHiSlope(); + void probeInflightHiUpward(uint64_t ackedBytes); + + [[nodiscard]] uint64_t getTargetInflightWithGain(float gain = 1.0) const; + [[nodiscard]] uint64_t getTargetInflightWithHeadroom() const; + [[nodiscard]] uint64_t getBDPWithGain(float gain = 1.0) const; + [[nodiscard]] uint64_t addQuantizationBudget(uint64_t input) const; + + bool isProbeBwState(const Bbr2CongestionController::State state); + Bandwidth getBandwidthSampleFromAck(const AckEvent& ackEvent); + bool isRenoCoexistenceProbeTime(); + + QuicConnectionStateBase& conn_; + bool appLimited_{false}; + TimePoint appLimitedLastSendTime_; + State state_{State::Startup}; + + // Data Rate Model Parameters + WindowedFilter, uint64_t, uint64_t> + maxBwFilter_; + Bandwidth bandwidthHi_, bandwidthLo_, bandwidth_; + uint64_t cycleCount_{0}; // TODO: this can be one bit + + // Data Volume Model Parameters + std::chrono::microseconds minRtt_{kDefaultMinRtt}; + folly::Optional minRttTimestamp_; + + folly::Optional probeRttMinTimestamp_; + std::chrono::microseconds probeRttMinValue_{kDefaultMinRtt}; + folly::Optional probeRttDoneTimestamp_; + + bool probeRttExpired_{false}; + + uint64_t sendQuantum_, inflightMax_, inflightHi_, inflightLo_; + folly::Optional extraAckedStartTimestamp_; + uint64_t extraAckedDelivered_; + WindowedFilter, uint64_t, uint64_t> + maxExtraAckedFilter_; + + // Responding to congestion + Bandwidth bandwidthLatest_; + uint64_t inflightLatest_{0}; + uint64_t lossBytesInRound_{0}; + uint64_t lossEventsInRound_{0}; + bool lossRoundStart_{false}; + uint64_t lossRoundEndBytesSent_{0}; + float lossPctInLastRound_{0.0f}; + uint64_t lossEventsInLastRound_{0}; + bool inLossRecovery_{false}; + + // Cwnd + uint64_t cwndBytes_; + uint64_t previousCwndBytes_{0}; + bool cwndLimitedInRound_{false}; + + bool idleRestart_{false}; + bool inPacketConservation_{false}; + TimePoint packetConservationStartTime_; + + // Round counting + uint64_t nextRoundDelivered_{0}; + bool roundStart_{false}; + uint64_t roundCount_{0}; + + bool filledPipe_{false}; + Bandwidth filledPipeBandwidth_; + uint64_t filledPipeCount_{0}; + + float pacingGain_{1.0}; + float cwndGain_{1.0}; + + // ProbeBW + enum class AckPhase : uint8_t { + ProbeStopping = 0, + ProbeStarting = 1, + ProbeRefilling = 2, + ProbeFeedback = 3, + }; + + uint64_t probeUpCount_{0}; + TimePoint probeBWCycleStart_; + uint64_t roundsSinceBwProbe_; + std::chrono::milliseconds bwProbeWait_; + uint64_t bwProbeSamples_; + AckPhase ackPhase_; + uint64_t probeUpRounds_{0}; + uint64_t probeUpAcks_{0}; +}; + +std::string bbr2StateToString(Bbr2CongestionController::State state); +} // namespace quic diff --git a/quic/congestion_control/CMakeLists.txt b/quic/congestion_control/CMakeLists.txt index af26d20c2..6ad78fc20 100644 --- a/quic/congestion_control/CMakeLists.txt +++ b/quic/congestion_control/CMakeLists.txt @@ -10,6 +10,7 @@ add_library( BbrBandwidthSampler.cpp BbrRttSampler.cpp BbrTesting.cpp + Bbr2.cpp CongestionControlFunctions.cpp CongestionControllerFactory.cpp Copa.cpp diff --git a/quic/congestion_control/CongestionController.h b/quic/congestion_control/CongestionController.h index 81a459449..2d2d095b8 100644 --- a/quic/congestion_control/CongestionController.h +++ b/quic/congestion_control/CongestionController.h @@ -21,6 +21,10 @@ struct BbrStats { uint8_t state; }; +struct Bbr2Stats { + uint8_t state; +}; + struct CopaStats { double deltaParam; bool useRttStanding; @@ -34,6 +38,7 @@ struct CubicStats { union CongestionControllerStats { struct BbrStats bbrStats; + struct Bbr2Stats bbr2Stats; struct CopaStats copaStats; struct CubicStats cubicStats; }; diff --git a/quic/congestion_control/CongestionControllerFactory.cpp b/quic/congestion_control/CongestionControllerFactory.cpp index d02d1a919..affbc717b 100644 --- a/quic/congestion_control/CongestionControllerFactory.cpp +++ b/quic/congestion_control/CongestionControllerFactory.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -52,6 +53,11 @@ DefaultCongestionControllerFactory::makeCongestionController( congestionController = std::move(bbr); break; } + case CongestionControlType::BBR2: { + auto bbr2 = std::make_unique(conn); + congestionController = std::move(bbr2); + break; + } case CongestionControlType::StaticCwnd: { throw QuicInternalException( "StaticCwnd Congestion Controller cannot be " diff --git a/quic/congestion_control/ServerCongestionControllerFactory.cpp b/quic/congestion_control/ServerCongestionControllerFactory.cpp index c44b10567..d44615eef 100644 --- a/quic/congestion_control/ServerCongestionControllerFactory.cpp +++ b/quic/congestion_control/ServerCongestionControllerFactory.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -55,6 +56,11 @@ ServerCongestionControllerFactory::makeCongestionController( congestionController = std::move(bbr); break; } + case CongestionControlType::BBR2: { + auto bbr2 = std::make_unique(conn); + congestionController = std::move(bbr2); + break; + } case CongestionControlType::StaticCwnd: { throw QuicInternalException( "StaticCwnd Congestion Controller cannot be "