diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..66e05c24f --- /dev/null +++ b/.travis.yml @@ -0,0 +1,45 @@ +# Facebook projects that use `fbcode_builder` for continuous integration +# share this Travis configuration to run builds via Docker. + +sudo: required + +# Docker disables IPv6 in containers by default. Enable it for unit tests that need [::1]. +before_script: + - if [[ "$TRAVIS_OS_NAME" != "osx" ]]; + then + sudo build/fbcode_builder/docker_enable_ipv6.sh; + fi + +env: + global: + - travis_cache_dir=$HOME/travis_ccache + # Travis times out after 50 minutes. Very generously leave 10 minutes + # for setup (e.g. cache download, compression, and upload), so we never + # fail to cache the progress we made. + - docker_build_timeout=40m + +cache: + # Our build caches can be 200-300MB, so increase the timeout to 7 minutes + # to make sure we never fail to cache the progress we made. + timeout: 420 + directories: + - $HOME/travis_ccache # see docker_build_with_ccache.sh + +# Ugh, `services:` must be in the matrix, or we get `docker: command not found` +# https://github.com/travis-ci/travis-ci/issues/5142 +matrix: + include: + - env: ['os_image=ubuntu:16.04', gcc_version=5] + services: [docker] + +addons: + apt: + packages: python2.7 + +script: + # We don't want to write the script inline because of Travis kludginess -- + # it looks like it escapes " and \ in scripts when using `matrix:`. + - ./build/fbcode_builder/travis_docker_build.sh + +notifications: + webhooks: https://code.facebook.com/travis/webhook/ diff --git a/build/deps/github_hashes/facebook/folly-rev.txt b/build/deps/github_hashes/facebook/folly-rev.txt new file mode 100644 index 000000000..9590feb1c --- /dev/null +++ b/build/deps/github_hashes/facebook/folly-rev.txt @@ -0,0 +1 @@ +Subproject commit a5802280d92076d7adf14fce30f7c828557c6efe diff --git a/build/fbcode_builder/.gitignore b/build/fbcode_builder/.gitignore new file mode 100644 index 000000000..b98f3edfa --- /dev/null +++ b/build/fbcode_builder/.gitignore @@ -0,0 +1,5 @@ +# Facebook-internal CI builds don't have write permission outside of the +# source tree, so we install all projects into this directory. +/facebook_ci +__pycache__/ +*.pyc diff --git a/build/fbcode_builder/CMake/FindGMock.cmake b/build/fbcode_builder/CMake/FindGMock.cmake new file mode 100644 index 000000000..cd042dd9c --- /dev/null +++ b/build/fbcode_builder/CMake/FindGMock.cmake @@ -0,0 +1,80 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# Find libgmock +# +# LIBGMOCK_DEFINES - List of defines when using libgmock. +# LIBGMOCK_INCLUDE_DIR - where to find gmock/gmock.h, etc. +# LIBGMOCK_LIBRARIES - List of libraries when using libgmock. +# LIBGMOCK_FOUND - True if libgmock found. + +IF (LIBGMOCK_INCLUDE_DIR) + # Already in cache, be silent + SET(LIBGMOCK_FIND_QUIETLY TRUE) +ENDIF () + +find_package(GTest CONFIG QUIET) +if (TARGET GTest::gmock) + get_target_property(LIBGMOCK_DEFINES GTest::gtest INTERFACE_COMPILE_DEFINITIONS) + if (NOT ${LIBGMOCK_DEFINES}) + # Explicitly set to empty string if not found to avoid it being + # set to NOTFOUND and breaking compilation + set(LIBGMOCK_DEFINES "") + endif() + get_target_property(LIBGMOCK_INCLUDE_DIR GTest::gtest INTERFACE_INCLUDE_DIRECTORIES) + set(LIBGMOCK_LIBRARIES GTest::gmock_main GTest::gmock GTest::gtest) + set(LIBGMOCK_FOUND ON) + message(STATUS "Found gmock via config, defines=${LIBGMOCK_DEFINES}, include=${LIBGMOCK_INCLUDE_DIR}, libs=${LIBGMOCK_LIBRARIES}") +else() + + FIND_PATH(LIBGMOCK_INCLUDE_DIR gmock/gmock.h) + + FIND_LIBRARY(LIBGMOCK_MAIN_LIBRARY_DEBUG NAMES gmock_maind) + FIND_LIBRARY(LIBGMOCK_MAIN_LIBRARY_RELEASE NAMES gmock_main) + FIND_LIBRARY(LIBGMOCK_LIBRARY_DEBUG NAMES gmockd) + FIND_LIBRARY(LIBGMOCK_LIBRARY_RELEASE NAMES gmock) + FIND_LIBRARY(LIBGTEST_LIBRARY_DEBUG NAMES gtestd) + FIND_LIBRARY(LIBGTEST_LIBRARY_RELEASE NAMES gtest) + + find_package(Threads REQUIRED) + INCLUDE(SelectLibraryConfigurations) + SELECT_LIBRARY_CONFIGURATIONS(LIBGMOCK_MAIN) + SELECT_LIBRARY_CONFIGURATIONS(LIBGMOCK) + SELECT_LIBRARY_CONFIGURATIONS(LIBGTEST) + + set(LIBGMOCK_LIBRARIES + ${LIBGMOCK_MAIN_LIBRARY} + ${LIBGMOCK_LIBRARY} + ${LIBGTEST_LIBRARY} + Threads::Threads + ) + + if(CMAKE_SYSTEM_NAME STREQUAL "Windows") + # The GTEST_LINKED_AS_SHARED_LIBRARY macro must be set properly on Windows. + # + # There isn't currently an easy way to determine if a library was compiled as + # a shared library on Windows, so just assume we've been built against a + # shared build of gmock for now. + SET(LIBGMOCK_DEFINES "GTEST_LINKED_AS_SHARED_LIBRARY=1" CACHE STRING "") + endif() + + # handle the QUIETLY and REQUIRED arguments and set LIBGMOCK_FOUND to TRUE if + # all listed variables are TRUE + INCLUDE(FindPackageHandleStandardArgs) + FIND_PACKAGE_HANDLE_STANDARD_ARGS( + GMock + DEFAULT_MSG + LIBGMOCK_MAIN_LIBRARY + LIBGMOCK_LIBRARY + LIBGTEST_LIBRARY + LIBGMOCK_LIBRARIES + LIBGMOCK_INCLUDE_DIR + ) + + MARK_AS_ADVANCED( + LIBGMOCK_DEFINES + LIBGMOCK_MAIN_LIBRARY + LIBGMOCK_LIBRARY + LIBGTEST_LIBRARY + LIBGMOCK_LIBRARIES + LIBGMOCK_INCLUDE_DIR + ) +endif() diff --git a/build/fbcode_builder/CMake/FindGflags.cmake b/build/fbcode_builder/CMake/FindGflags.cmake new file mode 100644 index 000000000..246ceacdd --- /dev/null +++ b/build/fbcode_builder/CMake/FindGflags.cmake @@ -0,0 +1,81 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# Find libgflags. +# There's a lot of compatibility cruft going on in here, both +# to deal with changes across the FB consumers of this and also +# to deal with variances in behavior of cmake itself. +# +# Since this file is named FindGflags.cmake the cmake convention +# is for the module to export both GFLAGS_FOUND and Gflags_FOUND. +# The convention expected by consumers is that we export the +# following variables, even though these do not match the cmake +# conventions: +# +# LIBGFLAGS_INCLUDE_DIR - where to find gflags/gflags.h, etc. +# LIBGFLAGS_LIBRARY - List of libraries when using libgflags. +# LIBGFLAGS_FOUND - True if libgflags found. +# +# We need to be able to locate gflags both from an installed +# cmake config file and just from the raw headers and libs, so +# test for the former and then the latter, and then stick +# the results together and export them into the variables +# listed above. +# +# For forwards compatibility, we export the following variables: +# +# gflags_INCLUDE_DIR - where to find gflags/gflags.h, etc. +# gflags_TARGET / GFLAGS_TARGET / gflags_LIBRARIES +# - List of libraries when using libgflags. +# gflags_FOUND - True if libgflags found. +# + +IF (LIBGFLAGS_INCLUDE_DIR) + # Already in cache, be silent + SET(Gflags_FIND_QUIETLY TRUE) +ENDIF () + +find_package(gflags CONFIG QUIET) +if (gflags_FOUND) + if (NOT Gflags_FIND_QUIETLY) + message(STATUS "Found gflags from package config ${gflags_CONFIG}") + endif() + # Re-export the config-specified libs with our local names + set(LIBGFLAGS_LIBRARY ${gflags_LIBRARIES}) + set(LIBGFLAGS_INCLUDE_DIR ${gflags_INCLUDE_DIR}) + set(LIBGFLAGS_FOUND ${gflags_FOUND}) + # cmake module compat + set(GFLAGS_FOUND ${gflags_FOUND}) + set(Gflags_FOUND ${gflags_FOUND}) +else() + FIND_PATH(LIBGFLAGS_INCLUDE_DIR gflags/gflags.h) + + FIND_LIBRARY(LIBGFLAGS_LIBRARY_DEBUG NAMES gflagsd gflags_staticd) + FIND_LIBRARY(LIBGFLAGS_LIBRARY_RELEASE NAMES gflags gflags_static) + + INCLUDE(SelectLibraryConfigurations) + SELECT_LIBRARY_CONFIGURATIONS(LIBGFLAGS) + + # handle the QUIETLY and REQUIRED arguments and set LIBGFLAGS_FOUND to TRUE if + # all listed variables are TRUE + INCLUDE(FindPackageHandleStandardArgs) + FIND_PACKAGE_HANDLE_STANDARD_ARGS(gflags DEFAULT_MSG LIBGFLAGS_LIBRARY LIBGFLAGS_INCLUDE_DIR) + # cmake module compat + set(Gflags_FOUND ${GFLAGS_FOUND}) + # compat with some existing FindGflags consumers + set(LIBGFLAGS_FOUND ${GFLAGS_FOUND}) + + # Compat with the gflags CONFIG based detection + set(gflags_FOUND ${GFLAGS_FOUND}) + set(gflags_INCLUDE_DIR ${LIBGFLAGS_INCLUDE_DIR}) + set(gflags_LIBRARIES ${LIBGFLAGS_LIBRARY}) + set(GFLAGS_TARGET ${LIBGFLAGS_LIBRARY}) + set(gflags_TARGET ${LIBGFLAGS_LIBRARY}) + + MARK_AS_ADVANCED(LIBGFLAGS_LIBRARY LIBGFLAGS_INCLUDE_DIR) +endif() + +# Compat with the gflags CONFIG based detection +if (LIBGFLAGS_FOUND AND NOT TARGET gflags) + add_library(gflags UNKNOWN IMPORTED) + set_target_properties(gflags PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${LIBGFLAGS_INCLUDE_DIR}") + set_target_properties(gflags PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES "C" IMPORTED_LOCATION "${LIBGFLAGS_LIBRARY}") +endif() diff --git a/build/fbcode_builder/CMake/FindGlog.cmake b/build/fbcode_builder/CMake/FindGlog.cmake new file mode 100644 index 000000000..a589b2e37 --- /dev/null +++ b/build/fbcode_builder/CMake/FindGlog.cmake @@ -0,0 +1,32 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# - Try to find Glog +# Once done, this will define +# +# GLOG_FOUND - system has Glog +# GLOG_INCLUDE_DIRS - the Glog include directories +# GLOG_LIBRARIES - link these to use Glog + +include(FindPackageHandleStandardArgs) + +find_library(GLOG_LIBRARY glog + PATHS ${GLOG_LIBRARYDIR}) + +find_path(GLOG_INCLUDE_DIR glog/logging.h + PATHS ${GLOG_INCLUDEDIR}) + +find_package_handle_standard_args(glog DEFAULT_MSG + GLOG_LIBRARY + GLOG_INCLUDE_DIR) + +mark_as_advanced( + GLOG_LIBRARY + GLOG_INCLUDE_DIR) + +set(GLOG_LIBRARIES ${GLOG_LIBRARY}) +set(GLOG_INCLUDE_DIRS ${GLOG_INCLUDE_DIR}) + +if (NOT TARGET glog::glog) + add_library(glog::glog UNKNOWN IMPORTED) + set_target_properties(glog::glog PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${GLOG_INCLUDE_DIRS}") + set_target_properties(glog::glog PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES "C" IMPORTED_LOCATION "${GLOG_LIBRARIES}") +endif() diff --git a/build/fbcode_builder/CMake/FindLibEvent.cmake b/build/fbcode_builder/CMake/FindLibEvent.cmake new file mode 100644 index 000000000..dd11ebd84 --- /dev/null +++ b/build/fbcode_builder/CMake/FindLibEvent.cmake @@ -0,0 +1,77 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# - Find LibEvent (a cross event library) +# This module defines +# LIBEVENT_INCLUDE_DIR, where to find LibEvent headers +# LIBEVENT_LIB, LibEvent libraries +# LibEvent_FOUND, If false, do not try to use libevent + +set(LibEvent_EXTRA_PREFIXES /usr/local /opt/local "$ENV{HOME}") +foreach(prefix ${LibEvent_EXTRA_PREFIXES}) + list(APPEND LibEvent_INCLUDE_PATHS "${prefix}/include") + list(APPEND LibEvent_LIB_PATHS "${prefix}/lib") +endforeach() + +find_package(Libevent CONFIG QUIET) +if (TARGET event) + # Re-export the config under our own names + + # Somewhat gross, but some vcpkg installed libevents have a relative + # `include` path exported into LIBEVENT_INCLUDE_DIRS, which triggers + # a cmake error because it resolves to the `include` dir within the + # folly repo, which is not something cmake allows to be in the + # INTERFACE_INCLUDE_DIRECTORIES. Thankfully on such a system the + # actual include directory is already part of the global include + # directories, so we can just skip it. + if (NOT "${LIBEVENT_INCLUDE_DIRS}" STREQUAL "include") + set(LIBEVENT_INCLUDE_DIR ${LIBEVENT_INCLUDE_DIRS}) + else() + set(LIBEVENT_INCLUDE_DIR) + endif() + + # Unfortunately, with a bare target name `event`, downstream consumers + # of the package that depends on `Libevent` located via CONFIG end + # up exporting just a bare `event` in their libraries. This is problematic + # because this in interpreted as just `-levent` with no library path. + # When libevent is not installed in the default installation prefix + # this results in linker errors. + # To resolve this, we ask cmake to lookup the full path to the library + # and use that instead. + cmake_policy(PUSH) + if(POLICY CMP0026) + # Allow reading the LOCATION property + cmake_policy(SET CMP0026 OLD) + endif() + get_target_property(LIBEVENT_LIB event LOCATION) + cmake_policy(POP) + + set(LibEvent_FOUND ${Libevent_FOUND}) + if (NOT LibEvent_FIND_QUIETLY) + message(STATUS "Found libevent from package config include=${LIBEVENT_INCLUDE_DIRS} lib=${LIBEVENT_LIB}") + endif() +else() + find_path(LIBEVENT_INCLUDE_DIR event.h PATHS ${LibEvent_INCLUDE_PATHS}) + find_library(LIBEVENT_LIB NAMES event PATHS ${LibEvent_LIB_PATHS}) + + if (LIBEVENT_LIB AND LIBEVENT_INCLUDE_DIR) + set(LibEvent_FOUND TRUE) + set(LIBEVENT_LIB ${LIBEVENT_LIB}) + else () + set(LibEvent_FOUND FALSE) + endif () + + if (LibEvent_FOUND) + if (NOT LibEvent_FIND_QUIETLY) + message(STATUS "Found libevent: ${LIBEVENT_LIB}") + endif () + else () + if (LibEvent_FIND_REQUIRED) + message(FATAL_ERROR "Could NOT find libevent.") + endif () + message(STATUS "libevent NOT found.") + endif () + + mark_as_advanced( + LIBEVENT_LIB + LIBEVENT_INCLUDE_DIR + ) +endif() diff --git a/build/fbcode_builder/CMake/FindPCRE.cmake b/build/fbcode_builder/CMake/FindPCRE.cmake new file mode 100644 index 000000000..32ccb3725 --- /dev/null +++ b/build/fbcode_builder/CMake/FindPCRE.cmake @@ -0,0 +1,11 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +include(FindPackageHandleStandardArgs) +find_path(PCRE_INCLUDE_DIR NAMES pcre.h) +find_library(PCRE_LIBRARY NAMES pcre) +find_package_handle_standard_args( + PCRE + DEFAULT_MSG + PCRE_LIBRARY + PCRE_INCLUDE_DIR +) +mark_as_advanced(PCRE_INCLUDE_DIR PCRE_LIBRARY) diff --git a/build/fbcode_builder/CMake/ThriftCppLibrary.cmake b/build/fbcode_builder/CMake/ThriftCppLibrary.cmake new file mode 100644 index 000000000..2613e888a --- /dev/null +++ b/build/fbcode_builder/CMake/ThriftCppLibrary.cmake @@ -0,0 +1,120 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# NOTE: If you change this file, fbcode/fboss/github/ThriftCppLibrary.cmake also +# needs to be changed. TODO: this should be handled via shipit. +function(add_thrift_cpp2_library LIB_NAME THRIFT_FILE) + # Parse the arguments + set(SERVICES) + set(DEPENDS) + set(GEN_ARGS) + set(mode "UNSET") + foreach(arg IN LISTS ARGN) + if("${arg}" STREQUAL "SERVICES") + set(mode "SERVICES") + elseif("${arg}" STREQUAL "DEPENDS") + set(mode "DEPENDS") + elseif("${arg}" STREQUAL "OPTIONS") + set(mode "OPTIONS") + else() + if("${mode}" STREQUAL "SERVICES") + list(APPEND SERVICES "${arg}") + elseif("${mode}" STREQUAL "DEPENDS") + list(APPEND DEPENDS "${arg}") + elseif("${mode}" STREQUAL "OPTIONS") + list(APPEND GEN_ARGS "${arg}") + else() + message( + FATAL_ERROR + "expected SERVICES, DEPENDS, or OPTIONS argument, found ${arg}" + ) + endif() + endif() + endforeach() + + get_filename_component(base ${THRIFT_FILE} NAME_WE) + get_filename_component( + output_dir + ${CMAKE_CURRENT_BINARY_DIR}/${THRIFT_FILE} + DIRECTORY + ) + + list(APPEND GEN_ARGS "include_prefix=${output_dir}") + # CMake 3.12 is finally getting a list(JOIN) function, but until then + # treating the list as a string and replacing the semicolons is good enough. + string(REPLACE ";" "," GEN_ARG_STR "${GEN_ARGS}") + + # Compute the list of generated files + list(APPEND generated_headers + ${output_dir}/gen-cpp2/${base}_constants.h + ${output_dir}/gen-cpp2/${base}_constants.cpp + ${output_dir}/gen-cpp2/${base}_types.h + ${output_dir}/gen-cpp2/${base}_types.tcc + ${output_dir}/gen-cpp2/${base}_types_custom_protocol.h + ) + list(APPEND generated_sources + ${output_dir}/gen-cpp2/${base}_data.h + ${output_dir}/gen-cpp2/${base}_data.cpp + ${output_dir}/gen-cpp2/${base}_types.cpp + ) + foreach(service IN LISTS SERVICES) + list(APPEND generated_headers + ${output_dir}/gen-cpp2/${service}.h + ${output_dir}/gen-cpp2/${service}.tcc + ${output_dir}/gen-cpp2/${service}AsyncClient.h + ${output_dir}/gen-cpp2/${service}_custom_protocol.h + ) + list(APPEND generated_sources + ${output_dir}/gen-cpp2/${service}.cpp + ${output_dir}/gen-cpp2/${service}AsyncClient.cpp + ${output_dir}/gen-cpp2/${service}_processmap_binary.cpp + ${output_dir}/gen-cpp2/${service}_processmap_compact.cpp + ) + endforeach() + + # Emit the rule to run the thrift compiler + add_custom_command( + OUTPUT + ${generated_headers} + ${generated_sources} + COMMAND + ${CMAKE_COMMAND} -E make_directory ${output_dir} + COMMAND + ${FBTHRIFT_COMPILER} + --strict + --templates ${FBTHRIFT_TEMPLATES_DIR} + --gen "mstch_cpp2:${GEN_ARG_STR}" + -I ${CMAKE_SOURCE_DIR} + -o ${output_dir} + ${CMAKE_CURRENT_SOURCE_DIR}/${THRIFT_FILE} + WORKING_DIRECTORY + ${CMAKE_BINARY_DIR} + MAIN_DEPENDENCY + ${THRIFT_FILE} + DEPENDS + ${DEPENDS} + ) + + # Now emit the library rule to compile the sources + add_library(${LIB_NAME} STATIC + ${generated_sources} + ) + set_property( + TARGET ${LIB_NAME} + PROPERTY PUBLIC_HEADER + ${generated_headers} + ) + target_include_directories( + ${LIB_NAME} + PUBLIC + ${CMAKE_SOURCE_DIR} + ${CMAKE_BINARY_DIR} + ${FOLLY_INCLUDE_DIR} + ${FBTHRIFT_INCLUDE_DIR} + ) + target_link_libraries( + ${LIB_NAME} + PUBLIC + ${DEPENDS} + FBThrift::thriftcpp2 + Folly::folly + ) +endfunction() diff --git a/build/fbcode_builder/README.docker b/build/fbcode_builder/README.docker new file mode 100644 index 000000000..4e9fa8a29 --- /dev/null +++ b/build/fbcode_builder/README.docker @@ -0,0 +1,44 @@ +## Debugging Docker builds + +To debug a a build failure, start up a shell inside the just-failed image as +follows: + +``` +docker ps -a | head # Grab the container ID +docker commit CONTAINER_ID # Grab the SHA string +docker run -it SHA_STRING /bin/bash +# Debug as usual, e.g. `./run-cmake.sh Debug`, `make`, `apt-get install gdb` +``` + +## A note on Docker security + +While the Dockerfile generated above is quite simple, you must be aware that +using Docker to run arbitrary code can present significant security risks: + + - Code signature validation is off by default (as of 2016), exposing you to + man-in-the-middle malicious code injection. + + - You implicitly trust the world -- a Dockerfile cannot annotate that + you trust the image `debian:8.6` because you trust a particular + certificate -- rather, you trust the name, and that it will never be + hijacked. + + - Sandboxing in the Linux kernel is not perfect, and the builds run code as + root. Any compromised code can likely escalate to the host system. + +Specifically, you must be very careful only to add trusted OS images to the +build flow. + +Consider setting this variable before running any Docker container -- this +will validate a signature on the base image before running code from it: + +``` +export DOCKER_CONTENT_TRUST=1 +``` + +Note that unless you go through the extra steps of notarizing the resulting +images, you will have to disable trust to enter intermediate images, e.g. + +``` +DOCKER_CONTENT_TRUST= docker run -it YOUR_IMAGE_ID /bin/bash +``` diff --git a/build/fbcode_builder/README.md b/build/fbcode_builder/README.md new file mode 100644 index 000000000..084601eeb --- /dev/null +++ b/build/fbcode_builder/README.md @@ -0,0 +1,60 @@ +# Easy builds for Facebook projects + +This is a Python 2.6+ library designed to simplify continuous-integration +(and other builds) of Facebook projects. + +For external Travis builds, the entry point is `travis_docker_build.sh`. + + +## Using Docker to reproduce a CI build + +If you are debugging or enhancing a CI build, you will want to do so from +host or virtual machine that can run a reasonably modern version of Docker: + +``` sh +./make_docker_context.py --help # See available options for OS & compiler +# Tiny wrapper that starts a Travis-like build with compile caching: +os_image=ubuntu:16.04 \ + gcc_version=5 \ + make_parallelism=2 \ + travis_cache_dir=~/travis_ccache \ + ./travis_docker_build.sh &> build_at_$(date +'%Y%m%d_%H%M%S').log +``` + +**IMPORTANT**: Read `fbcode_builder/README.docker` before diving in! + +Setting `travis_cache_dir` turns on [ccache](https://ccache.samba.org/), +saving a fresh copy of `ccache.tgz` after every build. This will invalidate +Docker's layer cache, foring it to rebuild starting right after OS package +setup, but the builds will be fast because all the compiles will be cached. +To iterate without invalidating the Docker layer cache, just `cd +/tmp/docker-context-*` and interact with the `Dockerfile` normally. Note +that the `docker-context-*` dirs preserve a copy of `ccache.tgz` as they +first used it. + + +# What to read next + +The *.py files are fairly well-documented. You might want to peruse them +in this order: + - shell_quoting.py + - fbcode_builder.py + - docker_builder.py + - make_docker_context.py + +As far as runs on Travis go, the control flow is: + - .travis.yml calls + - travis_docker_build.sh calls + - docker_build_with_ccache.sh + +This library also has an (unpublished) component targeting Facebook's +internal continuous-integration platform using the same build-step DSL. + + +# Contributing + +Please follow the ambient style (or PEP-8), and keep the code Python 2.6 +compatible -- since `fbcode_builder`'s only dependency is Docker, we want to +allow building projects on even fairly ancient base systems. We also wish +to be compatible with Python 3, and would appreciate it if you kept that +in mind while making changes also. diff --git a/build/fbcode_builder/docker_build_with_ccache.sh b/build/fbcode_builder/docker_build_with_ccache.sh new file mode 100755 index 000000000..e922810d5 --- /dev/null +++ b/build/fbcode_builder/docker_build_with_ccache.sh @@ -0,0 +1,219 @@ +#!/bin/bash -uex +# Copyright (c) Facebook, Inc. and its affiliates. +set -o pipefail # Be sure to `|| :` commands that are allowed to fail. + +# +# Future: port this to Python if you are making significant changes. +# + +# Parse command-line arguments +build_timeout="" # Default to no time-out +print_usage() { + echo "Usage: $0 [--build-timeout TIMEOUT_VAL] SAVE-CCACHE-TO-DIR" + echo "SAVE-CCACHE-TO-DIR is required. An empty string discards the ccache." +} +while [[ $# -gt 0 ]]; do + case "$1" in + --build-timeout) + shift + build_timeout="$1" + if [[ "$build_timeout" != "" ]] ; then + timeout "$build_timeout" true # fail early on invalid timeouts + fi + ;; + -h|--help) + print_usage + exit + ;; + *) + break + ;; + esac + shift +done +# There is one required argument, but an empty string is allowed. +if [[ "$#" != 1 ]] ; then + print_usage + exit 1 +fi +save_ccache_to_dir="$1" +if [[ "$save_ccache_to_dir" != "" ]] ; then + mkdir -p "$save_ccache_to_dir" # fail early if there's nowhere to save +else + echo "WARNING: Will not save /ccache from inside the Docker container" +fi + +rand_guid() { + echo "$(date +%s)_${RANDOM}_${RANDOM}_${RANDOM}_${RANDOM}" +} + +id=fbcode_builder_image_id=$(rand_guid) +logfile=$(mktemp) + +echo " + + +Running build with timeout '$build_timeout', label $id, and log in $logfile + + +" + +if [[ "$build_timeout" != "" ]] ; then + # Kill the container after $build_timeout. Using `/bin/timeout` would cause + # Docker to destroy the most recent container and lose its cache. + ( + sleep "$build_timeout" + echo "Build timed out after $build_timeout" 1>&2 + while true; do + maybe_container=$( + grep -E '^( ---> Running in [0-9a-f]+|FBCODE_BUILDER_EXIT)$' "$logfile" | + tail -n 1 | awk '{print $NF}' + ) + if [[ "$maybe_container" == "FBCODE_BUILDER_EXIT" ]] ; then + echo "Time-out successfully terminated build" 1>&2 + break + fi + echo "Time-out: trying to kill $maybe_container" 1>&2 + # This kill fail if we get unlucky, try again soon. + docker kill "$maybe_container" || sleep 5 + done + ) & +fi + +build_exit_code=0 +# `docker build` is allowed to fail, and `pipefail` means we must check the +# failure explicitly. +if ! docker build --label="$id" . 2>&1 | tee "$logfile" ; then + build_exit_code="${PIPESTATUS[0]}" + # NB: We are going to deliberately forge ahead even if `tee` failed. + # If it did, we have a problem with tempfile creation, and all is sad. + echo "Build failed with code $build_exit_code, trying to save ccache" 1>&2 +fi +# Stop trying to kill the container. +echo $'\nFBCODE_BUILDER_EXIT' >> "$logfile" + +if [[ "$save_ccache_to_dir" == "" ]] ; then + echo "Not inspecting Docker build, since saving the ccache wasn't requested." + exit "$build_exit_code" +fi + +img=$(docker images --filter "label=$id" -a -q) +if [[ "$img" == "" ]] ; then + docker images -a + echo "In the above list, failed to find most recent image with $id" 1>&2 + # Usually, the above `docker kill` will leave us with an up-to-the-second + # container, from which we can extract the cache. However, if that fails + # for any reason, this loop will instead grab the latest available image. + # + # It's possible for this log search to get confused due to the output of + # the build command itself, but since our builds aren't **trying** to + # break cache, we probably won't randomly hit an ID from another build. + img=$( + grep -E '^ ---> (Running in [0-9a-f]+|[0-9a-f]+)$' "$logfile" | tac | + sed 's/Running in /container_/;s/ ---> //;' | ( + while read -r x ; do + # Both docker commands below print an image ID to stdout on + # success, so we just need to know when to stop. + if [[ "$x" =~ container_.* ]] ; then + if docker commit "${x#container_}" ; then + break + fi + elif docker inspect --type image -f '{{.Id}}' "$x" ; then + break + fi + done + ) + ) + if [[ "$img" == "" ]] ; then + echo "Failed to find valid container or image ID in log $logfile" 1>&2 + exit 1 + fi +elif [[ "$(echo "$img" | wc -l)" != 1 ]] ; then + # Shouldn't really happen, but be explicit if it does. + echo "Multiple images with label $id, taking the latest of:" + echo "$img" + img=$(echo "$img" | head -n 1) +fi + +container_name="fbcode_builder_container_$(rand_guid)" +echo "Starting $container_name from latest image of the build with $id --" +echo "$img" + +# ccache collection must be done outside of the Docker build steps because +# we need to be able to kill it on timeout. +# +# This step grows the max cache size to slightly exceed than the working set +# of a successful build. This simple design persists the max size in the +# cache directory itself (the env var CCACHE_MAXSIZE does not even work with +# older ccaches like the one on 14.04). +# +# Future: copy this script into the Docker image via Dockerfile. +( + # By default, fbcode_builder creates an unsigned image, so the `docker + # run` below would fail if DOCKER_CONTENT_TRUST were set. So we unset it + # just for this one run. + export DOCKER_CONTENT_TRUST= + # CAUTION: The inner bash runs without -uex, so code accordingly. + docker run --user root --name "$container_name" "$img" /bin/bash -c ' + build_exit_code='"$build_exit_code"' + + # Might be useful if debugging whether max cache size is too small? + grep " Cleaning up cache directory " /tmp/ccache.log + + export CCACHE_DIR=/ccache + ccache -s + + echo "Total bytes in /ccache:"; + total_bytes=$(du -sb /ccache | awk "{print \$1}") + echo "$total_bytes" + + echo "Used bytes in /ccache:"; + used_bytes=$( + du -sb $(find /ccache -type f -newermt @$( + cat /FBCODE_BUILDER_CCACHE_START_TIME + )) | awk "{t += \$1} END {print t}" + ) + echo "$used_bytes" + + # Goal: set the max cache to 750MB over 125% of the usage of a + # successful build. If this is too small, it takes too long to get a + # cache fully warmed up. Plus, ccache cleans 100-200MB before reaching + # the max cache size, so a large margin is essential to prevent misses. + desired_mb=$(( 750 + used_bytes / 800000 )) # 125% in decimal MB: 1e6/1.25 + if [[ "$build_exit_code" != "0" ]] ; then + # For a bad build, disallow shrinking the max cache size. Instead of + # the max cache size, we use on-disk size, which ccache keeps at least + # 150MB under the actual max size, hence the 400MB safety margin. + cur_max_mb=$(( 400 + total_bytes / 1000000 )) # ccache uses decimal MB + if [[ "$desired_mb" -le "$cur_max_mb" ]] ; then + desired_mb="" + fi + fi + + if [[ "$desired_mb" != "" ]] ; then + echo "Updating cache size to $desired_mb MB" + ccache -M "${desired_mb}M" + ccache -s + fi + + # Subshell because `time` the binary may not be installed. + if (time tar czf /ccache.tgz /ccache) ; then + ls -l /ccache.tgz + else + # This `else` ensures we never overwrite the current cache with + # partial data in case of error, even if somebody adds code below. + rm /ccache.tgz + exit 1 + fi + ' +) + +echo "Updating $save_ccache_to_dir/ccache.tgz" +# This will not delete the existing cache if `docker run` didn't make one +docker cp "$container_name:/ccache.tgz" "$save_ccache_to_dir/" + +# Future: it'd be nice if Travis allowed us to retry if the build timed out, +# since we'll make more progress thanks to the cache. As-is, we have to +# wait for the next commit to land. +echo "Build exited with code $build_exit_code" +exit "$build_exit_code" diff --git a/build/fbcode_builder/docker_builder.py b/build/fbcode_builder/docker_builder.py new file mode 100644 index 000000000..aa251f8a4 --- /dev/null +++ b/build/fbcode_builder/docker_builder.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +''' + +Extends FBCodeBuilder to produce Docker context directories. + +In order to get the largest iteration-time savings from Docker's build +caching, you will want to: + - Use fine-grained steps as appropriate (e.g. separate make & make install), + - Start your action sequence with the lowest-risk steps, and with the steps + that change the least often, and + - Put the steps that you are debugging towards the very end. + +''' +import logging +import os +import shutil +import tempfile + +from fbcode_builder import FBCodeBuilder +from shell_quoting import ( + raw_shell, shell_comment, shell_join, ShellQuoted, path_join +) +from utils import recursively_flatten_list, run_command + + +class DockerFBCodeBuilder(FBCodeBuilder): + + def _user(self): + return self.option('user', 'root') + + def _change_user(self): + return ShellQuoted('USER {u}').format(u=self._user()) + + def setup(self): + # Please add RPM-based OSes here as appropriate. + # + # To allow exercising non-root installs -- we change users after the + # system packages are installed. TODO: For users not defined in the + # image, we should probably `useradd`. + return self.step('Setup', [ + # Docker's FROM does not understand shell quoting. + ShellQuoted('FROM {}'.format(self.option('os_image'))), + # /bin/sh syntax is a pain + ShellQuoted('SHELL ["/bin/bash", "-c"]'), + ] + self.install_debian_deps() + [self._change_user()] + + [self.workdir(self.option('prefix')), + self.create_python_venv(), + self.python_venv()]) + + def python_venv(self): + # To both avoid calling venv activate on each RUN command AND to ensure + # it is present when the resulting container is run add to PATH + actions = [] + if self.option("PYTHON_VENV", "OFF") == "ON": + actions = ShellQuoted('ENV PATH={p}:$PATH').format( + p=path_join(self.option('prefix'), "venv", "bin")) + return(actions) + + def step(self, name, actions): + assert '\n' not in name, 'Name {0} would span > 1 line'.format(name) + b = ShellQuoted('') + return [ShellQuoted('### {0} ###'.format(name)), b] + actions + [b] + + def run(self, shell_cmd): + return ShellQuoted('RUN {cmd}').format(cmd=shell_cmd) + + def workdir(self, dir): + return [ + # As late as Docker 1.12.5, this results in `build` being owned + # by root:root -- the explicit `mkdir` works around the bug: + # USER nobody + # WORKDIR build + ShellQuoted('USER root'), + ShellQuoted('RUN mkdir -p {d} && chown {u} {d}').format( + d=dir, u=self._user() + ), + self._change_user(), + ShellQuoted('WORKDIR {dir}').format(dir=dir), + ] + + def comment(self, comment): + # This should not be a command since we don't want comment changes + # to invalidate the Docker build cache. + return shell_comment(comment) + + def copy_local_repo(self, repo_dir, dest_name): + fd, archive_path = tempfile.mkstemp( + prefix='local_repo_{0}_'.format(dest_name), + suffix='.tgz', + dir=os.path.abspath(self.option('docker_context_dir')), + ) + os.close(fd) + run_command('tar', 'czf', archive_path, '.', cwd=repo_dir) + return [ + ShellQuoted('ADD {archive} {dest_name}').format( + archive=os.path.basename(archive_path), dest_name=dest_name + ), + # Docker permissions make very little sense... see also workdir() + ShellQuoted('USER root'), + ShellQuoted('RUN chown -R {u} {d}').format( + d=dest_name, u=self._user() + ), + self._change_user(), + ] + + def _render_impl(self, steps): + return raw_shell(shell_join('\n', recursively_flatten_list(steps))) + + def debian_ccache_setup_steps(self): + source_ccache_tgz = self.option('ccache_tgz', '') + if not source_ccache_tgz: + logging.info('Docker ccache not enabled') + return [] + + dest_ccache_tgz = os.path.join( + self.option('docker_context_dir'), 'ccache.tgz' + ) + + try: + try: + os.link(source_ccache_tgz, dest_ccache_tgz) + except OSError: + logging.exception( + 'Hard-linking {s} to {d} failed, falling back to copy' + .format(s=source_ccache_tgz, d=dest_ccache_tgz) + ) + shutil.copyfile(source_ccache_tgz, dest_ccache_tgz) + except Exception: + logging.exception( + 'Failed to copy or link {s} to {d}, aborting' + .format(s=source_ccache_tgz, d=dest_ccache_tgz) + ) + raise + + return [ + # Separate layer so that in development we avoid re-downloads. + self.run(ShellQuoted('apt-get install -yq ccache')), + ShellQuoted('ADD ccache.tgz /'), + ShellQuoted( + # Set CCACHE_DIR before the `ccache` invocations below. + 'ENV CCACHE_DIR=/ccache ' + # No clang support for now, so it's easiest to hardcode gcc. + 'CC="ccache gcc" CXX="ccache g++" ' + # Always log for ease of debugging. For real FB projects, + # this log is several megabytes, so dumping it to stdout + # would likely exceed the Travis log limit of 4MB. + # + # On a local machine, `docker cp` will get you the data. To + # get the data out from Travis, I would compress and dump + # uuencoded bytes to the log -- for Bistro this was about + # 600kb or 8000 lines: + # + # apt-get install sharutils + # bzip2 -9 < /tmp/ccache.log | uuencode -m ccache.log.bz2 + 'CCACHE_LOGFILE=/tmp/ccache.log' + ), + self.run(ShellQuoted( + # Future: Skipping this part made this Docker step instant, + # saving ~1min of build time. It's unclear if it is the + # chown or the du, but probably the chown -- since a large + # part of the cost is incurred at image save time. + # + # ccache.tgz may be empty, or may have the wrong + # permissions. + 'mkdir -p /ccache && time chown -R nobody /ccache && ' + 'time du -sh /ccache && ' + # Reset stats so `docker_build_with_ccache.sh` can print + # useful values at the end of the run. + 'echo === Prev run stats === && ccache -s && ccache -z && ' + # Record the current time to let travis_build.sh figure out + # the number of bytes in the cache that are actually used -- + # this is crucial for tuning the maximum cache size. + 'date +%s > /FBCODE_BUILDER_CCACHE_START_TIME && ' + # The build running as `nobody` should be able to write here + 'chown nobody /tmp/ccache.log' + )), + ] diff --git a/build/fbcode_builder/docker_enable_ipv6.sh b/build/fbcode_builder/docker_enable_ipv6.sh new file mode 100755 index 000000000..3752f6f5e --- /dev/null +++ b/build/fbcode_builder/docker_enable_ipv6.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# Copyright (c) Facebook, Inc. and its affiliates. + + +# `daemon.json` is normally missing, but let's log it in case that changes. +touch /etc/docker/daemon.json +service docker stop +echo '{"ipv6": true, "fixed-cidr-v6": "2001:db8:1::/64"}' > /etc/docker/daemon.json +service docker start +# Fail early if docker failed on start -- add `- sudo dockerd` to debug. +docker info +# Paranoia log: what if our config got overwritten? +cat /etc/docker/daemon.json diff --git a/build/fbcode_builder/fbcode_builder.py b/build/fbcode_builder/fbcode_builder.py new file mode 100644 index 000000000..75b063167 --- /dev/null +++ b/build/fbcode_builder/fbcode_builder.py @@ -0,0 +1,384 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +''' + +This is a small DSL to describe builds of Facebook's open-source projects +that are published to Github from a single internal repo, including projects +that depend on folly, wangle, proxygen, fbthrift, etc. + +This file defines the interface of the DSL, and common utilieis, but you +will have to instantiate a specific builder, with specific options, in +order to get work done -- see e.g. make_docker_context.py. + +== Design notes == + +Goals: + + - A simple declarative language for what needs to be checked out & built, + how, in what order. + + - The same specification should work for external continuous integration + builds (e.g. Travis + Docker) and for internal VM-based continuous + integration builds. + + - One should be able to build without root, and to install to a prefix. + +Non-goals: + + - General usefulness. The only point of this is to make it easier to build + and test Facebook's open-source services. + +Ideas for the future -- these may not be very good :) + + - Especially on Ubuntu 14.04 the current initial setup is inefficient: + we add PPAs after having installed a bunch of packages -- this prompts + reinstalls of large amounts of code. We also `apt-get update` a few + times. + + - A "shell script" builder. Like DockerFBCodeBuilder, but outputs a + shell script that runs outside of a container. Or maybe even + synchronously executes the shell commands, `make`-style. + + - A "Makefile" generator. That might make iterating on builds even quicker + than what you can currently get with Docker build caching. + + - Generate a rebuild script that can be run e.g. inside the built Docker + container by tagging certain steps with list-inheriting Python objects: + * do change directories + * do NOT `git clone` -- if we want to update code this should be a + separate script that e.g. runs rebase on top of specific targets + across all the repos. + * do NOT install software (most / all setup can be skipped) + * do NOT `autoreconf` or `configure` + * do `make` and `cmake` + + - If we get non-Debian OSes, part of ccache setup should be factored out. +''' + +import os +import re + +from shell_quoting import path_join, shell_join, ShellQuoted + + +def _read_project_github_hashes(): + base_dir = 'deps/github_hashes/' # trailing slash used in regex below + for dirname, _, files in os.walk(base_dir): + for filename in files: + path = os.path.join(dirname, filename) + with open(path) as f: + m_proj = re.match('^' + base_dir + '(.*)-rev\.txt$', path) + if m_proj is None: + raise RuntimeError('Not a hash file? {0}'.format(path)) + m_hash = re.match('^Subproject commit ([0-9a-f]+)\n$', f.read()) + if m_hash is None: + raise RuntimeError('No hash in {0}'.format(path)) + yield m_proj.group(1), m_hash.group(1) + + +class FBCodeBuilder(object): + + def __init__(self, **kwargs): + self._options_do_not_access = kwargs # Use .option() instead. + # This raises upon detecting options that are specified but unused, + # because otherwise it is very easy to make a typo in option names. + self.options_used = set() + self._github_hashes = dict(_read_project_github_hashes()) + + def __repr__(self): + return '{0}({1})'.format( + self.__class__.__name__, + ', '.join( + '{0}={1}'.format(k, repr(v)) + for k, v in self._options_do_not_access.items() + ) + ) + + def option(self, name, default=None): + value = self._options_do_not_access.get(name, default) + if value is None: + raise RuntimeError('Option {0} is required'.format(name)) + self.options_used.add(name) + return value + + def has_option(self, name): + return name in self._options_do_not_access + + def add_option(self, name, value): + if name in self._options_do_not_access: + raise RuntimeError('Option {0} already set'.format(name)) + self._options_do_not_access[name] = value + + # + # Abstract parts common to every installation flow + # + + def render(self, steps): + ''' + + Converts nested actions to your builder's expected output format. + Typically takes the output of build(). + + ''' + res = self._render_impl(steps) # Implementation-dependent + # Now that the output is rendered, we expect all options to have + # been used. + unused_options = set(self._options_do_not_access) + unused_options -= self.options_used + if unused_options: + raise RuntimeError( + 'Unused options: {0} -- please check if you made a typo ' + 'in any of them. Those that are truly not useful should ' + 'be not be set so that this typo detection can be useful.' + .format(unused_options) + ) + return res + + def build(self, steps): + if not steps: + raise RuntimeError('Please ensure that the config you are passing ' + 'contains steps') + return [self.setup(), self.diagnostics()] + steps + + def setup(self): + 'Your builder may want to install packages here.' + raise NotImplementedError + + def diagnostics(self): + 'Log some system diagnostics before/after setup for ease of debugging' + # The builder's repr is not used in a command to avoid pointlessly + # invalidating Docker's build cache. + return self.step('Diagnostics', [ + self.comment('Builder {0}'.format(repr(self))), + self.run(ShellQuoted('hostname')), + self.run(ShellQuoted('cat /etc/issue || echo no /etc/issue')), + self.run(ShellQuoted('g++ --version || echo g++ not installed')), + self.run(ShellQuoted('cmake --version || echo cmake not installed')), + ]) + + def step(self, name, actions): + 'A labeled collection of actions or other steps' + raise NotImplementedError + + def run(self, shell_cmd): + 'Run this bash command' + raise NotImplementedError + + def workdir(self, dir): + 'Create this directory if it does not exist, and change into it' + raise NotImplementedError + + def copy_local_repo(self, dir, dest_name): + ''' + Copy the local repo at `dir` into this step's `workdir()`, analog of: + cp -r /path/to/folly folly + ''' + raise NotImplementedError + + def debian_deps(self): + return [ + 'autoconf-archive', + 'bison', + 'build-essential', + 'cmake', + 'curl', + 'flex', + 'git', + 'gperf', + 'joe', + 'libboost-all-dev', + 'libcap-dev', + 'libdouble-conversion-dev', + 'libevent-dev', + 'libgflags-dev', + 'libgoogle-glog-dev', + 'libkrb5-dev', + 'libpcre3-dev', + 'libpthread-stubs0-dev', + 'libnuma-dev', + 'libsasl2-dev', + 'libsnappy-dev', + 'libsqlite3-dev', + 'libssl-dev', + 'libtool', + 'netcat-openbsd', + 'pkg-config', + 'sudo', + 'unzip', + 'wget', + 'python3-venv', + ] + + # + # Specific build helpers + # + + def install_debian_deps(self): + actions = [ + self.run( + ShellQuoted('apt-get update && apt-get install -yq {deps}').format( + deps=shell_join(' ', ( + ShellQuoted(dep) for dep in self.debian_deps()))) + ), + ] + gcc_version = self.option('gcc_version') + + # Make the selected GCC the default before building anything + actions.extend([ + self.run(ShellQuoted('apt-get install -yq {c} {cpp}').format( + c=ShellQuoted('gcc-{v}').format(v=gcc_version), + cpp=ShellQuoted('g++-{v}').format(v=gcc_version), + )), + self.run(ShellQuoted( + 'update-alternatives --install /usr/bin/gcc gcc {c} 40 ' + '--slave /usr/bin/g++ g++ {cpp}' + ).format( + c=ShellQuoted('/usr/bin/gcc-{v}').format(v=gcc_version), + cpp=ShellQuoted('/usr/bin/g++-{v}').format(v=gcc_version), + )), + self.run(ShellQuoted('update-alternatives --config gcc')), + ]) + + actions.extend(self.debian_ccache_setup_steps()) + + return self.step('Install packages for Debian-based OS', actions) + + def create_python_venv(self): + action = [] + if self.option("PYTHON_VENV", "OFF") == "ON": + action = self.run(ShellQuoted("python3 -m venv {p}").format( + p=path_join(self.option('prefix'), "venv"))) + return(action) + + def python_venv(self): + action = [] + if self.option("PYTHON_VENV", "OFF") == "ON": + action = ShellQuoted("source {p}").format( + p=path_join(self.option('prefix'), "venv", "bin", "activate")) + return(action) + + def debian_ccache_setup_steps(self): + return [] # It's ok to ship a renderer without ccache support. + + def github_project_workdir(self, project, path): + # Only check out a non-default branch if requested. This especially + # makes sense when building from a local repo. + git_hash = self.option( + '{0}:git_hash'.format(project), + # Any repo that has a hash in deps/github_hashes defaults to + # that, with the goal of making builds maximally consistent. + self._github_hashes.get(project, '') + ) + maybe_change_branch = [ + self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)), + ] if git_hash else [] + + base_dir = self.option('projects_dir') + + local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '') + return self.step('Check out {0}, workdir {1}'.format(project, path), [ + self.workdir(base_dir), + self.run( + ShellQuoted('git clone https://github.com/{p}').format(p=project) + ) if not local_repo_dir else self.copy_local_repo( + local_repo_dir, os.path.basename(project) + ), + self.workdir(path_join(base_dir, os.path.basename(project), path)), + ] + maybe_change_branch) + + def fb_github_project_workdir(self, project_and_path, github_org='facebook'): + 'This helper lets Facebook-internal CI special-cases FB projects' + project, path = project_and_path.split('/', 1) + return self.github_project_workdir(github_org + '/' + project, path) + + def _make_vars(self, make_vars): + return shell_join(' ', ( + ShellQuoted('{k}={v}').format(k=k, v=v) + for k, v in ({} if make_vars is None else make_vars).items() + )) + + def parallel_make(self, make_vars=None): + return self.run(ShellQuoted('make -j {n} VERBOSE=1 {vars}').format( + n=self.option('make_parallelism'), + vars=self._make_vars(make_vars), + )) + + def make_and_install(self, make_vars=None): + return [ + self.parallel_make(make_vars), + self.run(ShellQuoted('make install VERBOSE=1 {vars}').format( + vars=self._make_vars(make_vars), + )), + ] + + def configure(self, name=None): + autoconf_options = {} + if name is not None: + autoconf_options.update( + self.option('{0}:autoconf_options'.format(name), {}) + ) + return [ + self.run(ShellQuoted( + 'LDFLAGS="$LDFLAGS -L"{p}"/lib -Wl,-rpath="{p}"/lib" ' + 'CFLAGS="$CFLAGS -I"{p}"/include" ' + 'CPPFLAGS="$CPPFLAGS -I"{p}"/include" ' + 'PY_PREFIX={p} ' + './configure --prefix={p} {args}' + ).format( + p=self.option('prefix'), + args=shell_join(' ', ( + ShellQuoted('{k}={v}').format(k=k, v=v) + for k, v in autoconf_options.items() + )), + )), + ] + + def autoconf_install(self, name): + return self.step('Build and install {0}'.format(name), [ + self.run(ShellQuoted('autoreconf -ivf')), + ] + self.configure() + self.make_and_install()) + + def cmake_configure(self, name, cmake_path='..'): + cmake_defines = { + 'BUILD_SHARED_LIBS': 'ON', + 'CMAKE_INSTALL_PREFIX': self.option('prefix'), + } + cmake_defines.update( + self.option('{0}:cmake_defines'.format(name), {}) + ) + return [ + self.run(ShellQuoted( + 'CXXFLAGS="$CXXFLAGS -fPIC -isystem "{p}"/include" ' + 'CFLAGS="$CFLAGS -fPIC -isystem "{p}"/include" ' + 'cmake {args} {cmake_path}' + ).format( + p=self.option('prefix'), + args=shell_join(' ', ( + ShellQuoted('-D{k}={v}').format(k=k, v=v) + for k, v in cmake_defines.items() + )), + cmake_path=cmake_path, + )), + ] + + def cmake_install(self, name, cmake_path='..'): + return self.step( + 'Build and install {0}'.format(name), + self.cmake_configure(name, cmake_path) + self.make_and_install() + ) + + def fb_github_autoconf_install(self, project_and_path, github_org='facebook'): + return [ + self.fb_github_project_workdir(project_and_path, github_org), + self.autoconf_install(project_and_path), + ] + + def fb_github_cmake_install(self, project_and_path, cmake_path='..', github_org='facebook'): + return [ + self.fb_github_project_workdir(project_and_path, github_org), + self.cmake_install(project_and_path, cmake_path), + ] diff --git a/build/fbcode_builder/fbcode_builder_config.py b/build/fbcode_builder/fbcode_builder_config.py new file mode 100644 index 000000000..c8f868a51 --- /dev/null +++ b/build/fbcode_builder/fbcode_builder_config.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +'Demo config, so that `make_docker_context.py --help` works in this directory.' + +config = { + 'fbcode_builder_spec': lambda _builder: { + 'depends_on': [], + 'steps': [], + }, + 'github_project': 'demo/project', +} diff --git a/build/fbcode_builder/make_docker_context.py b/build/fbcode_builder/make_docker_context.py new file mode 100755 index 000000000..30aad9e82 --- /dev/null +++ b/build/fbcode_builder/make_docker_context.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +''' +Reads `fbcode_builder_config.py` from the current directory, and prepares a +Docker context directory to build this project. Prints to stdout the path +to the context directory. + +Try `.../make_docker_context.py --help` from a project's `build/` directory. + +By default, the Docker context directory will be in /tmp. It will always +contain a Dockerfile, and might also contain copies of your local repos, and +other data needed for the build container. +''' + +import os +import tempfile +import textwrap + +from docker_builder import DockerFBCodeBuilder +from parse_args import parse_args_to_fbcode_builder_opts + + +def make_docker_context( + get_steps_fn, github_project, opts=None, default_context_dir=None +): + ''' + Returns a path to the Docker context directory. See parse_args.py. + + Helper for making a command-line utility that writes your project's + Dockerfile and associated data into a (temporary) directory. Your main + program might look something like this: + + print(make_docker_context( + lambda builder: [builder.step(...), ...], + 'facebook/your_project', + )) + ''' + + if opts is None: + opts = {} + + valid_versions = ( + ('ubuntu:16.04', '5'), + ) + + def add_args(parser): + parser.add_argument( + '--docker-context-dir', metavar='DIR', + default=default_context_dir, + help='Write the Dockerfile and its context into this directory. ' + 'If empty, make a temporary directory. Default: %(default)s.', + ) + parser.add_argument( + '--user', metavar='NAME', default=opts.get('user', 'nobody'), + help='Build and install as this user. Default: %(default)s.', + ) + parser.add_argument( + '--prefix', metavar='DIR', + default=opts.get('prefix', '/home/install'), + help='Install all libraries in this prefix. Default: %(default)s.', + ) + parser.add_argument( + '--projects-dir', metavar='DIR', + default=opts.get('projects_dir', '/home'), + help='Place project code directories here. Default: %(default)s.', + ) + parser.add_argument( + '--os-image', metavar='IMG', choices=zip(*valid_versions)[0], + default=opts.get('os_image', valid_versions[0][0]), + help='Docker OS image -- be sure to use only ones you trust (See ' + 'README.docker). Choices: %(choices)s. Default: %(default)s.', + ) + parser.add_argument( + '--gcc-version', metavar='VER', + choices=set(zip(*valid_versions)[1]), + default=opts.get('gcc_version', valid_versions[0][1]), + help='Choices: %(choices)s. Default: %(default)s.', + ) + parser.add_argument( + '--make-parallelism', metavar='NUM', type=int, + default=opts.get('make_parallelism', 1), + help='Use `make -j` on multi-CPU systems with lots of RAM. ' + 'Default: %(default)s.', + ) + parser.add_argument( + '--local-repo-dir', metavar='DIR', + help='If set, build {0} from a local directory instead of Github.' + .format(github_project), + ) + parser.add_argument( + '--ccache-tgz', metavar='PATH', + help='If set, enable ccache for the build. To initialize the ' + 'cache, first try to hardlink, then to copy --cache-tgz ' + 'as ccache.tgz into the --docker-context-dir.' + ) + + opts = parse_args_to_fbcode_builder_opts( + add_args, + # These have add_argument() calls, others are set via --option. + ( + 'docker_context_dir', + 'user', + 'prefix', + 'projects_dir', + 'os_image', + 'gcc_version', + 'make_parallelism', + 'local_repo_dir', + 'ccache_tgz', + ), + opts, + help=textwrap.dedent(''' + + Reads `fbcode_builder_config.py` from the current directory, and + prepares a Docker context directory to build {github_project} and + its dependencies. Prints to stdout the path to the context + directory. + + Pass --option {github_project}:git_hash SHA1 to build something + other than the master branch from Github. + + Or, pass --option {github_project}:local_repo_dir LOCAL_PATH to + build from a local repo instead of cloning from Github. + + Usage: + (cd $(./make_docker_context.py) && docker build . 2>&1 | tee log) + + '''.format(github_project=github_project)), + ) + + # This allows travis_docker_build.sh not to know the main Github project. + local_repo_dir = opts.pop('local_repo_dir', None) + if local_repo_dir is not None: + opts['{0}:local_repo_dir'.format(github_project)] = local_repo_dir + + if (opts.get('os_image'), opts.get('gcc_version')) not in valid_versions: + raise Exception( + 'Due to 4/5 ABI changes (std::string), we can only use {0}'.format( + ' / '.join('GCC {1} on {0}'.format(*p) for p in valid_versions) + ) + ) + + if opts.get('docker_context_dir') is None: + opts['docker_context_dir'] = tempfile.mkdtemp(prefix='docker-context-') + elif not os.path.exists(opts.get('docker_context_dir')): + os.makedirs(opts.get('docker_context_dir')) + + builder = DockerFBCodeBuilder(**opts) + context_dir = builder.option('docker_context_dir') # Mark option "in-use" + # The renderer may also populate some files into the context_dir. + dockerfile = builder.render(get_steps_fn(builder)) + + with os.fdopen(os.open( + os.path.join(context_dir, 'Dockerfile'), + os.O_RDWR | os.O_CREAT | os.O_EXCL, # Do not overwrite existing files + 0o644, + ), 'w') as f: + f.write(dockerfile) + + return context_dir + + +if __name__ == '__main__': + from utils import read_fbcode_builder_config, build_fbcode_builder_config + + # Load a spec from the current directory + config = read_fbcode_builder_config('fbcode_builder_config.py') + print(make_docker_context( + build_fbcode_builder_config(config), + config['github_project'], + )) diff --git a/build/fbcode_builder/parse_args.py b/build/fbcode_builder/parse_args.py new file mode 100644 index 000000000..def9e504d --- /dev/null +++ b/build/fbcode_builder/parse_args.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +'Argument parsing logic shared by all fbcode_builder CLI tools.' + +import argparse +import logging + +from shell_quoting import raw_shell, ShellQuoted + + +def parse_args_to_fbcode_builder_opts(add_args_fn, top_level_opts, opts, help): + ''' + + Provides some standard arguments: --debug, --option, --shell-quoted-option + + Then, calls `add_args_fn(parser)` to add application-specific arguments. + + `opts` are first used as defaults for the various command-line + arguments. Then, the parsed arguments are mapped back into `opts`, + which then become the values for `FBCodeBuilder.option()`, to be used + both by the builder and by `get_steps_fn()`. + + `help` is printed in response to the `--help` argument. + + ''' + top_level_opts = set(top_level_opts) + + parser = argparse.ArgumentParser( + description=help, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + + add_args_fn(parser) + + parser.add_argument( + '--option', nargs=2, metavar=('KEY', 'VALUE'), action='append', + default=[ + (k, v) for k, v in opts.items() + if k not in top_level_opts and not isinstance(v, ShellQuoted) + ], + help='Set project-specific options. These are assumed to be raw ' + 'strings, to be shell-escaped as needed. Default: %(default)s.', + ) + parser.add_argument( + '--shell-quoted-option', nargs=2, metavar=('KEY', 'VALUE'), + action='append', + default=[ + (k, raw_shell(v)) for k, v in opts.items() + if k not in top_level_opts and isinstance(v, ShellQuoted) + ], + help='Set project-specific options. These are assumed to be shell-' + 'quoted, and may be used in commands as-is. Default: %(default)s.', + ) + + parser.add_argument('--debug', action='store_true', help='Log more') + args = parser.parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.debug else logging.INFO, + format='%(levelname)s: %(message)s' + ) + + # Map command-line args back into opts. + logging.debug('opts before command-line arguments: {0}'.format(opts)) + + new_opts = {} + for key in top_level_opts: + val = getattr(args, key) + # Allow clients to unset a default by passing a value of None in opts + if val is not None: + new_opts[key] = val + for key, val in args.option: + new_opts[key] = val + for key, val in args.shell_quoted_option: + new_opts[key] = ShellQuoted(val) + + logging.debug('opts after command-line arguments: {0}'.format(new_opts)) + + return new_opts diff --git a/build/fbcode_builder/shell_builder.py b/build/fbcode_builder/shell_builder.py new file mode 100644 index 000000000..5bb41fe57 --- /dev/null +++ b/build/fbcode_builder/shell_builder.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +''' +shell_builder.py allows running the fbcode_builder logic +on the host rather than in a container. + +It emits a bash script with set -exo pipefail configured such that +any failing step will cause the script to exit with failure. + +== How to run it? == + +cd build +python fbcode_builder/shell_builder.py > ~/run.sh +bash ~/run.sh +''' + +import os +import distutils.spawn + +from fbcode_builder import FBCodeBuilder +from shell_quoting import ( + raw_shell, shell_comment, shell_join, ShellQuoted +) +from utils import recursively_flatten_list + + +class ShellFBCodeBuilder(FBCodeBuilder): + def _render_impl(self, steps): + return raw_shell(shell_join('\n', recursively_flatten_list(steps))) + + def workdir(self, dir): + return [ + ShellQuoted('mkdir -p {d} && cd {d}').format( + d=dir + ), + ] + + def run(self, shell_cmd): + return ShellQuoted('{cmd}').format(cmd=shell_cmd) + + def step(self, name, actions): + assert '\n' not in name, 'Name {0} would span > 1 line'.format(name) + b = ShellQuoted('') + return [ShellQuoted('### {0} ###'.format(name)), b] + actions + [b] + + def setup(self): + steps = [ + ShellQuoted('set -exo pipefail'), + ] + [self.create_python_venv(), self.python_venv()] + if self.has_option('ccache_dir'): + ccache_dir = self.option('ccache_dir') + steps += [ + ShellQuoted( + # Set CCACHE_DIR before the `ccache` invocations below. + 'export CCACHE_DIR={ccache_dir} ' + 'CC="ccache ${{CC:-gcc}}" CXX="ccache ${{CXX:-g++}}"' + ).format(ccache_dir=ccache_dir) + ] + return steps + + def comment(self, comment): + return shell_comment(comment) + + def copy_local_repo(self, dir, dest_name): + return [ + ShellQuoted('cp -r {dir} {dest_name}').format( + dir=dir, + dest_name=dest_name + ), + ] + + +def find_project_root(): + here = os.path.dirname(os.path.realpath(__file__)) + maybe_root = os.path.dirname(os.path.dirname(here)) + if os.path.isdir(os.path.join(maybe_root, '.git')): + return maybe_root + raise RuntimeError( + "I expected shell_builder.py to be in the " + "build/fbcode_builder subdir of a git repo") + + +def persistent_temp_dir(repo_root): + escaped = repo_root.replace('/', 'sZs').replace('\\', 'sZs').replace(':', '') + return os.path.join(os.path.expandvars("$HOME"), '.fbcode_builder-' + escaped) + + +if __name__ == '__main__': + from utils import read_fbcode_builder_config, build_fbcode_builder_config + repo_root = find_project_root() + temp = persistent_temp_dir(repo_root) + + config = read_fbcode_builder_config('fbcode_builder_config.py') + builder = ShellFBCodeBuilder() + + builder.add_option('projects_dir', temp) + if distutils.spawn.find_executable('ccache'): + builder.add_option('ccache_dir', + os.environ.get('CCACHE_DIR', os.path.join(temp, '.ccache'))) + builder.add_option('prefix', os.path.join(temp, 'installed')) + builder.add_option('make_parallelism', 4) + builder.add_option( + '{project}:local_repo_dir'.format(project=config['github_project']), + repo_root) + make_steps = build_fbcode_builder_config(config) + steps = make_steps(builder) + print(builder.render(steps)) diff --git a/build/fbcode_builder/shell_quoting.py b/build/fbcode_builder/shell_quoting.py new file mode 100644 index 000000000..f3b968a6d --- /dev/null +++ b/build/fbcode_builder/shell_quoting.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +''' + +Almost every FBCodeBuilder string is ultimately passed to a shell. Escaping +too little or too much tends to be the most common error. The utilities in +this file give a systematic way of avoiding such bugs: + - When you write literal strings destined for the shell, use `ShellQuoted`. + - When these literal strings are parameterized, use `ShellQuoted.format`. + - Any parameters that are raw strings get `shell_quote`d automatically, + while any ShellQuoted parameters will be left intact. + - Use `path_join` to join path components. + - Use `shell_join` to join already-quoted command arguments or shell lines. + +''' + +import os + +from collections import namedtuple + + +class ShellQuoted(namedtuple('ShellQuoted', ('do_not_use_raw_str',))): + ''' + + Wrap a string with this to make it transparent to shell_quote(). It + will almost always suffice to use ShellQuoted.format(), path_join(), + or shell_join(). + + If you really must, use raw_shell() to access the raw string. + + ''' + + def __new__(cls, s): + 'No need to nest ShellQuoted.' + return super(ShellQuoted, cls).__new__( + cls, s.do_not_use_raw_str if isinstance(s, ShellQuoted) else s + ) + + def __str__(self): + raise RuntimeError( + 'One does not simply convert {0} to a string -- use path_join() ' + 'or ShellQuoted.format() instead'.format(repr(self)) + ) + + def __repr__(self): + return '{0}({1})'.format( + self.__class__.__name__, repr(self.do_not_use_raw_str) + ) + + def format(self, **kwargs): + ''' + + Use instead of str.format() when the arguments are either + `ShellQuoted()` or raw strings needing to be `shell_quote()`d. + + Positional args are deliberately not supported since they are more + error-prone. + + ''' + return ShellQuoted(self.do_not_use_raw_str.format(**dict( + (k, shell_quote(v).do_not_use_raw_str) for k, v in kwargs.items() + ))) + + +def shell_quote(s): + 'Quotes a string if it is not already quoted' + return s if isinstance(s, ShellQuoted) \ + else ShellQuoted("'" + str(s).replace("'", "'\\''") + "'") + + +def raw_shell(s): + 'Not a member of ShellQuoted so we get a useful error for raw strings' + if isinstance(s, ShellQuoted): + return s.do_not_use_raw_str + raise RuntimeError('{0} should have been ShellQuoted'.format(s)) + + +def shell_join(delim, it): + 'Joins an iterable of ShellQuoted with a delimiter between each two' + return ShellQuoted(delim.join(raw_shell(s) for s in it)) + + +def path_join(*args): + 'Joins ShellQuoted and raw pieces of paths to make a shell-quoted path' + return ShellQuoted(os.path.join(*[ + raw_shell(shell_quote(s)) for s in args + ])) + + +def shell_comment(c): + 'Do not shell-escape raw strings in comments, but do handle line breaks.' + return ShellQuoted('# {c}').format(c=ShellQuoted( + (raw_shell(c) if isinstance(c, ShellQuoted) else c) + .replace('\n', '\n# ') + )) diff --git a/quic/AUTODEPS b/build/fbcode_builder/specs/__init__.py similarity index 100% rename from quic/AUTODEPS rename to build/fbcode_builder/specs/__init__.py diff --git a/build/fbcode_builder/specs/fbthrift.py b/build/fbcode_builder/specs/fbthrift.py new file mode 100644 index 000000000..d982fc798 --- /dev/null +++ b/build/fbcode_builder/specs/fbthrift.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.folly as folly +import specs.fizz as fizz +import specs.fmt as fmt +import specs.rsocket as rsocket +import specs.sodium as sodium +import specs.wangle as wangle +import specs.zstd as zstd + +from shell_quoting import ShellQuoted + + +def fbcode_builder_spec(builder): + # This API should change rarely, so build the latest tag instead of master. + builder.add_option( + 'no1msd/mstch:git_hash', + ShellQuoted('$(git describe --abbrev=0 --tags)') + ) + return { + 'depends_on': [folly, fizz, fmt, sodium, rsocket, wangle, zstd], + 'steps': [ + # This isn't a separete spec, since only fbthrift uses mstch. + builder.github_project_workdir('no1msd/mstch', 'build'), + builder.cmake_install('no1msd/mstch'), + builder.fb_github_cmake_install('fbthrift/thrift'), + ], + } diff --git a/build/fbcode_builder/specs/fbzmq.py b/build/fbcode_builder/specs/fbzmq.py new file mode 100644 index 000000000..588b34967 --- /dev/null +++ b/build/fbcode_builder/specs/fbzmq.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.fbthrift as fbthrift +import specs.folly as folly +import specs.gmock as gmock +import specs.sodium as sodium +import specs.sigar as sigar + +from shell_quoting import ShellQuoted + + +def fbcode_builder_spec(builder): + builder.add_option('zeromq/libzmq:git_hash', 'v4.2.2') + return { + 'depends_on': [folly, fbthrift, gmock, sodium, sigar], + 'steps': [ + builder.github_project_workdir('zeromq/libzmq', '.'), + builder.step('Build and install zeromq/libzmq', [ + builder.run(ShellQuoted('./autogen.sh')), + builder.configure(), + builder.make_and_install(), + ]), + + builder.fb_github_project_workdir('fbzmq/fbzmq/build', 'facebook'), + builder.step('Build and install fbzmq/fbzmq/build', [ + builder.cmake_configure('fbzmq/fbzmq/build'), + # we need the pythonpath to find the thrift compiler + builder.run(ShellQuoted( + 'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages ' + 'make -j {n}' + ).format(p=builder.option('prefix'), n=builder.option('make_parallelism'))), + builder.run(ShellQuoted('make install')), + ]), + ], + } diff --git a/build/fbcode_builder/specs/fizz.py b/build/fbcode_builder/specs/fizz.py new file mode 100644 index 000000000..20fc321aa --- /dev/null +++ b/build/fbcode_builder/specs/fizz.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.folly as folly +import specs.sodium as sodium + + +def fbcode_builder_spec(builder): + return { + 'depends_on': [folly, sodium], + 'steps': [ + builder.fb_github_cmake_install( + 'fizz/fizz/build', + github_org='facebookincubator', + ), + ], + } diff --git a/build/fbcode_builder/specs/fmt.py b/build/fbcode_builder/specs/fmt.py new file mode 100644 index 000000000..b68534c0d --- /dev/null +++ b/build/fbcode_builder/specs/fmt.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + + +def fbcode_builder_spec(builder): + builder.add_option('fmtlib/fmt:git_hash', '5.3.0') + return { + 'steps': [ + builder.github_project_workdir('fmtlib/fmt', 'build'), + builder.cmake_install('fmtlib/fmt'), + ], + } diff --git a/build/fbcode_builder/specs/folly.py b/build/fbcode_builder/specs/folly.py new file mode 100644 index 000000000..3d128b9c0 --- /dev/null +++ b/build/fbcode_builder/specs/folly.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + + +def fbcode_builder_spec(builder): + return { + 'steps': [ + # on macOS the filesystem is typically case insensitive. + # We need to ensure that the CWD is not the folly source + # dir when we build, otherwise the system will decide + # that `folly/String.h` is the file it wants when including + # `string.h` and the build will fail. + builder.fb_github_project_workdir('folly/_build'), + builder.cmake_install('facebook/folly'), + ], + } diff --git a/build/fbcode_builder/specs/gmock.py b/build/fbcode_builder/specs/gmock.py new file mode 100644 index 000000000..8b0562f7e --- /dev/null +++ b/build/fbcode_builder/specs/gmock.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + + +def fbcode_builder_spec(builder): + builder.add_option('google/googletest:git_hash', 'release-1.8.1') + builder.add_option( + 'google/googletest:cmake_defines', + { + 'BUILD_GTEST': 'ON', + # Avoid problems with MACOSX_RPATH + 'BUILD_SHARED_LIBS': 'OFF', + } + ) + return { + 'steps': [ + builder.github_project_workdir('google/googletest', 'build'), + builder.cmake_install('google/googletest'), + ], + } diff --git a/build/fbcode_builder/specs/proxygen.py b/build/fbcode_builder/specs/proxygen.py new file mode 100644 index 000000000..b937e425a --- /dev/null +++ b/build/fbcode_builder/specs/proxygen.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.folly as folly +import specs.fizz as fizz +import specs.sodium as sodium +import specs.wangle as wangle + + +def fbcode_builder_spec(builder): + return { + 'depends_on': [folly, wangle, fizz, sodium], + 'steps': [ + builder.fb_github_autoconf_install('proxygen/proxygen'), + ], + } diff --git a/build/fbcode_builder/specs/re2.py b/build/fbcode_builder/specs/re2.py new file mode 100644 index 000000000..b6b81ab94 --- /dev/null +++ b/build/fbcode_builder/specs/re2.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + + +def fbcode_builder_spec(builder): + return { + 'steps': [ + builder.github_project_workdir('google/re2', 'build'), + builder.cmake_install('google/re2'), + ], + } diff --git a/build/fbcode_builder/specs/rocksdb.py b/build/fbcode_builder/specs/rocksdb.py new file mode 100644 index 000000000..c7d7c6ac2 --- /dev/null +++ b/build/fbcode_builder/specs/rocksdb.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + + +def fbcode_builder_spec(builder): + builder.add_option("rocksdb/_build:cmake_defines", { + "USE_RTTI": "1", + "PORTABLE": "ON", + }) + return { + "steps": [ + builder.fb_github_cmake_install("rocksdb/_build"), + ], + } diff --git a/build/fbcode_builder/specs/rsocket.py b/build/fbcode_builder/specs/rsocket.py new file mode 100644 index 000000000..740958526 --- /dev/null +++ b/build/fbcode_builder/specs/rsocket.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.gmock as gmock +import specs.folly as folly + + +def fbcode_builder_spec(builder): + return { + 'depends_on': [folly, gmock], + 'steps': [ + builder.fb_github_cmake_install( + 'rsocket-cpp/rsocket', + github_org='rsocket'), + ], + } diff --git a/build/fbcode_builder/specs/sigar.py b/build/fbcode_builder/specs/sigar.py new file mode 100644 index 000000000..acb64e63f --- /dev/null +++ b/build/fbcode_builder/specs/sigar.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from shell_quoting import ShellQuoted + + +def fbcode_builder_spec(builder): + builder.add_option( + 'hyperic/sigar:autoconf_options', {'CFLAGS' : '-fgnu89-inline'}) + return { + 'steps': [ + builder.github_project_workdir('hyperic/sigar', '.'), + builder.step('Build and install sigar', [ + builder.run(ShellQuoted('./autogen.sh')), + builder.configure('hyperic/sigar'), + builder.make_and_install(), + ]), + ], + } diff --git a/build/fbcode_builder/specs/sodium.py b/build/fbcode_builder/specs/sodium.py new file mode 100644 index 000000000..52bb0006e --- /dev/null +++ b/build/fbcode_builder/specs/sodium.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from shell_quoting import ShellQuoted + + +def fbcode_builder_spec(builder): + builder.add_option('jedisct1/libsodium:git_hash', 'stable') + return { + 'steps': [ + builder.github_project_workdir('jedisct1/libsodium', '.'), + builder.step('Build and install jedisct1/libsodium', [ + builder.run(ShellQuoted('./autogen.sh')), + builder.configure(), + builder.make_and_install(), + ]), + ], + } diff --git a/build/fbcode_builder/specs/wangle.py b/build/fbcode_builder/specs/wangle.py new file mode 100644 index 000000000..15cf3ff46 --- /dev/null +++ b/build/fbcode_builder/specs/wangle.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.folly as folly +import specs.fizz as fizz +import specs.sodium as sodium + + +def fbcode_builder_spec(builder): + # Projects that simply depend on Wangle need not spend time on tests. + builder.add_option('wangle/wangle/build:cmake_defines', {'BUILD_TESTS': 'OFF'}) + return { + 'depends_on': [folly, fizz, sodium], + 'steps': [ + builder.fb_github_cmake_install('wangle/wangle/build'), + ], + } diff --git a/build/fbcode_builder/specs/zstd.py b/build/fbcode_builder/specs/zstd.py new file mode 100644 index 000000000..d24385dd7 --- /dev/null +++ b/build/fbcode_builder/specs/zstd.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from shell_quoting import ShellQuoted + + +def fbcode_builder_spec(builder): + # This API should change rarely, so build the latest tag instead of master. + builder.add_option( + 'facebook/zstd:git_hash', + ShellQuoted('$(git describe --abbrev=0 --tags origin/master)') + ) + return { + 'steps': [ + builder.github_project_workdir('facebook/zstd', '.'), + builder.step('Build and install zstd', [ + builder.make_and_install(make_vars={ + 'PREFIX': builder.option('prefix'), + }) + ]), + ], + } diff --git a/build/fbcode_builder/travis_docker_build.sh b/build/fbcode_builder/travis_docker_build.sh new file mode 100755 index 000000000..9733bb206 --- /dev/null +++ b/build/fbcode_builder/travis_docker_build.sh @@ -0,0 +1,42 @@ +#!/bin/bash -uex +# Copyright (c) Facebook, Inc. and its affiliates. +# .travis.yml in the top-level dir explains why this is a separate script. +# Read the docs: ./make_docker_context.py --help + +os_image=${os_image?Must be set by Travis} +gcc_version=${gcc_version?Must be set by Travis} +make_parallelism=${make_parallelism:-4} +# ccache is off unless requested +travis_cache_dir=${travis_cache_dir:-} +# The docker build never times out, unless specified +docker_build_timeout=${docker_build_timeout:-} + +cur_dir="$(readlink -f "$(dirname "$0")")" + +if [[ "$travis_cache_dir" == "" ]]; then + echo "ccache disabled, enable by setting env. var. travis_cache_dir" + ccache_tgz="" +elif [[ -e "$travis_cache_dir/ccache.tgz" ]]; then + ccache_tgz="$travis_cache_dir/ccache.tgz" +else + echo "$travis_cache_dir/ccache.tgz does not exist, starting with empty cache" + ccache_tgz=$(mktemp) + tar -T /dev/null -czf "$ccache_tgz" +fi + +docker_context_dir=$( + cd "$cur_dir/.." # Let the script find our fbcode_builder_config.py + "$cur_dir/make_docker_context.py" \ + --os-image "$os_image" \ + --gcc-version "$gcc_version" \ + --make-parallelism "$make_parallelism" \ + --local-repo-dir "$cur_dir/../.." \ + --ccache-tgz "$ccache_tgz" +) +cd "${docker_context_dir?Failed to make Docker context directory}" + +# Make it safe to iterate on the .sh in the tree while the script runs. +cp "$cur_dir/docker_build_with_ccache.sh" . +exec ./docker_build_with_ccache.sh \ + --build-timeout "$docker_build_timeout" \ + "$travis_cache_dir" diff --git a/build/fbcode_builder/utils.py b/build/fbcode_builder/utils.py new file mode 100644 index 000000000..bdf7b01d5 --- /dev/null +++ b/build/fbcode_builder/utils.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +'Miscellaneous utility functions.' + +import itertools +import logging +import os +import shutil +import subprocess +import sys + +from contextlib import contextmanager + + +def recursively_flatten_list(l): + return itertools.chain.from_iterable( + (recursively_flatten_list(i) if type(i) is list else (i,)) + for i in l + ) + + +def run_command(*cmd, **kwargs): + 'The stdout of most fbcode_builder utilities is meant to be parsed.' + logging.debug('Running: {0} with {1}'.format(cmd, kwargs)) + kwargs['stdout'] = sys.stderr + subprocess.check_call(cmd, **kwargs) + + +@contextmanager +def make_temp_dir(d): + os.mkdir(d) + try: + yield d + finally: + shutil.rmtree(d, ignore_errors=True) + + +def _inner_read_config(path): + ''' + Helper to read a named config file. + The grossness with the global is a workaround for this python bug: + https://bugs.python.org/issue21591 + The bug prevents us from defining either a local function or a lambda + in the scope of read_fbcode_builder_config below. + ''' + global _project_dir + full_path = os.path.join(_project_dir, path) + return read_fbcode_builder_config(full_path) + + +def read_fbcode_builder_config(filename): + # Allow one spec to read another + # When doing so, treat paths as relative to the config's project directory. + # _project_dir is a "local" for _inner_read_config; see the comments + # in that function for an explanation of the use of global. + global _project_dir + _project_dir = os.path.dirname(filename) + + scope = {'read_fbcode_builder_config': _inner_read_config} + with open(filename) as config_file: + code = compile(config_file.read(), filename, mode='exec') + exec(code, scope) + return scope['config'] + + +def steps_for_spec(builder, spec, processed_modules=None): + ''' + Sets `builder` configuration, and returns all the builder steps + necessary to build `spec` and its dependencies. + + Traverses the dependencies in depth-first order, honoring the sequencing + in each 'depends_on' list. + ''' + if processed_modules is None: + processed_modules = set() + steps = [] + for module in spec.get('depends_on', []): + if module not in processed_modules: + processed_modules.add(module) + steps.extend(steps_for_spec( + builder, + module.fbcode_builder_spec(builder), + processed_modules + )) + steps.extend(spec.get('steps', [])) + return steps + + +def build_fbcode_builder_config(config): + return lambda builder: builder.build( + steps_for_spec(builder, config['fbcode_builder_spec'](builder)) + ) diff --git a/cmake/FindSodium.cmake b/cmake/FindSodium.cmake new file mode 100644 index 000000000..c664ccbe3 --- /dev/null +++ b/cmake/FindSodium.cmake @@ -0,0 +1,294 @@ +# Written in 2016 by Henrik Steffen Gaßmann +# +# To the extent possible under law, the author(s) have dedicated all +# copyright and related and neighboring rights to this software to the +# public domain worldwide. This software is distributed without any warranty. +# +# You should have received a copy of the CC0 Public Domain Dedication +# along with this software. If not, see +# +# http://creativecommons.org/publicdomain/zero/1.0/ +# +######################################################################## +# Tries to find the local libsodium installation. +# +# On Windows the sodium_DIR environment variable is used as a default +# hint which can be overridden by setting the corresponding cmake variable. +# +# Once done the following variables will be defined: +# +# sodium_FOUND +# sodium_INCLUDE_DIR +# sodium_LIBRARY_DEBUG +# sodium_LIBRARY_RELEASE +# +# +# Furthermore an imported "sodium" target is created. +# + +if (CMAKE_C_COMPILER_ID STREQUAL "GNU" + OR CMAKE_C_COMPILER_ID STREQUAL "Clang") + set(_GCC_COMPATIBLE 1) +endif() + +# static library option +if (NOT DEFINED sodium_USE_STATIC_LIBS) + option(sodium_USE_STATIC_LIBS "enable to statically link against sodium" OFF) +endif() +if(NOT (sodium_USE_STATIC_LIBS EQUAL sodium_USE_STATIC_LIBS_LAST)) + unset(sodium_LIBRARY CACHE) + unset(sodium_LIBRARY_DEBUG CACHE) + unset(sodium_LIBRARY_RELEASE CACHE) + unset(sodium_DLL_DEBUG CACHE) + unset(sodium_DLL_RELEASE CACHE) + set(sodium_USE_STATIC_LIBS_LAST ${sodium_USE_STATIC_LIBS} CACHE INTERNAL "internal change tracking variable") +endif() + + +######################################################################## +# UNIX +if (UNIX) + # import pkg-config + find_package(PkgConfig QUIET) + if (PKG_CONFIG_FOUND) + pkg_check_modules(sodium_PKG QUIET libsodium) + endif() + + if(sodium_USE_STATIC_LIBS) + foreach(_libname ${sodium_PKG_STATIC_LIBRARIES}) + if (NOT _libname MATCHES "^lib.*\\.a$") # ignore strings already ending with .a + list(INSERT sodium_PKG_STATIC_LIBRARIES 0 "lib${_libname}.a") + endif() + endforeach() + list(REMOVE_DUPLICATES sodium_PKG_STATIC_LIBRARIES) + + # if pkgconfig for libsodium doesn't provide + # static lib info, then override PKG_STATIC here.. + if (NOT sodium_PKG_STATIC_FOUND) + set(sodium_PKG_STATIC_LIBRARIES libsodium.a) + endif() + + set(XPREFIX sodium_PKG_STATIC) + else() + if (NOT sodium_PKG_FOUND) + set(sodium_PKG_LIBRARIES sodium) + endif() + + set(XPREFIX sodium_PKG) + endif() + + find_path(sodium_INCLUDE_DIR sodium.h + HINTS ${${XPREFIX}_INCLUDE_DIRS} + ) + find_library(sodium_LIBRARY_DEBUG NAMES ${${XPREFIX}_LIBRARIES} + HINTS ${${XPREFIX}_LIBRARY_DIRS} + ) + find_library(sodium_LIBRARY_RELEASE NAMES ${${XPREFIX}_LIBRARIES} + HINTS ${${XPREFIX}_LIBRARY_DIRS} + ) + + +######################################################################## +# Windows +elseif (WIN32) + set(sodium_DIR "$ENV{sodium_DIR}" CACHE FILEPATH "sodium install directory") + mark_as_advanced(sodium_DIR) + + find_path(sodium_INCLUDE_DIR sodium.h + HINTS ${sodium_DIR} + PATH_SUFFIXES include + ) + + if (MSVC) + # detect target architecture + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/arch.cpp" [=[ + #if defined _M_IX86 + #error ARCH_VALUE x86_32 + #elif defined _M_X64 + #error ARCH_VALUE x86_64 + #endif + #error ARCH_VALUE unknown + ]=]) + try_compile(_UNUSED_VAR "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/arch.cpp" + OUTPUT_VARIABLE _COMPILATION_LOG + ) + string(REGEX REPLACE ".*ARCH_VALUE ([a-zA-Z0-9_]+).*" "\\1" _TARGET_ARCH "${_COMPILATION_LOG}") + + # construct library path + if (_TARGET_ARCH STREQUAL "x86_32") + string(APPEND _PLATFORM_PATH "Win32") + elseif(_TARGET_ARCH STREQUAL "x86_64") + string(APPEND _PLATFORM_PATH "x64") + else() + message(FATAL_ERROR "the ${_TARGET_ARCH} architecture is not supported by Findsodium.cmake.") + endif() + string(APPEND _PLATFORM_PATH "/$$CONFIG$$") + + if (MSVC_VERSION LESS 1900) + math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 60") + else() + math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 50") + endif() + string(APPEND _PLATFORM_PATH "/v${_VS_VERSION}") + + if (sodium_USE_STATIC_LIBS) + string(APPEND _PLATFORM_PATH "/static") + else() + string(APPEND _PLATFORM_PATH "/dynamic") + endif() + + string(REPLACE "$$CONFIG$$" "Debug" _DEBUG_PATH_SUFFIX "${_PLATFORM_PATH}") + string(REPLACE "$$CONFIG$$" "Release" _RELEASE_PATH_SUFFIX "${_PLATFORM_PATH}") + + find_library(sodium_LIBRARY_DEBUG libsodium.lib + HINTS ${sodium_DIR} + PATH_SUFFIXES ${_DEBUG_PATH_SUFFIX} + ) + find_library(sodium_LIBRARY_RELEASE libsodium.lib + HINTS ${sodium_DIR} + PATH_SUFFIXES ${_RELEASE_PATH_SUFFIX} + ) + if (NOT sodium_USE_STATIC_LIBS) + set(CMAKE_FIND_LIBRARY_SUFFIXES_BCK ${CMAKE_FIND_LIBRARY_SUFFIXES}) + set(CMAKE_FIND_LIBRARY_SUFFIXES ".dll") + find_library(sodium_DLL_DEBUG libsodium + HINTS ${sodium_DIR} + PATH_SUFFIXES ${_DEBUG_PATH_SUFFIX} + ) + find_library(sodium_DLL_RELEASE libsodium + HINTS ${sodium_DIR} + PATH_SUFFIXES ${_RELEASE_PATH_SUFFIX} + ) + set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_BCK}) + endif() + + elseif(_GCC_COMPATIBLE) + if (sodium_USE_STATIC_LIBS) + find_library(sodium_LIBRARY_DEBUG libsodium.a + HINTS ${sodium_DIR} + PATH_SUFFIXES lib + ) + find_library(sodium_LIBRARY_RELEASE libsodium.a + HINTS ${sodium_DIR} + PATH_SUFFIXES lib + ) + else() + find_library(sodium_LIBRARY_DEBUG libsodium.dll.a + HINTS ${sodium_DIR} + PATH_SUFFIXES lib + ) + find_library(sodium_LIBRARY_RELEASE libsodium.dll.a + HINTS ${sodium_DIR} + PATH_SUFFIXES lib + ) + + file(GLOB _DLL + LIST_DIRECTORIES false + RELATIVE "${sodium_DIR}/bin" + "${sodium_DIR}/bin/libsodium*.dll" + ) + find_library(sodium_DLL_DEBUG ${_DLL} libsodium + HINTS ${sodium_DIR} + PATH_SUFFIXES bin + ) + find_library(sodium_DLL_RELEASE ${_DLL} libsodium + HINTS ${sodium_DIR} + PATH_SUFFIXES bin + ) + endif() + else() + message(FATAL_ERROR "this platform is not supported by FindSodium.cmake") + endif() + + +######################################################################## +# unsupported +else() + message(FATAL_ERROR "this platform is not supported by FindSodium.cmake") +endif() + + +######################################################################## +# common stuff + +# extract sodium version +if (sodium_INCLUDE_DIR) + set(_VERSION_HEADER "${_INCLUDE_DIR}/sodium/version.h") + if (EXISTS _VERSION_HEADER) + file(READ "${_VERSION_HEADER}" _VERSION_HEADER_CONTENT) + string(REGEX REPLACE ".*#[ \t]*define[ \t]*SODIUM_VERSION_STRING[ \t]*\"([^\n]*)\".*" "\\1" + sodium_VERSION "${_VERSION_HEADER_CONTENT}") + set(sodium_VERSION "${sodium_VERSION}" PARENT_SCOPE) + endif() +endif() + +# communicate results +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + Sodium # The name must be either uppercase or match the filename case. + REQUIRED_VARS + sodium_LIBRARY_RELEASE + sodium_LIBRARY_DEBUG + sodium_INCLUDE_DIR + VERSION_VAR + sodium_VERSION +) + +if(Sodium_FOUND) + set(sodium_LIBRARIES + optimized ${sodium_LIBRARY_RELEASE} debug ${sodium_LIBRARY_DEBUG}) +endif() + +# mark file paths as advanced +mark_as_advanced(sodium_INCLUDE_DIR) +mark_as_advanced(sodium_LIBRARY_DEBUG) +mark_as_advanced(sodium_LIBRARY_RELEASE) +if (WIN32) + mark_as_advanced(sodium_DLL_DEBUG) + mark_as_advanced(sodium_DLL_RELEASE) +endif() + +# create imported target +if(sodium_USE_STATIC_LIBS) + set(_LIB_TYPE STATIC) +else() + set(_LIB_TYPE SHARED) +endif() +add_library(sodium ${_LIB_TYPE} IMPORTED) + +set_target_properties(sodium PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${sodium_INCLUDE_DIR}" + IMPORTED_LINK_INTERFACE_LANGUAGES "C" +) + +if (sodium_USE_STATIC_LIBS) + set_target_properties(sodium PROPERTIES + INTERFACE_COMPILE_DEFINITIONS "SODIUM_STATIC" + IMPORTED_LOCATION "${sodium_LIBRARY_RELEASE}" + IMPORTED_LOCATION_DEBUG "${sodium_LIBRARY_DEBUG}" + ) +else() + if (UNIX) + set_target_properties(sodium PROPERTIES + IMPORTED_LOCATION "${sodium_LIBRARY_RELEASE}" + IMPORTED_LOCATION_DEBUG "${sodium_LIBRARY_DEBUG}" + ) + elseif (WIN32) + set_target_properties(sodium PROPERTIES + IMPORTED_IMPLIB "${sodium_LIBRARY_RELEASE}" + IMPORTED_IMPLIB_DEBUG "${sodium_LIBRARY_DEBUG}" + ) + if (NOT (sodium_DLL_DEBUG MATCHES ".*-NOTFOUND")) + set_target_properties(sodium PROPERTIES + IMPORTED_LOCATION_DEBUG "${sodium_DLL_DEBUG}" + ) + endif() + if (NOT (sodium_DLL_RELEASE MATCHES ".*-NOTFOUND")) + set_target_properties(sodium PROPERTIES + IMPORTED_LOCATION_RELWITHDEBINFO "${sodium_DLL_RELEASE}" + IMPORTED_LOCATION_MINSIZEREL "${sodium_DLL_RELEASE}" + IMPORTED_LOCATION_RELEASE "${sodium_DLL_RELEASE}" + ) + endif() + endif() +endif() diff --git a/quic/BUCK b/quic/BUCK deleted file mode 100644 index 58f67618f..000000000 --- a/quic/BUCK +++ /dev/null @@ -1,31 +0,0 @@ -fb_xplat_cxx_library( - name = "quic", - srcs = glob([ - "api/*.cpp", - "client/*.cpp", - "codec/*.cpp", - "common/*.cpp", - "congestion_control/*.cpp", - "flowcontrol/*.cpp", - "happyeyeballs/*.cpp", - "logging/*.cpp", - "loss/*.cpp", - "state/*.cpp", - ]), - header_namespace = "", - exported_headers = subdir_glob( - [ - ("", "api/QuicSocket.h"), - ("", "client/QuicClientTransport.h"), - ("", "client/handshake/QuicPskCache.h"), - ], - prefix = "quic", - ), - apple_sdks = (IOS, MACOSX, APPLETVOS), - visibility = ["PUBLIC"], - deps = [ - "fbsource//xplat/folly:extended", - "fbsource//xplat/folly:molly", - "fbsource//xplat/third-party/boost:boost", - ], -) diff --git a/quic/BUILD_MODE.bzl b/quic/BUILD_MODE.bzl deleted file mode 100644 index bfde526ca..000000000 --- a/quic/BUILD_MODE.bzl +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2017 Facebook - -""" build mode definitions for quic """ - -load("@fbcode_macros//build_defs:create_build_mode.bzl", "create_build_mode") - -_extra_cflags = [ -] - -_common_flags = [ - "-Wformat", - "-Wformat-security", - "-Wunused-variable", - "-Wsign-compare", -] - -_extra_clang_flags = _common_flags + [ - # Default value for clang (3.4) is 256, change it to GCC's default value - # (https://fburl.com/23278774). - "-ftemplate-depth=900", - "-Wmismatched-tags", - # Only check shadowing with Clang: gcc complains about constructor - # argument shadowing - "-Wshadow", -] - -_extra_gcc_flags = _common_flags + [ - "-Wall", -] - -_mode = create_build_mode( - c_flags = _extra_cflags, - clang_flags = _extra_clang_flags, - gcc_flags = _extra_gcc_flags, -) - -_modes = { - "dbg": _mode, - "dbgo": _mode, - "dev": _mode, - "opt": _mode, -} - -def get_modes(): - """ Return modes for this file """ - return _modes diff --git a/quic/QuicConstants.h b/quic/QuicConstants.h index 5744f060d..56bf2c04a 100644 --- a/quic/QuicConstants.h +++ b/quic/QuicConstants.h @@ -108,46 +108,26 @@ enum class TransportErrorCode : uint16_t { TLS_FATAL_ALERT_RECEIVED = 0x203, }; -enum class ApplicationErrorCode : uint16_t { - STOPPING = 0x00, - // HTTP2/QUIC error codes - HTTP_NO_ERROR = 0x01, - HTTP_PUSH_REFUSED = 0x02, - HTTP_INTERNAL_ERROR = 0x03, - HTTP_PUSH_ALREADY_IN_CACHE = 0x04, - HTTP_REQUEST_CANCELLED = 0x05, - HTTP_INCOMPLETE_REQUEST = 0x06, - HTTP_CONNECT_ERROR = 0x07, - HTTP_EXCESSIVE_LOAD = 0x08, - HTTP_VERSION_FALLBACK = 0x09, - HTTP_WRONG_STREAM = 0x0A, - HTTP_PUSH_LIMIT_EXCEEDED = 0x0B, - HTTP_DUPLICATE_PUSH = 0x0C, - HTTP_UNKNOWN_STREAM_TYPE = 0x0D, - HTTP_WRONG_STREAM_COUNT = 0x0E, - HTTP_CLOSED_CRITICAL_STREAM = 0x0F, - HTTP_WRONG_STREAM_DIRECTION = 0x10, - HTTP_EARLY_RESPONSE = 0x11, - HTTP_MISSING_SETTINGS = 0x12, - HTTP_UNEXPECTED_FRAME = 0x13, - HTTP_REQUEST_REJECTED = 0x14, - HTTP_QPACK_DECOMPRESSION_FAILED = 0xE0, - HTTP_QPACK_DECODER_STREAM_ERROR = 0xE1, - HTTP_QPACK_ENCODER_STREAM_ERROR = 0xE2, - HTTP_GENERAL_PROTOCOL_ERROR = 0xFF, - HTTP_MALFORMED_FRAME_DATA = 0x0100, - HTTP_MALFORMED_FRAME_HEADERS = 0x0101, - HTTP_MALFORMED_FRAME_PRIORITY = 0x0102, - HTTP_MALFORMED_FRAME_CANCEL_PUSH = 0x0103, - HTTP_MALFORMED_FRAME_SETTINGS = 0x0104, - HTTP_MALFORMED_FRAME_PUSH_PROMISE = 0x0105, - HTTP_MALFORMED_FRAME_GOAWAY = 0x0107, - HTTP_MALFORMED_FRAME_MAX_PUSH_ID = 0x010D, - HTTP_MALFORMED_FRAME = 0x01FF, - // Internal use only - INTERNAL_ERROR = 0xF1, - GIVEUP_ZERO_RTT = 0xF2 +/** + * Application error codes are opaque to QUIC transport. Each application + * protocol can define its own error codes. + */ +using ApplicationErrorCode = uint16_t; + +/** + * Example application error codes, or codes that can be used by very simple + * applications. Note: by convention error code 0 means no error. + * + * It is convenient to use not strongly typed enums so they are implicitly + * castable to ints, but to get the scoping semantics we enclose it in a + * namespace of the same name. + */ +namespace GenericApplicationErrorCode { +enum GenericApplicationErrorCode : uint16_t { + NO_ERROR = 0x0000, + UNKNOWN = 0xFFFF }; +} enum class LocalErrorCode : uint32_t { // Local errors @@ -225,15 +205,12 @@ constexpr int kRttBeta = 4; // recommendation. This is not a bug. constexpr std::chrono::microseconds kDefaultInitialRtt = std::chrono::microseconds(50 * 1000); -constexpr std::chrono::microseconds kMinTLPTimeout = - std::chrono::microseconds(10 * 1000); // HHWheelTimer tick interval constexpr std::chrono::microseconds kGranularity = std::chrono::microseconds(10 * 1000); constexpr uint32_t kReorderingThreshold = 3; -constexpr double kTimeReorderingFraction = 0.125; constexpr auto kPacketToSendForRTO = 2; @@ -248,6 +225,9 @@ constexpr uint64_t kDefaultMaxBurstPackets = 10; constexpr std::chrono::microseconds kDefaultPacingTimerTickInterval{1000}; // Congestion control: +constexpr std::chrono::microseconds::rep kPersistentCongestionThreshold = 3; +constexpr std::chrono::microseconds::rep kPersistentCongestionPeriodFactor = + (((std::chrono::microseconds::rep)1 << kPersistentCongestionThreshold) - 1); enum class CongestionControlType : uint8_t { Cubic, NewReno, Copa, BBR, None }; constexpr uint64_t kInitCwndInMss = 10; diff --git a/quic/QuicException.cpp b/quic/QuicException.cpp index 6e32b17a3..014543bf1 100644 --- a/quic/QuicException.cpp +++ b/quic/QuicException.cpp @@ -151,89 +151,12 @@ std::string toString(TransportErrorCode code) { return "Unknown error"; } -std::string toString(ApplicationErrorCode code) { - switch (code) { - case ApplicationErrorCode::STOPPING: - return "Stopping"; - case ApplicationErrorCode::HTTP_NO_ERROR: - return "HTTP: No error"; - case ApplicationErrorCode::HTTP_PUSH_REFUSED: - return "HTTP: Client refused pushed content"; - case ApplicationErrorCode::HTTP_INTERNAL_ERROR: - return "HTTP: Internal error"; - case ApplicationErrorCode::HTTP_PUSH_ALREADY_IN_CACHE: - return "HTTP: Pushed content already cached"; - case ApplicationErrorCode::HTTP_REQUEST_CANCELLED: - return "HTTP: Data no longer needed"; - case ApplicationErrorCode::HTTP_INCOMPLETE_REQUEST: - return "HTTP: Stream terminated early"; - case ApplicationErrorCode::HTTP_CONNECT_ERROR: - return "HTTP: Reset or error on CONNECT request"; - case ApplicationErrorCode::HTTP_EXCESSIVE_LOAD: - return "HTTP: Peer generating excessive load"; - case ApplicationErrorCode::HTTP_VERSION_FALLBACK: - return "HTTP: Retry over HTTP/1.1"; - case ApplicationErrorCode::HTTP_WRONG_STREAM: - return "HTTP: A frame was sent on the wrong stream"; - case ApplicationErrorCode::HTTP_PUSH_LIMIT_EXCEEDED: - return "HTTP: Maximum Push ID exceeded"; - case ApplicationErrorCode::HTTP_DUPLICATE_PUSH: - return "HTTP: Push ID was fulfilled multiple times"; - case ApplicationErrorCode::HTTP_UNKNOWN_STREAM_TYPE: - return "HTTP: Unknown unidirectional stream type"; - case ApplicationErrorCode::HTTP_WRONG_STREAM_COUNT: - return "HTTP: Too many unidirectional streams"; - case ApplicationErrorCode::HTTP_CLOSED_CRITICAL_STREAM: - return "HTTP: Critical stream was closed"; - case ApplicationErrorCode::HTTP_WRONG_STREAM_DIRECTION: - return "HTTP: Unidirectional stream in wrong direction"; - case ApplicationErrorCode::HTTP_EARLY_RESPONSE: - return "HTTP: Remainder of request not needed"; - case ApplicationErrorCode::HTTP_MISSING_SETTINGS: - return "HTTP: No SETTINGS frame received"; - case ApplicationErrorCode::HTTP_UNEXPECTED_FRAME: - return "HTTP: Unexpected frame from client"; - case ApplicationErrorCode::HTTP_REQUEST_REJECTED: - return "HTTP: Server did not process request"; - case ApplicationErrorCode::HTTP_QPACK_DECOMPRESSION_FAILED: - return "HTTP: QPACK decompression failed"; - case ApplicationErrorCode::HTTP_QPACK_DECODER_STREAM_ERROR: - return "HTTP: Error on QPACK decoder stream"; - case ApplicationErrorCode::HTTP_QPACK_ENCODER_STREAM_ERROR: - return "HTTP: Error on QPACK encoder stream"; - case ApplicationErrorCode::HTTP_GENERAL_PROTOCOL_ERROR: - return "HTTP: General protocol error"; - case ApplicationErrorCode::HTTP_MALFORMED_FRAME_DATA: - return "HTTP: Malformed DATA frame"; - case ApplicationErrorCode::HTTP_MALFORMED_FRAME_HEADERS: - return "HTTP: Malformed HEADERS frame"; - case ApplicationErrorCode::HTTP_MALFORMED_FRAME_PRIORITY: - return "HTTP: Malformed PRIORITY frame"; - case ApplicationErrorCode::HTTP_MALFORMED_FRAME_CANCEL_PUSH: - return "HTTP: Malformed CANCEL_PUSH frame"; - case ApplicationErrorCode::HTTP_MALFORMED_FRAME_SETTINGS: - return "HTTP: Malformed SETTINGS frame"; - case ApplicationErrorCode::HTTP_MALFORMED_FRAME_PUSH_PROMISE: - return "HTTP: Malformed PUSH_PROMISE frame"; - case ApplicationErrorCode::HTTP_MALFORMED_FRAME_GOAWAY: - return "HTTP: Malformed GOAWAY frame"; - case ApplicationErrorCode::HTTP_MALFORMED_FRAME_MAX_PUSH_ID: - return "HTTP: Malformed MAX_PUSH_ID frame"; - case ApplicationErrorCode::HTTP_MALFORMED_FRAME: - return "HTTP: Malformed frame"; - case ApplicationErrorCode::INTERNAL_ERROR: - return "Internal error"; - case ApplicationErrorCode::GIVEUP_ZERO_RTT: - return "Give up Zero RTT"; - } - LOG(WARNING) << "toString has unhandled ErrorCode"; - return "Unknown error"; -} - std::string toString(QuicErrorCode code) { return folly::variant_match( code, - [](ApplicationErrorCode errorCode) { return toString(errorCode); }, + [](ApplicationErrorCode errorCode) { + return folly::to(errorCode); + }, [](LocalErrorCode errorCode) { return toString(errorCode); }, [](TransportErrorCode errorCode) { return toString(errorCode); }); } diff --git a/quic/QuicException.h b/quic/QuicException.h index b9cb646c2..9c2f8c524 100644 --- a/quic/QuicException.h +++ b/quic/QuicException.h @@ -83,7 +83,6 @@ class QuicApplicationException : public std::runtime_error { */ std::string toString(TransportErrorCode code); std::string toString(LocalErrorCode code); -std::string toString(ApplicationErrorCode code); std::string toString(QuicErrorCode code); std::string toString( const std::pair>& error); diff --git a/quic/TARGETS b/quic/TARGETS deleted file mode 100644 index d369f3ef6..000000000 --- a/quic/TARGETS +++ /dev/null @@ -1,37 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") - -cpp_library( - name = "constants", - srcs = [ - "QuicConstants.cpp", - ], - headers = [ - "QuicConstants.h", - ], - deps = [ - "//folly:range", - "//folly:string", - ], - external_deps = [ - "boost", - ], -) - -cpp_library( - name = "exception", - srcs = [ - "QuicException.cpp", - ], - headers = [ - "QuicException.h", - ], - deps = [ - ":constants", - "//folly:overload", - ], - external_deps = [ - "glog", - ], -) diff --git a/quic/api/QuicStreamAsyncTransport.h b/quic/api/QuicStreamAsyncTransport.h index a5f86483d..0932f2bb3 100644 --- a/quic/api/QuicStreamAsyncTransport.h +++ b/quic/api/QuicStreamAsyncTransport.h @@ -144,7 +144,7 @@ class QuicStreamAsyncTransport : public folly::AsyncTransportWrapper, if (writeBuf_.empty()) { shutdownWrite(); } else { - sock_->resetStream(id_, quic::ApplicationErrorCode::STOPPING); + sock_->resetStream(id_, quic::GenericApplicationErrorCode::UNKNOWN); VLOG(4) << "Reset stream from shutdownWriteNow"; } } diff --git a/quic/api/QuicTransportBase.cpp b/quic/api/QuicTransportBase.cpp index 8de7e4d52..b2dce5b1f 100644 --- a/quic/api/QuicTransportBase.cpp +++ b/quic/api/QuicTransportBase.cpp @@ -590,7 +590,7 @@ QuicTransportBase::setReadCallbackInternal( } else { readCb = cb; if (readCb == nullptr) { - return stopSending(id, ApplicationErrorCode::HTTP_NO_ERROR); + return stopSending(id, GenericApplicationErrorCode::NO_ERROR); } } updateReadLooper(); @@ -1885,7 +1885,9 @@ void QuicTransportBase::scheduleLossTimeout(std::chrono::milliseconds timeout) { if (closeState_ == CloseState::CLOSED) { return; } - getEventBase()->timer().scheduleTimeout(&lossTimeout_, timeout); + auto& wheelTimer = getEventBase()->timer(); + timeout = timeMax(timeout, wheelTimer.getTickInterval()); + wheelTimer.scheduleTimeout(&lossTimeout_, timeout); } void QuicTransportBase::scheduleAckTimeout() { diff --git a/quic/api/QuicTransportFunctions.cpp b/quic/api/QuicTransportFunctions.cpp index 4bf5f2a5e..7f48690d5 100644 --- a/quic/api/QuicTransportFunctions.cpp +++ b/quic/api/QuicTransportFunctions.cpp @@ -357,6 +357,8 @@ void updateConnection( } if (pureAck) { ++conn.outstandingPureAckPacketsCount; + } else { + conn.lossState.lastRetransmittablePacketSentTime = pkt.time; } if (pkt.associatedEvent) { CHECK_EQ(packetNumberSpace, PacketNumberSpace::AppData); diff --git a/quic/api/TARGETS b/quic/api/TARGETS deleted file mode 100644 index a9bd5f5c8..000000000 --- a/quic/api/TARGETS +++ /dev/null @@ -1,71 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") - -cpp_library( - name = "transport", - srcs = [ - "IoBufQuicBatch.cpp", - "QuicBatchWriter.cpp", - "QuicPacketScheduler.cpp", - "QuicTransportBase.cpp", - "QuicTransportFunctions.cpp", - ], - headers = [ - "IoBufQuicBatch.h", - "QuicBatchWriter.h", - "QuicPacketScheduler.h", - "QuicPacketScheduler-inl.h", - "QuicSocket.h", - "QuicTransportBase.h", - "QuicTransportFunctions.h", - ], - deps = [ - "//folly:exception_wrapper", - "//folly:expected", - "//folly:optional", - "//folly:overload", - "//folly:scope_guard", - "//folly/io:iobuf", - "//folly/io/async:async_base", - "//folly/io/async:async_udp_socket", - "//quic:constants", - "//quic:exception", - "//quic/codec:codec", - "//quic/codec:pktbuilder", - "//quic/codec:pktrebuilder", - "//quic/codec:types", - "//quic/common:looper", - "//quic/common:time_util", - "//quic/common:timers", - "//quic/congestion_control:congestion_controller_factory", - "//quic/congestion_control:copa", - "//quic/congestion_control:cubic", - "//quic/congestion_control:newreno", - "//quic/flowcontrol:flow_control", - "//quic/happyeyeballs:happyeyeballs", - "//quic/logging:logging", - "//quic/loss:loss", - "//quic/state:pacing_functions", - "//quic/state:simple_frame_functions", - "//quic/state:state_functions", - "//quic/state:state_machine", - "//quic/state:stream_functions", - "//quic/state/stream:stream", - ], - external_deps = [ - "boost", - ], -) - -cpp_library( - name = "stream_transport", - headers = [ - "QuicStreamAsyncTransport.h", - ], - deps = [ - ":transport", - "//folly/io:iobuf", - "//folly/io/async:async_transport", - ], -) diff --git a/quic/api/test/QuicTransportBaseTest.cpp b/quic/api/test/QuicTransportBaseTest.cpp index 0c3bb4417..fedd74cdc 100644 --- a/quic/api/test/QuicTransportBaseTest.cpp +++ b/quic/api/test/QuicTransportBaseTest.cpp @@ -151,6 +151,10 @@ class TestQuicTransport false); } + std::chrono::milliseconds getLossTimeoutRemainingTime() const { + return lossTimeout_.getTimeRemaining(); + } + void onReadData(const folly::SocketAddress&, NetworkData&& data) { if (!data.data) { return; @@ -977,6 +981,15 @@ TEST_F(QuicTransportImplTest, ReadDataAlsoChecksLossAlarm) { transport.reset(); } +TEST_F(QuicTransportImplTest, LossTimeoutNoLessThanTickInterval) { + auto tickInterval = evb->timer().getTickInterval(); + transport->scheduleLossTimeout(tickInterval - std::chrono::milliseconds(1)); + EXPECT_NEAR( + tickInterval.count(), + transport->getLossTimeoutRemainingTime().count(), + 2); +} + TEST_F(QuicTransportImplTest, CloseStreamAfterReadError) { auto stream1 = transport->createBidirectionalStream().value(); @@ -1131,9 +1144,12 @@ TEST_P(QuicTransportImplTestClose, TestNotifyPendingConnWriteOnCloseWithError) { transport->notifyPendingWriteOnConnection(&wcb); if (GetParam()) { EXPECT_CALL( - wcb, onConnectionWriteError(IsError(ApplicationErrorCode::STOPPING))); + wcb, + onConnectionWriteError( + IsAppError(GenericApplicationErrorCode::UNKNOWN))); transport->close(std::make_pair( - QuicErrorCode(ApplicationErrorCode::STOPPING), std::string("Bye"))); + QuicErrorCode(GenericApplicationErrorCode::UNKNOWN), + std::string("Bye"))); } else { transport->close(folly::none); } @@ -1157,9 +1173,11 @@ TEST_P(QuicTransportImplTestClose, TestNotifyPendingWriteOnCloseWithError) { if (GetParam()) { EXPECT_CALL( wcb, - onStreamWriteError(stream, IsError(ApplicationErrorCode::STOPPING))); + onStreamWriteError( + stream, IsAppError(GenericApplicationErrorCode::UNKNOWN))); transport->close(std::make_pair( - QuicErrorCode(ApplicationErrorCode::STOPPING), std::string("Bye"))); + QuicErrorCode(GenericApplicationErrorCode::UNKNOWN), + std::string("Bye"))); } else { transport->close(folly::none); } @@ -1199,8 +1217,9 @@ TEST_F(QuicTransportImplTest, TestGracefulCloseWithActiveStream) { EXPECT_TRUE(transport->notifyPendingWriteOnConnection(&wcbConn).hasError()); EXPECT_TRUE( transport->registerDeliveryCallback(stream, 2, &deliveryCb).hasError()); - EXPECT_TRUE(transport->resetStream(stream, ApplicationErrorCode::STOPPING) - .hasError()); + EXPECT_TRUE( + transport->resetStream(stream, GenericApplicationErrorCode::UNKNOWN) + .hasError()); transport->addDataToStream( stream, StreamBuffer(IOBuf::copyBuffer("hello"), 0, false)); @@ -1250,8 +1269,9 @@ TEST_F(QuicTransportImplTest, TestGracefulCloseWithNoActiveStream) { EXPECT_TRUE(transport->notifyPendingWriteOnConnection(&wcbConn).hasError()); EXPECT_TRUE( transport->registerDeliveryCallback(stream, 2, &deliveryCb).hasError()); - EXPECT_TRUE(transport->resetStream(stream, ApplicationErrorCode::STOPPING) - .hasError()); + EXPECT_TRUE( + transport->resetStream(stream, GenericApplicationErrorCode::UNKNOWN) + .hasError()); } TEST_F(QuicTransportImplTest, TestImmediateClose) { @@ -1261,10 +1281,14 @@ TEST_F(QuicTransportImplTest, TestImmediateClose) { MockReadCallback rcb; MockDeliveryCallback deliveryCb; EXPECT_CALL( - wcb, onStreamWriteError(stream, IsError(ApplicationErrorCode::STOPPING))); + wcb, + onStreamWriteError( + stream, IsAppError(GenericApplicationErrorCode::UNKNOWN))); EXPECT_CALL( - wcbConn, onConnectionWriteError(IsError(ApplicationErrorCode::STOPPING))); - EXPECT_CALL(rcb, readError(stream, IsError(ApplicationErrorCode::STOPPING))); + wcbConn, + onConnectionWriteError(IsAppError(GenericApplicationErrorCode::UNKNOWN))); + EXPECT_CALL( + rcb, readError(stream, IsAppError(GenericApplicationErrorCode::UNKNOWN))); EXPECT_CALL(deliveryCb, onCanceled(stream, _)); EXPECT_CALL(connCallback, onConnectionError(_)).Times(0); @@ -1277,7 +1301,8 @@ TEST_F(QuicTransportImplTest, TestImmediateClose) { transport->writeChain( stream, IOBuf::copyBuffer("hello"), true, false, &deliveryCb); transport->close(std::make_pair( - QuicErrorCode(ApplicationErrorCode::STOPPING), std::string("Error"))); + QuicErrorCode(GenericApplicationErrorCode::UNKNOWN), + std::string("Error"))); ASSERT_TRUE(transport->transportClosed); EXPECT_FALSE(transport->createBidirectionalStream()); @@ -1287,8 +1312,9 @@ TEST_F(QuicTransportImplTest, TestImmediateClose) { EXPECT_TRUE(transport->notifyPendingWriteOnConnection(&wcbConn).hasError()); EXPECT_TRUE( transport->registerDeliveryCallback(stream, 2, &deliveryCb).hasError()); - EXPECT_TRUE(transport->resetStream(stream, ApplicationErrorCode::STOPPING) - .hasError()); + EXPECT_TRUE( + transport->resetStream(stream, GenericApplicationErrorCode::UNKNOWN) + .hasError()); transport->addDataToStream( stream, StreamBuffer(IOBuf::copyBuffer("hello"), 0, false)); @@ -1452,7 +1478,7 @@ TEST_F(QuicTransportImplTest, UnidirectionalInvalidReadFuncs) { transport->resumeRead(stream).thenOrThrow([&](auto) {}), folly::Unexpected::BadExpectedAccess); EXPECT_THROW( - transport->stopSending(stream, ApplicationErrorCode::STOPPING) + transport->stopSending(stream, GenericApplicationErrorCode::UNKNOWN) .thenOrThrow([&](auto) {}), folly::Unexpected::BadExpectedAccess); } @@ -1481,7 +1507,7 @@ TEST_F(QuicTransportImplTest, UnidirectionalInvalidWriteFuncs) { .thenOrThrow([&](auto) {}), folly::Unexpected::BadExpectedAccess); EXPECT_THROW( - transport->resetStream(stream, ApplicationErrorCode::STOPPING) + transport->resetStream(stream, GenericApplicationErrorCode::UNKNOWN) .thenOrThrow([&](auto) {}), folly::Unexpected::BadExpectedAccess); } diff --git a/quic/api/test/QuicTransportFunctionsTest.cpp b/quic/api/test/QuicTransportFunctionsTest.cpp index 6c354e923..cc832cc49 100644 --- a/quic/api/test/QuicTransportFunctionsTest.cpp +++ b/quic/api/test/QuicTransportFunctionsTest.cpp @@ -438,9 +438,9 @@ TEST_F(QuicTransportFunctionsTest, TestUpdateConnectionPureAckCounter) { writeDataToQuicStream(*stream1, nullptr, true); conn->pendingEvents.resets.emplace( - 1, RstStreamFrame(1, ApplicationErrorCode::STOPPING, 0)); + 1, RstStreamFrame(1, GenericApplicationErrorCode::UNKNOWN, 0)); auto packet2 = buildEmptyPacket(*conn, PacketNumberSpace::Handshake); - RstStreamFrame rstFrame(1, ApplicationErrorCode::STOPPING, 0); + RstStreamFrame rstFrame(1, GenericApplicationErrorCode::UNKNOWN, 0); packet2.packet.frames.push_back(std::move(rstFrame)); updateConnection( @@ -1134,7 +1134,7 @@ TEST_F(QuicTransportFunctionsTest, ProbingNotWriteOtherFrames) { auto socket = std::make_unique(&evb); auto rawSocket = socket.get(); auto stream1 = conn->streamManager->createNextBidirectionalStream().value(); - RstStreamFrame rstFrame(stream1->id, ApplicationErrorCode::STOPPING, 0); + RstStreamFrame rstFrame(stream1->id, GenericApplicationErrorCode::UNKNOWN, 0); conn->pendingEvents.resets.emplace(stream1->id, rstFrame); conn->pendingEvents.connWindowUpdate = true; conn->streamManager->queueWindowUpdate(stream1->id); @@ -1504,7 +1504,8 @@ TEST_F(QuicTransportFunctionsTest, ClearRstFromPendingEvents) { auto conn = createConn(); auto stream = conn->streamManager->createNextBidirectionalStream().value(); auto packet = buildEmptyPacket(*conn, PacketNumberSpace::Handshake); - RstStreamFrame rstStreamFrame(stream->id, ApplicationErrorCode::STOPPING, 0); + RstStreamFrame rstStreamFrame( + stream->id, GenericApplicationErrorCode::UNKNOWN, 0); packet.packet.frames.push_back(rstStreamFrame); conn->pendingEvents.resets.emplace(stream->id, rstStreamFrame); updateConnection( @@ -1522,7 +1523,8 @@ TEST_F(QuicTransportFunctionsTest, ClonedRst) { auto packetEvent = conn->ackStates.appDataAckState.nextPacketNum; auto stream = conn->streamManager->createNextBidirectionalStream().value(); auto packet = buildEmptyPacket(*conn, PacketNumberSpace::AppData); - RstStreamFrame rstStreamFrame(stream->id, ApplicationErrorCode::STOPPING, 0); + RstStreamFrame rstStreamFrame( + stream->id, GenericApplicationErrorCode::UNKNOWN, 0); packet.packet.frames.push_back(rstStreamFrame); conn->outstandingPacketEvents.insert(packetEvent); // This shall not crash @@ -1548,7 +1550,8 @@ TEST_F(QuicTransportFunctionsTest, TimeoutBasedRetxCountUpdate) { auto stream = conn->streamManager->createNextBidirectionalStream().value(); conn->lossState.timeoutBasedRetxCount = 246; auto packet = buildEmptyPacket(*conn, PacketNumberSpace::AppData); - RstStreamFrame rstStreamFrame(stream->id, ApplicationErrorCode::STOPPING, 0); + RstStreamFrame rstStreamFrame( + stream->id, GenericApplicationErrorCode::UNKNOWN, 0); packet.packet.frames.push_back(rstStreamFrame); PacketEvent packetEvent = 100; conn->outstandingPacketEvents.insert(packetEvent); diff --git a/quic/api/test/QuicTransportTest.cpp b/quic/api/test/QuicTransportTest.cpp index 93a237502..0ad4bf0ce 100644 --- a/quic/api/test/QuicTransportTest.cpp +++ b/quic/api/test/QuicTransportTest.cpp @@ -758,7 +758,7 @@ TEST_F(QuicTransportTest, WritePendingAckIfHavingData) { TEST_F(QuicTransportTest, RstStream) { auto streamId = transport_->createBidirectionalStream().value(); EXPECT_CALL(*socket_, write(_, _)).WillOnce(Invoke(bufLength)); - transport_->resetStream(streamId, ApplicationErrorCode::STOPPING); + transport_->resetStream(streamId, GenericApplicationErrorCode::UNKNOWN); loopForWrites(); EXPECT_EQ(1, transport_->getConnectionState().outstandingPackets.size()); auto packet = @@ -770,7 +770,7 @@ TEST_F(QuicTransportTest, RstStream) { for (auto& frame : all_frames(packet.frames)) { EXPECT_EQ(streamId, frame.streamId); EXPECT_EQ(0, frame.offset); - EXPECT_EQ(ApplicationErrorCode::STOPPING, frame.errorCode); + EXPECT_EQ(GenericApplicationErrorCode::UNKNOWN, frame.errorCode); rstFound = true; } EXPECT_TRUE(rstFound); @@ -790,7 +790,7 @@ TEST_F(QuicTransportTest, RstStream) { TEST_F(QuicTransportTest, StopSending) { auto streamId = transport_->createBidirectionalStream().value(); EXPECT_CALL(*socket_, write(_, _)).WillOnce(Invoke(bufLength)); - transport_->stopSending(streamId, ApplicationErrorCode::STOPPING); + transport_->stopSending(streamId, GenericApplicationErrorCode::UNKNOWN); loopForWrites(); EXPECT_EQ(1, transport_->getConnectionState().outstandingPackets.size()); auto packet = @@ -804,7 +804,7 @@ TEST_F(QuicTransportTest, StopSending) { simpleFrame, [&](const StopSendingFrame& frame) { EXPECT_EQ(streamId, frame.streamId); - EXPECT_EQ(ApplicationErrorCode::STOPPING, frame.errorCode); + EXPECT_EQ(GenericApplicationErrorCode::UNKNOWN, frame.errorCode); foundStopSending = true; }, [&](auto&) {}); @@ -1229,7 +1229,7 @@ TEST_F(QuicTransportTest, RstWrittenStream) { auto currentWriteOffset = stream->currentWriteOffset; EXPECT_CALL(*socket_, write(_, _)).WillOnce(Invoke(bufLength)); - transport_->resetStream(streamId, ApplicationErrorCode::STOPPING); + transport_->resetStream(streamId, GenericApplicationErrorCode::UNKNOWN); loopForWrites(); // 2 packets are outstanding: one for Stream frame one for RstStream frame: EXPECT_EQ(2, transport_->getConnectionState().outstandingPackets.size()); @@ -1243,7 +1243,7 @@ TEST_F(QuicTransportTest, RstWrittenStream) { for (auto& frame : all_frames(packet.frames)) { EXPECT_EQ(streamId, frame.streamId); EXPECT_EQ(currentWriteOffset, frame.offset); - EXPECT_EQ(ApplicationErrorCode::STOPPING, frame.errorCode); + EXPECT_EQ(GenericApplicationErrorCode::UNKNOWN, frame.errorCode); foundReset = true; } EXPECT_TRUE(foundReset); @@ -1259,7 +1259,7 @@ TEST_F(QuicTransportTest, RstWrittenStream) { TEST_F(QuicTransportTest, RstStreamUDPWriteFailNonFatal) { auto streamId = transport_->createBidirectionalStream().value(); EXPECT_CALL(*socket_, write(_, _)).WillOnce(SetErrnoAndReturn(EAGAIN, -1)); - transport_->resetStream(streamId, ApplicationErrorCode::STOPPING); + transport_->resetStream(streamId, GenericApplicationErrorCode::UNKNOWN); loopForWrites(); EXPECT_EQ(1, transport_->getConnectionState().outstandingPackets.size()); @@ -1272,7 +1272,7 @@ TEST_F(QuicTransportTest, RstStreamUDPWriteFailNonFatal) { bool foundReset = false; for (auto& frame : all_frames(packet.frames)) { EXPECT_EQ(streamId, frame.streamId); - EXPECT_EQ(ApplicationErrorCode::STOPPING, frame.errorCode); + EXPECT_EQ(GenericApplicationErrorCode::UNKNOWN, frame.errorCode); foundReset = true; } EXPECT_TRUE(foundReset); @@ -1293,7 +1293,7 @@ TEST_F(QuicTransportTest, RstStreamUDPWriteFailFatal) { auto streamId = transport_->createBidirectionalStream().value(); EXPECT_CALL(*socket_, write(_, _)) .WillRepeatedly(SetErrnoAndReturn(EBADF, -1)); - transport_->resetStream(streamId, ApplicationErrorCode::STOPPING); + transport_->resetStream(streamId, GenericApplicationErrorCode::UNKNOWN); loopForWrites(); EXPECT_TRUE(transport_->getConnectionState().outstandingPackets.empty()); @@ -1311,7 +1311,7 @@ TEST_F(QuicTransportTest, WriteAfterSendRst) { ASSERT_TRUE(stream); auto currentWriteOffset = stream->currentWriteOffset; EXPECT_CALL(*socket_, write(_, _)).WillOnce(Invoke(bufLength)); - transport_->resetStream(streamId, ApplicationErrorCode::STOPPING); + transport_->resetStream(streamId, GenericApplicationErrorCode::UNKNOWN); loopForWrites(); EXPECT_TRUE(isState(*stream)); @@ -1339,7 +1339,7 @@ TEST_F(QuicTransportTest, WriteAfterSendRst) { for (auto& frame : all_frames(packet.frames)) { EXPECT_EQ(streamId, frame.streamId); EXPECT_EQ(currentWriteOffset, frame.offset); - EXPECT_EQ(ApplicationErrorCode::STOPPING, frame.errorCode); + EXPECT_EQ(GenericApplicationErrorCode::UNKNOWN, frame.errorCode); foundReset = true; } EXPECT_TRUE(foundReset); @@ -1351,13 +1351,15 @@ TEST_F(QuicTransportTest, WriteAfterSendRst) { TEST_F(QuicTransportTest, DoubleReset) { auto streamId = transport_->createBidirectionalStream().value(); EXPECT_CALL(*socket_, write(_, _)).WillOnce(Invoke(bufLength)); - EXPECT_FALSE(transport_->resetStream(streamId, ApplicationErrorCode::STOPPING) - .hasError()); + EXPECT_FALSE( + transport_->resetStream(streamId, GenericApplicationErrorCode::UNKNOWN) + .hasError()); loopForWrites(); // Then reset again, which is a no-op: - EXPECT_FALSE(transport_->resetStream(streamId, ApplicationErrorCode::STOPPING) - .hasError()); + EXPECT_FALSE( + transport_->resetStream(streamId, GenericApplicationErrorCode::UNKNOWN) + .hasError()); } TEST_F(QuicTransportTest, WriteStreamDataSetLossAlarm) { diff --git a/quic/api/test/TARGETS b/quic/api/test/TARGETS deleted file mode 100644 index ad3761953..000000000 --- a/quic/api/test/TARGETS +++ /dev/null @@ -1,118 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") -load("@fbcode_macros//build_defs:cpp_unittest.bzl", "cpp_unittest") - -cpp_library( - name = "mocks", - headers = [ - "MockQuicSocket.h", - "MockQuicStats.h", - "Mocks.h", - ], - deps = [ - "//folly/io/async:async_base", - "//folly/portability:gmock", - "//quic:exception", - "//quic/api:transport", - "//quic/codec:types", - "//quic/common:timers", - "//quic/server:server", - "//quic/state:state_machine", - "//quic/state:stats_callback", - ], -) - -cpp_unittest( - name = "QuicTransportTest", - srcs = [ - "QuicTransportTest.cpp", - ], - deps = [ - ":mocks", - "//fizz/crypto/aead/test:mocks", - "//folly:random", - "//folly/io:iobuf", - "//folly/io/async/test:mocks", - "//quic/api:transport", - "//quic/common:timers", - "//quic/common/test:test_utils", - "//quic/server/state:server", - "//quic/state:stream_functions", - "//quic/state/test:mocks", - ], - external_deps = [ - ("googletest", None, "gmock"), - ], -) - -cpp_unittest( - name = "QuicTransportBaseTest", - srcs = [ - "QuicTransportBaseTest.cpp", - ], - deps = [ - ":mocks", - "//folly/io/async/test:mocks", - "//folly/portability:gmock", - "//folly/portability:gtest", - "//quic/api:transport", - "//quic/codec:types", - "//quic/common/test:test_utils", - "//quic/server/state:server", - "//quic/state:stream_functions", - "//quic/state/test:mocks", - ], -) - -cpp_unittest( - name = "QuicTransportFunctionsTest", - srcs = [ - "QuicTransportFunctionsTest.cpp", - ], - deps = [ - ":mocks", - "//folly/io/async/test:mocks", - "//quic/api:transport", - "//quic/common/test:test_utils", - "//quic/server/state:server", - "//quic/state/test:mocks", - ], -) - -cpp_unittest( - name = "QuicPacketSchedulerTest", - srcs = [ - "QuicPacketSchedulerTest.cpp", - ], - deps = [ - ":mocks", - "//folly/portability:gtest", - "//quic/api:transport", - "//quic/client/state:client", - "//quic/codec:pktbuilder", - "//quic/common/test:test_utils", - "//quic/server/state:server", - ], -) - -cpp_unittest( - name = "IoBufQuicBatchTest", - srcs = [ - "IoBufQuicBatchTest.cpp", - ], - deps = [ - "//quic/api:transport", - "//quic/state:state_machine", - ], -) - -cpp_unittest( - name = "QuicBatchWriterTest", - srcs = [ - "QuicBatchWriterTest.cpp", - ], - deps = [ - "//quic/api:transport", - ], -) diff --git a/quic/client/TARGETS b/quic/client/TARGETS deleted file mode 100644 index 1528fd08a..000000000 --- a/quic/client/TARGETS +++ /dev/null @@ -1,28 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") - -cpp_library( - name = "client", - srcs = [ - "QuicClientTransport.cpp", - ], - headers = [ - "QuicClientTransport.h", - ], - deps = [ - "//folly:network_address", - "//folly:random", - "//folly/io/async:async_udp_socket", - "//folly/portability:sockets", - "//quic/api:transport", - "//quic/client/handshake:client_extension", - "//quic/client/handshake:psk_cache", - "//quic/client/state:client", - "//quic/flowcontrol:flow_control", - "//quic/happyeyeballs:happyeyeballs", - "//quic/loss:loss", - "//quic/state:ack_handler", - "//quic/state:pacing_functions", - ], -) diff --git a/quic/client/handshake/TARGETS b/quic/client/handshake/TARGETS deleted file mode 100644 index 66082f513..000000000 --- a/quic/client/handshake/TARGETS +++ /dev/null @@ -1,54 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") - -cpp_library( - name = "client_handshake", - srcs = [ - "ClientHandshake.cpp", - ], - headers = [ - "ClientHandshake.h", - ], - deps = [ - ":client_extension", - ":psk_cache", - "//fizz/client:early_data_rejection", - "//fizz/client:fizz_client_context", - "//fizz/client:protocol", - "//fizz/client:psk_cache", - "//fizz/protocol:default_certificate_verifier", - "//fizz/protocol:protocol", - "//folly:exception_wrapper", - "//folly/io:iobuf", - "//folly/io/async:delayed_destruction", - "//quic:constants", - "//quic:exception", - "//quic/handshake:handshake", - "//quic/state:state_machine", - "//quic/state:stream_functions", - ], -) - -cpp_library( - name = "client_extension", - headers = [ - "ClientTransportParametersExtension.h", - ], - deps = [ - "//fizz/client:client_extensions", - "//quic/handshake:transport_parameters", - ], -) - -cpp_library( - name = "psk_cache", - headers = [ - "QuicPskCache.h", - ], - deps = [ - "//fizz/client:psk_cache", - "//folly:optional", - "//quic:constants", - ], -) diff --git a/quic/client/handshake/test/TARGETS b/quic/client/handshake/test/TARGETS deleted file mode 100644 index eb232bb7f..000000000 --- a/quic/client/handshake/test/TARGETS +++ /dev/null @@ -1,53 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") -load("@fbcode_macros//build_defs:cpp_unittest.bzl", "cpp_unittest") - -cpp_unittest( - name = "ClientHandshakeTest", - srcs = [ - "ClientHandshakeTest.cpp", - ], - deps = [ - ":mock_psk_cache", - "//fizz/crypto/test:TestUtil", - "//fizz/protocol/test:mocks", - "//fizz/server:protocol", - "//fizz/server/test:mocks", - "//folly/io/async:scoped_event_base_thread", - "//folly/io/async:ssl_context", - "//folly/io/async/test:mocks", - "//folly/ssl:init", - "//quic/client/handshake:client_handshake", - "//quic/common/test:test_utils", - "//quic/state:state_machine", - "//quic/state:stream_functions", - ], - external_deps = [ - ("googletest", None, "gmock"), - ], -) - -cpp_unittest( - name = "ClientTransportParametersTest", - srcs = [ - "ClientTransportParametersTest.cpp", - ], - deps = [ - "//fizz/protocol/test:test_messages", - "//quic/client/handshake:client_extension", - "//quic/common/test:test_utils", - ], -) - -cpp_library( - name = "mock_psk_cache", - headers = [ - "MockQuicPskCache.h", - ], - deps = [ - "//folly:optional", - "//folly/portability:gmock", - "//quic/client/handshake:psk_cache", - ], -) diff --git a/quic/client/state/TARGETS b/quic/client/state/TARGETS deleted file mode 100644 index c873d2c3f..000000000 --- a/quic/client/state/TARGETS +++ /dev/null @@ -1,22 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") - -cpp_library( - name = "client", - srcs = [ - "ClientStateMachine.cpp", - ], - headers = [ - "ClientStateMachine.h", - ], - deps = [ - "//folly/io/async:async_socket_exception", - "//quic/client/handshake:client_handshake", - "//quic/congestion_control:cubic", - "//quic/flowcontrol:flow_control", - "//quic/handshake:transport_parameters", - "//quic/state:state_functions", - "//quic/state:state_machine", - ], -) diff --git a/quic/client/test/QuicClientTransportTest.cpp b/quic/client/test/QuicClientTransportTest.cpp index f5e165b3a..c44b68262 100644 --- a/quic/client/test/QuicClientTransportTest.cpp +++ b/quic/client/test/QuicClientTransportTest.cpp @@ -2173,7 +2173,7 @@ TEST_P( EXPECT_CALL(readCb, readError(streamId, _)); if (GetParam()) { client->close(std::make_pair( - QuicErrorCode(ApplicationErrorCode::STOPPING), + QuicErrorCode(GenericApplicationErrorCode::UNKNOWN), std::string("stopping"))); EXPECT_TRUE(verifyFramePresent( socketWrites, *makeHandshakeCodec())); @@ -2322,7 +2322,7 @@ TEST_P(QuicClientTransportAfterStartTestClose, CloseConnectionWithError) { socketWrites.clear(); if (GetParam()) { client->close(std::make_pair( - QuicErrorCode(ApplicationErrorCode::STOPPING), + QuicErrorCode(GenericApplicationErrorCode::UNKNOWN), std::string("stopping"))); EXPECT_TRUE(verifyFramePresent( socketWrites, *makeEncryptedCodec())); @@ -2598,7 +2598,7 @@ TEST_F( QuicClientTransportAfterStartTest, ReceiveRstStreamNonExistentClientStream) { StreamId streamId = 0x04; - RstStreamFrame rstFrame(streamId, ApplicationErrorCode::INTERNAL_ERROR, 0); + RstStreamFrame rstFrame(streamId, GenericApplicationErrorCode::UNKNOWN, 0); ShortHeader header( ProtectionType::KeyPhaseZero, *originalConnId, appDataPacketNum++); RegularQuicPacketBuilder builder( @@ -2617,7 +2617,7 @@ TEST_F( // A RstStreamFrame will be written to sock when we receive a RstStreamFrame // even on a nonexist server stream: StreamId streamId = 0x01; - RstStreamFrame rstFrame(streamId, ApplicationErrorCode::INTERNAL_ERROR, 0); + RstStreamFrame rstFrame(streamId, GenericApplicationErrorCode::UNKNOWN, 0); ShortHeader header( ProtectionType::KeyPhaseZero, *originalConnId, appDataPacketNum++); RegularQuicPacketBuilder builder( @@ -2638,7 +2638,7 @@ TEST_F(QuicClientTransportAfterStartTest, ReceiveRstStreamNewStream) { client->createBidirectionalStream(false /* replaySafe */).value(); client->registerDeliveryCallback(streamId, 0, &deliveryCallback); - RstStreamFrame rstFrame(streamId, ApplicationErrorCode::INTERNAL_ERROR, 0); + RstStreamFrame rstFrame(streamId, GenericApplicationErrorCode::UNKNOWN, 0); ShortHeader header( ProtectionType::KeyPhaseZero, *originalConnId, appDataPacketNum++); RegularQuicPacketBuilder builder( @@ -2688,7 +2688,7 @@ TEST_F(QuicClientTransportAfterStartTest, ReceiveRstStreamAfterEom) { EXPECT_CALL(readCb, readError(streamId, _)); RstStreamFrame rstFrame( - streamId, ApplicationErrorCode::INTERNAL_ERROR, data->length()); + streamId, GenericApplicationErrorCode::UNKNOWN, data->length()); ShortHeader header( ProtectionType::KeyPhaseZero, *originalConnId, appDataPacketNum++); RegularQuicPacketBuilder builder( @@ -3065,7 +3065,7 @@ TEST_F( // Test whether packets from the remote endpoint can be processed after a // version negotiation - RstStreamFrame rstFrame(streamId, ApplicationErrorCode::INTERNAL_ERROR, 0); + RstStreamFrame rstFrame(streamId, GenericApplicationErrorCode::UNKNOWN, 0); ShortHeader header( ProtectionType::KeyPhaseZero, *originalConnId, appDataPacketNum++); RegularQuicPacketBuilder builder( @@ -3249,7 +3249,7 @@ TEST_F(QuicClientTransportAfterStartTest, SendReset) { client->registerDeliveryCallback(streamId, 100, &deliveryCallback); EXPECT_CALL(deliveryCallback, onCanceled(streamId, 100)); EXPECT_CALL(readCb, readError(streamId, _)); - client->resetStream(streamId, ApplicationErrorCode::STOPPING); + client->resetStream(streamId, GenericApplicationErrorCode::UNKNOWN); loopForWrites(); verifyShortPackets(sentPackets); @@ -3314,7 +3314,7 @@ TEST_F(QuicClientTransportAfterStartTest, ResetClearsPendingLoss) { std::find(pendingLossStreams.begin(), pendingLossStreams.end(), streamId); ASSERT_TRUE(it != pendingLossStreams.end()); - client->resetStream(streamId, ApplicationErrorCode::STOPPING); + client->resetStream(streamId, GenericApplicationErrorCode::UNKNOWN); it = std::find(pendingLossStreams.begin(), pendingLossStreams.end(), streamId); ASSERT_TRUE(it == pendingLossStreams.end()); @@ -3330,7 +3330,7 @@ TEST_F(QuicClientTransportAfterStartTest, LossAfterResetStream) { loopForWrites(); ASSERT_FALSE(client->getConn().outstandingPackets.empty()); - client->resetStream(streamId, ApplicationErrorCode::STOPPING); + client->resetStream(streamId, GenericApplicationErrorCode::UNKNOWN); RegularQuicWritePacket* forceLossPacket = CHECK_NOTNULL(findPacketWithStream(client->getNonConstConn(), streamId)); @@ -3355,7 +3355,7 @@ TEST_F(QuicClientTransportAfterStartTest, SendResetAfterEom) { EXPECT_CALL(deliveryCallback, onCanceled(streamId, 100)); client->writeChain(streamId, IOBuf::copyBuffer("hello"), true, false); - client->resetStream(streamId, ApplicationErrorCode::STOPPING); + client->resetStream(streamId, GenericApplicationErrorCode::UNKNOWN); loopForWrites(); verifyShortPackets(sentPackets); const auto& readCbs = client->getReadCallbacks(); @@ -3436,8 +3436,8 @@ TEST_F(QuicClientTransportAfterStartTest, SendResetSyncOnAck) { EXPECT_CALL(deliveryCallback, onDeliveryAck(streamId, _, _)) .WillOnce(Invoke([&](auto, auto, auto) { - client->resetStream(streamId, ApplicationErrorCode::STOPPING); - client->resetStream(streamId2, ApplicationErrorCode::STOPPING); + client->resetStream(streamId, GenericApplicationErrorCode::UNKNOWN); + client->resetStream(streamId2, GenericApplicationErrorCode::UNKNOWN); })); auto packet1 = packetToBuf(createStreamPacket( *serverChosenConnId /* src */, @@ -3557,7 +3557,7 @@ TEST_F(QuicClientTransportAfterStartTest, ReceiveApplicationClose) { std::move(header), 0 /* largestAcked */); ApplicationCloseFrame appClose( - ApplicationErrorCode::STOPPING, + GenericApplicationErrorCode::UNKNOWN, "Stand clear of the closing doors, please"); writeFrame(std::move(appClose), builder); auto packet = packetToBuf(std::move(builder).buildPacket()); @@ -3566,7 +3566,7 @@ TEST_F(QuicClientTransportAfterStartTest, ReceiveApplicationClose) { EXPECT_CALL( clientConnCallback, - onConnectionError(IsError(ApplicationErrorCode::STOPPING))); + onConnectionError(IsAppError(GenericApplicationErrorCode::UNKNOWN))); deliverDataWithoutErrorCheck(packet->coalesce()); // Now the transport should be closed EXPECT_EQ( diff --git a/quic/client/test/TARGETS b/quic/client/test/TARGETS deleted file mode 100644 index a46d9e415..000000000 --- a/quic/client/test/TARGETS +++ /dev/null @@ -1,29 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_unittest.bzl", "cpp_unittest") - -cpp_unittest( - name = "QuicClientTransportTest", - srcs = [ - "QuicClientTransportTest.cpp", - ], - deps = [ - "//fizz/crypto/aead/test:mocks", - "//folly/futures:core", - "//folly/io:iobuf", - "//folly/io/async:scoped_event_base_thread", - "//folly/io/async/test:mocks", - "//folly/portability:gmock", - "//folly/portability:gtest", - "//quic/api/test:mocks", - "//quic/client:client", - "//quic/client/handshake/test:mock_psk_cache", - "//quic/codec:types", - "//quic/common/test:test_utils", - "//quic/congestion_control:congestion_controller_factory", - "//quic/handshake:transport_parameters", - "//quic/samples/echo:EchoHandler", - "//quic/server:server", - "//quic/state/test:mocks", - ], -) diff --git a/quic/codec/Decode.cpp b/quic/codec/Decode.cpp index bcff15973..6e0da21cb 100644 --- a/quic/codec/Decode.cpp +++ b/quic/codec/Decode.cpp @@ -215,8 +215,8 @@ RstStreamFrame decodeRstStreamFrame(folly::io::Cursor& cursor) { quic::TransportErrorCode::FRAME_ENCODING_ERROR, quic::FrameType::RST_STREAM); } - auto errorCode = static_cast( - cursor.readBE::type>()); + auto errorCode = + static_cast(cursor.readBE()); auto offset = decodeQuicInteger(cursor); if (UNLIKELY(!offset)) { throw QuicTransportException( @@ -242,8 +242,8 @@ StopSendingFrame decodeStopSendingFrame(folly::io::Cursor& cursor) { quic::TransportErrorCode::FRAME_ENCODING_ERROR, quic::FrameType::STOP_SENDING); } - auto errorCode = static_cast( - cursor.readBE::type>()); + auto errorCode = + static_cast(cursor.readBE()); return StopSendingFrame(folly::to(streamId->first), errorCode); } @@ -558,8 +558,7 @@ ConnectionCloseFrame decodeConnectionCloseFrame(folly::io::Cursor& cursor) { } ApplicationCloseFrame decodeApplicationCloseFrame(folly::io::Cursor& cursor) { - auto detailedCode = - cursor.readBE::type>(); + auto detailedCode = cursor.readBE(); auto errorCode = static_cast(detailedCode); auto reasonPhraseLength = decodeQuicInteger(cursor); if (UNLIKELY( diff --git a/quic/codec/QuicWriteCodec.cpp b/quic/codec/QuicWriteCodec.cpp index 34be03228..f8d36f4ad 100644 --- a/quic/codec/QuicWriteCodec.cpp +++ b/quic/codec/QuicWriteCodec.cpp @@ -300,8 +300,7 @@ size_t writeSimpleFrame( builder.write(packetType); builder.write(streamId); builder.writeBE( - static_cast::type>( - stopSendingFrame.errorCode)); + static_cast(stopSendingFrame.errorCode)); builder.appendFrame(std::move(stopSendingFrame)); return stopSendingFrameSize; } @@ -410,8 +409,7 @@ size_t writeFrame(QuicWriteFrame&& frame, PacketBuilderInterface& builder) { builder.write(packetType); builder.write(streamId); builder.writeBE( - static_cast::type>( - rstStreamFrame.errorCode)); + static_cast(rstStreamFrame.errorCode)); builder.write(offset); builder.appendFrame(std::move(rstStreamFrame)); return rstStreamFrameSize; @@ -570,9 +568,8 @@ size_t writeFrame(QuicWriteFrame&& frame, PacketBuilderInterface& builder) { applicationCloseFrame.reasonPhrase.size(); if (packetSpaceCheck(spaceLeft, applicationCloseFrameSize)) { builder.write(packetType); - builder.writeBE( - static_cast::type>( - applicationCloseFrame.errorCode)); + builder.writeBE(static_cast( + applicationCloseFrame.errorCode)); builder.write(reasonLength); builder.push( (const uint8_t*)applicationCloseFrame.reasonPhrase.data(), diff --git a/quic/codec/TARGETS b/quic/codec/TARGETS deleted file mode 100644 index c430363db..000000000 --- a/quic/codec/TARGETS +++ /dev/null @@ -1,149 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") - -cpp_library( - name = "types", - srcs = [ - "DefaultConnectionIdAlgo.cpp", - "PacketNumber.cpp", - "QuicConnectionId.cpp", - "QuicInteger.cpp", - "Types.cpp", - ], - headers = [ - "ConnectionIdAlgo.h", - "DefaultConnectionIdAlgo.h", - "PacketNumber.h", - "QuicConnectionId.h", - "QuicInteger.h", - "Types.h", - ], - deps = [ - "//folly:conv", - "//folly:optional", - "//folly:overload", - "//folly:random", - "//folly:string", - "//folly/container:array", - "//folly/hash:hash", - "//folly/io:iobuf", - "//folly/lang:bits", - "//quic:constants", - "//quic:exception", - "//quic/common:interval_set", - ], - external_deps = [ - "boost", - "glog", - ], -) - -cpp_library( - name = "packet_number_cipher", - srcs = [ - "PacketNumberCipher.cpp", - ], - deps = [ - ":decode", - ":types", - "//folly:optional", - "//folly/io:iobuf", - "//folly/ssl:openssl_ptr_types", - ], -) - -cpp_library( - name = "decode", - srcs = [ - "Decode.cpp", - ], - headers = [ - "Decode.h", - ], - compiler_flags = [ - "-fstrict-aliasing", - ], - deps = [ - ":types", - "//folly:string", - "//folly/io:iobuf", - "//quic:exception", - ], -) - -cpp_library( - name = "pktbuilder", - srcs = [ - "QuicPacketBuilder.cpp", - ], - headers = [ - "QuicPacketBuilder.h", - ], - compiler_flags = [ - "-fstrict-aliasing", - ], - deps = [ - ":types", - "//folly:random", - "//quic/handshake:handshake", - ], -) - -cpp_library( - name = "pktrebuilder", - srcs = [ - "QuicPacketRebuilder.cpp", - ], - headers = [ - "QuicPacketRebuilder.h", - ], - deps = [ - ":codec", - ":pktbuilder", - "//quic/flowcontrol:flow_control", - "//quic/state:simple_frame_functions", - "//quic/state:state_machine", - "//quic/state:stream_functions", - ], -) - -cpp_library( - name = "header_codec", - srcs = [ - "QuicHeaderCodec.cpp", - ], - headers = [ - "QuicHeaderCodec.h", - ], - deps = [ - ":decode", - ":types", - "//folly:optional", - "//quic:exception", - ], -) - -cpp_library( - name = "codec", - srcs = [ - "QuicReadCodec.cpp", - "QuicWriteCodec.cpp", - ], - headers = [ - "QuicReadCodec.h", - "QuicWriteCodec.h", - ], - deps = [ - ":decode", - ":pktbuilder", - ":types", - "//folly:optional", - "//folly/io:iobuf", - "//quic:constants", - "//quic:exception", - "//quic/common:interval_set", - "//quic/handshake:handshake", - "//quic/state:ack_states", - ], -) diff --git a/quic/codec/Types.h b/quic/codec/Types.h index f60055a81..54866f314 100644 --- a/quic/codec/Types.h +++ b/quic/codec/Types.h @@ -164,6 +164,11 @@ struct ReadCryptoFrame { } } + ReadCryptoFrame(ReadCryptoFrame&& other) noexcept { + offset = other.offset; + data = std::move(other.data); + } + ReadCryptoFrame& operator=(const ReadCryptoFrame& other) { offset = other.offset; if (other.data) { @@ -172,6 +177,12 @@ struct ReadCryptoFrame { return *this; } + ReadCryptoFrame& operator=(ReadCryptoFrame&& other) { + offset = other.offset; + data = std::move(other.data); + return *this; + } + bool operator==(const ReadCryptoFrame& other) const { folly::IOBufEqualTo eq; return offset == other.offset && eq(data, other.data); @@ -287,6 +298,13 @@ struct ReadStreamFrame { fin = other.fin; } + ReadStreamFrame(ReadStreamFrame&& other) noexcept { + streamId = other.streamId; + offset = other.offset; + data = std::move(other.data); + fin = other.fin; + } + ReadStreamFrame& operator=(const ReadStreamFrame& other) { streamId = other.streamId; offset = other.offset; @@ -297,6 +315,14 @@ struct ReadStreamFrame { return *this; } + ReadStreamFrame& operator=(ReadStreamFrame&& other) { + streamId = other.streamId; + offset = other.offset; + data = std::move(other.data); + fin = other.fin; + return *this; + } + bool operator==(const ReadStreamFrame& other) const { folly::IOBufEqualTo eq; return streamId == other.streamId && offset == other.offset && diff --git a/quic/codec/test/QuicPacketRebuilderTest.cpp b/quic/codec/test/QuicPacketRebuilderTest.cpp index 89b082030..71a1c035d 100644 --- a/quic/codec/test/QuicPacketRebuilderTest.cpp +++ b/quic/codec/test/QuicPacketRebuilderTest.cpp @@ -178,7 +178,9 @@ TEST_F(QuicPacketRebuilderTest, RebuildAfterResetStream) { // Then we reset the stream invokeStreamStateMachine( - conn, *stream, StreamEvents::SendReset(ApplicationErrorCode::STOPPING)); + conn, + *stream, + StreamEvents::SendReset(GenericApplicationErrorCode::UNKNOWN)); RegularQuicPacketBuilder regularBuilder2( kDefaultUDPSendPacketLen, shortHeader, 0 /* largestAcked */); PacketRebuilder rebuilder(regularBuilder2, conn); diff --git a/quic/codec/test/QuicWriteCodecTest.cpp b/quic/codec/test/QuicWriteCodecTest.cpp index 034d6b958..8a01e01c3 100644 --- a/quic/codec/test/QuicWriteCodecTest.cpp +++ b/quic/codec/test/QuicWriteCodecTest.cpp @@ -1082,14 +1082,15 @@ TEST_F(QuicWriteCodecTest, DecodeAppCloseLarge) { std::string reasonPhrase; reasonPhrase.resize(kMaxReasonPhraseLength + 10); ApplicationCloseFrame applicationCloseFrame( - ApplicationErrorCode::STOPPING, reasonPhrase); + GenericApplicationErrorCode::UNKNOWN, reasonPhrase); writeFrame(applicationCloseFrame, pktBuilder); auto builtOut = std::move(pktBuilder).buildPacket(); auto regularPacket = builtOut.first; auto resultAppCloseFrame = boost::get(regularPacket.frames[0]); - EXPECT_EQ(ApplicationErrorCode::STOPPING, resultAppCloseFrame.errorCode); + EXPECT_EQ( + GenericApplicationErrorCode::UNKNOWN, resultAppCloseFrame.errorCode); EXPECT_EQ(resultAppCloseFrame.reasonPhrase, reasonPhrase); auto wireBuf = std::move(builtOut.second); @@ -1192,7 +1193,7 @@ TEST_F(QuicWriteCodecTest, WriteRstStream) { MockQuicPacketBuilder pktBuilder; setupCommonExpects(pktBuilder); StreamId id = 0xBAAD; - ApplicationErrorCode errorCode = ApplicationErrorCode::STOPPING; + ApplicationErrorCode errorCode = GenericApplicationErrorCode::UNKNOWN; uint64_t offset = 0xF00D; RstStreamFrame rstStreamFrame(id, errorCode, offset); auto rstStreamBytesWritten = writeFrame(rstStreamFrame, pktBuilder); @@ -1221,7 +1222,7 @@ TEST_F(QuicWriteCodecTest, NoSpaceForRst) { pktBuilder.remaining_ = 1; setupCommonExpects(pktBuilder); StreamId id = 0xBAAD; - ApplicationErrorCode errorCode = ApplicationErrorCode::STOPPING; + ApplicationErrorCode errorCode = GenericApplicationErrorCode::UNKNOWN; uint64_t offset = 0xF00D; RstStreamFrame rstStreamFrame(id, errorCode, offset); EXPECT_EQ(0, writeFrame(rstStreamFrame, pktBuilder)); @@ -1314,7 +1315,7 @@ TEST_F(QuicWriteCodecTest, WriteStopSending) { MockQuicPacketBuilder pktBuilder; setupCommonExpects(pktBuilder); StreamId streamId = 10; - auto errorCode = ApplicationErrorCode::STOPPING; + auto errorCode = GenericApplicationErrorCode::UNKNOWN; StopSendingFrame stopSending(streamId, errorCode); auto bytesWritten = writeSimpleFrame(stopSending, pktBuilder); diff --git a/quic/codec/test/TARGETS b/quic/codec/test/TARGETS deleted file mode 100644 index cb7ec38d9..000000000 --- a/quic/codec/test/TARGETS +++ /dev/null @@ -1,177 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") -load("@fbcode_macros//build_defs:cpp_unittest.bzl", "cpp_unittest") - -cpp_library( - name = "mocks", - headers = [ - "Mocks.h", - ], - deps = [ - "//folly/portability:gmock", - "//folly/portability:gtest", - "//quic/codec:pktbuilder", - "//quic/common/test:test_utils", - ], -) - -cpp_unittest( - name = "QuicHeaderCodec", - srcs = [ - "QuicHeaderCodecTest.cpp", - ], - deps = [ - "//folly:optional", - "//folly:overload", - "//folly/portability:gtest", - "//quic:exception", - "//quic/codec:header_codec", - "//quic/common/test:test_utils", - ], -) - -cpp_unittest( - name = "QuicReadCodecTests", - srcs = [ - "QuicReadCodecTest.cpp", - ], - deps = [ - "//folly/io:iobuf", - "//folly/portability:gtest", - "//quic:exception", - "//quic/codec:codec", - "//quic/common/test:test_utils", - ], -) - -cpp_unittest( - name = "QuicWriteCodecTests", - srcs = [ - "QuicWriteCodecTest.cpp", - ], - compiler_flags = [ - "-ftemplate-backtrace-limit=0", - ], - deps = [ - ":mocks", - "//folly:random", - "//folly/io:iobuf", - "//folly/portability:gmock", - "//folly/portability:gtest", - "//quic:exception", - "//quic/codec:codec", - "//quic/codec:decode", - "//quic/codec:types", - "//quic/common/test:test_utils", - ], -) - -cpp_unittest( - name = "TypesTests", - srcs = [ - "TypesTest.cpp", - ], - deps = [ - "//folly:overload", - "//folly:string", - "//folly/container:array", - "//folly/io:iobuf", - "//folly/portability:gtest", - "//quic:exception", - "//quic/codec:decode", - "//quic/codec:types", - "//quic/common/test:test_utils", - ], -) - -cpp_unittest( - name = "PacketNumberTest", - srcs = [ - "PacketNumberTest.cpp", - ], - deps = [ - "//folly/portability:gtest", - "//quic/codec:types", - ], -) - -cpp_unittest( - name = "PacketNumberCipherTest", - srcs = [ - "PacketNumberCipherTest.cpp", - ], - deps = [ - "//folly:string", - "//folly/portability:gtest", - "//quic/codec:packet_number_cipher", - ], -) - -cpp_unittest( - name = "DecodeTests", - srcs = [ - "DecodeTest.cpp", - ], - deps = [ - "//folly:random", - "//folly/container:array", - "//folly/io:iobuf", - "//folly/portability:gtest", - "//quic/codec:codec", - "//quic/codec:decode", - "//quic/codec:types", - "//quic/common/test:test_utils", - ], -) - -cpp_unittest( - name = "QuicConnectionIdTest", - srcs = [ - "QuicConnectionIdTest.cpp", - ], - deps = [ - "//folly/portability:gtest", - "//quic/codec:types", - ], -) - -cpp_unittest( - name = "QuicPacketBuilderTest", - srcs = [ - "QuicPacketBuilderTest.cpp", - "QuicPacketRebuilderTest.cpp", - ], - deps = [ - ":mocks", - "//folly:random", - "//folly/portability:gtest", - "//quic/codec:codec", - "//quic/codec:pktbuilder", - "//quic/codec:pktrebuilder", - "//quic/codec:types", - "//quic/common/test:test_utils", - "//quic/handshake:handshake", - "//quic/server/state:server", - "//quic/state:state_functions", - "//quic/state:state_machine", - "//quic/state:stream_functions", - "//quic/state/stream:stream", - ], -) - -cpp_unittest( - name = "QuicIntegerTest", - srcs = [ - "QuicIntegerTest.cpp", - ], - deps = [ - "//folly:expected", - "//folly:optional", - "//folly:string", - "//folly/io:iobuf", - "//folly/portability:gtest", - "//quic:exception", - "//quic/codec:types", - ], -) diff --git a/quic/common/TARGETS b/quic/common/TARGETS deleted file mode 100644 index eb216c3ca..000000000 --- a/quic/common/TARGETS +++ /dev/null @@ -1,55 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") - -cpp_library( - name = "interval_set", - headers = [ - "IntervalSet.h", - "IntervalSet-inl.h", - ], - deps = [ - "//folly:likely", - "//folly:optional", - ], -) - -cpp_library( - name = "time_util", - headers = [ - "TimeUtil.h", - ], - deps = [ - "//folly:traits", - ], -) - -cpp_library( - name = "timers", - srcs = [ - "Timers.cpp", - ], - headers = [ - "Timers.h", - ], - deps = [ - "//folly/experimental:timerfd", - "//folly/io/async:async_base", - ], -) - -cpp_library( - name = "looper", - srcs = [ - "FunctionLooper.cpp", - ], - headers = [ - "FunctionLooper.h", - ], - deps = [ - ":timers", - "//folly:function", - "//folly:scope_guard", - "//folly/io/async:async_base", - ], -) diff --git a/quic/common/TimeUtil.h b/quic/common/TimeUtil.h index f582fe822..cb3e99bf8 100644 --- a/quic/common/TimeUtil.h +++ b/quic/common/TimeUtil.h @@ -9,6 +9,7 @@ #pragma once #include +#include #include #include diff --git a/quic/common/test/TARGETS b/quic/common/test/TARGETS deleted file mode 100644 index 63028abc8..000000000 --- a/quic/common/test/TARGETS +++ /dev/null @@ -1,77 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") -load("@fbcode_macros//build_defs:cpp_unittest.bzl", "cpp_unittest") - -cpp_unittest( - name = "IntervalSetTest", - srcs = [ - "IntervalSetTest.cpp", - ], - deps = [ - "//quic/common:interval_set", - ], -) - -cpp_unittest( - name = "FunctionLooperTest", - srcs = [ - "FunctionLooperTest.cpp", - ], - deps = [ - "//quic/common:looper", - ], -) - -cpp_library( - name = "test_utils", - srcs = [ - "TestUtils.cpp", - ], - headers = [ - "QuicCodecUtils.h", - "TestUtils.h", - ], - deps = [ - "//fizz/client:fizz_client_context", - "//fizz/crypto/aead/test:mocks", - "//fizz/crypto/test:TestUtil", - "//fizz/protocol/test:mocks", - "//fizz/server:fizz_server_context", - "//folly:overload", - "//folly/io/async/test:mocks", - "//folly/ssl:init", - "//quic/api:transport", - "//quic/client/handshake:psk_cache", - "//quic/codec:pktbuilder", - "//quic/codec:types", - "//quic/handshake/test:mocks", - "//quic/server/state:server", - "//quic/state:ack_states", - "//quic/state:state_machine", - ], - external_deps = [ - "boost", - ], -) - -cpp_unittest( - name = "QuicCodecUtilsTest", - srcs = [ - "QuicCodecUtilsTest.cpp", - ], - deps = [ - ":test_utils", - "//folly/portability:gtest", - ], -) - -cpp_unittest( - name = "TimeUtilTest", - srcs = [ - "TimeUtilTest.cpp", - ], - deps = [ - "//quic/common:time_util", - ], -) diff --git a/quic/common/test/TestUtils.cpp b/quic/common/test/TestUtils.cpp index bf2b01711..47e48d3b4 100644 --- a/quic/common/test/TestUtils.cpp +++ b/quic/common/test/TestUtils.cpp @@ -276,7 +276,7 @@ void setupCtxWithTestCert(fizz::server::FizzServerContext& ctx) { std::unique_ptr createNoOpAead() { // Fake that the handshake has already occured - auto aead = std::make_unique(); + auto aead = std::make_unique>(); ON_CALL(*aead, _encrypt(_, _, _)) .WillByDefault(Invoke([&](auto& buf, auto, auto) { if (buf) { @@ -299,7 +299,7 @@ std::unique_ptr createNoOpAead() { } std::unique_ptr createNoOpHeaderCipher() { - auto headerCipher = std::make_unique(); + auto headerCipher = std::make_unique>(); ON_CALL(*headerCipher, mask(_)).WillByDefault(Return(HeaderProtectionMask{})); ON_CALL(*headerCipher, keyLength()).WillByDefault(Return(16)); return headerCipher; diff --git a/quic/common/test/TestUtils.h b/quic/common/test/TestUtils.h index f2b0f01ff..f2db021b8 100644 --- a/quic/common/test/TestUtils.h +++ b/quic/common/test/TestUtils.h @@ -172,6 +172,28 @@ MATCHER_P(IsError, error, "") { return matchError(arg, error); } +inline bool matchAppError( + std::pair errorCode, + ApplicationErrorCode error) { + return folly::variant_match( + errorCode.first, + [&](ApplicationErrorCode err) { return err == error; }, + [](auto) { return false; }); +} + +inline bool matchAppError( + std::pair> errorCode, + ApplicationErrorCode error) { + return folly::variant_match( + errorCode.first, + [&](ApplicationErrorCode err) { return err == error; }, + [](auto) { return false; }); +} + +MATCHER_P(IsAppError, error, "") { + return matchAppError(arg, error); +} + void updateAckState( QuicConnectionStateBase& conn, PacketNumberSpace pnSpace, diff --git a/quic/congestion_control/Copa.cpp b/quic/congestion_control/Copa.cpp index d1e1e5da7..789e23c03 100644 --- a/quic/congestion_control/Copa.cpp +++ b/quic/congestion_control/Copa.cpp @@ -256,14 +256,13 @@ void Copa::onPacketLoss(const LossEvent& loss) { << " inflight=" << bytesInFlight_ << " " << conn_; DCHECK(loss.largestLostPacketNum.hasValue()); subtractAndCheckUnderflow(bytesInFlight_, loss.lostBytes); -} - -void Copa::onRTOVerified() { - // TODO See if we should go to slowStart here - VLOG(10) << __func__ << " writable=" << getWritableBytes() - << " cwnd=" << cwndBytes_ << " inflight=" << bytesInFlight_ << " " - << conn_; - cwndBytes_ = conn_.transportSettings.minCwndInMss * conn_.udpSendPacketLen; + if (loss.persistentCongestion) { + // TODO See if we should go to slowStart here + VLOG(10) << __func__ << " writable=" << getWritableBytes() + << " cwnd=" << cwndBytes_ << " inflight=" << bytesInFlight_ << " " + << conn_; + cwndBytes_ = conn_.transportSettings.minCwndInMss * conn_.udpSendPacketLen; + } } uint64_t Copa::getWritableBytes() const noexcept { diff --git a/quic/congestion_control/Copa.h b/quic/congestion_control/Copa.h index a3f83168c..9af3ae100 100644 --- a/quic/congestion_control/Copa.h +++ b/quic/congestion_control/Copa.h @@ -31,7 +31,6 @@ class Copa : public CongestionController { explicit Copa(QuicConnectionStateBase& conn); void onRemoveBytesFromInflight(uint64_t) override; void onPacketSent(const OutstandingPacket& packet) override; - void onRTOVerified() override; void onPacketAckOrLoss(folly::Optional, folly::Optional) override; diff --git a/quic/congestion_control/NewReno.cpp b/quic/congestion_control/NewReno.cpp index 65e9402b6..1d1d121df 100644 --- a/quic/congestion_control/NewReno.cpp +++ b/quic/congestion_control/NewReno.cpp @@ -103,13 +103,12 @@ void NewReno::onPacketLoss(const LossEvent& loss) { << " writable=" << getWritableBytes() << " cwnd=" << cwndBytes_ << " inflight=" << bytesInFlight_ << " " << conn_; } -} - -void NewReno::onRTOVerified() { - VLOG(10) << __func__ << " writable=" << getWritableBytes() - << " cwnd=" << cwndBytes_ << " inflight=" << bytesInFlight_ << " " - << conn_; - cwndBytes_ = conn_.transportSettings.minCwndInMss * conn_.udpSendPacketLen; + if (loss.persistentCongestion) { + VLOG(10) << __func__ << " writable=" << getWritableBytes() + << " cwnd=" << cwndBytes_ << " inflight=" << bytesInFlight_ << " " + << conn_; + cwndBytes_ = conn_.transportSettings.minCwndInMss * conn_.udpSendPacketLen; + } } uint64_t NewReno::getWritableBytes() const noexcept { diff --git a/quic/congestion_control/NewReno.h b/quic/congestion_control/NewReno.h index a48831334..94b0ac8dc 100644 --- a/quic/congestion_control/NewReno.h +++ b/quic/congestion_control/NewReno.h @@ -22,7 +22,6 @@ class NewReno : public CongestionController { void onPacketSent(const OutstandingPacket& packet) override; void onPacketAckOrLoss(folly::Optional, folly::Optional) override; - void onRTOVerified() override; uint64_t getWritableBytes() const noexcept override; uint64_t getCongestionWindow() const noexcept override; diff --git a/quic/congestion_control/QuicCubic.cpp b/quic/congestion_control/QuicCubic.cpp index fa34e817b..be44372a4 100644 --- a/quic/congestion_control/QuicCubic.cpp +++ b/quic/congestion_control/QuicCubic.cpp @@ -52,7 +52,14 @@ uint64_t Cubic::getCongestionWindow() const noexcept { return cwndBytes_; } -void Cubic::onRTOVerified() { +/** + * TODO: onPersistentCongestion entirely depends on how long a loss period is, + * not how much a sender sends during that period. If the connection is app + * limited and loss happens after that, it looks like a long loss period but it + * may not really be a persistent congestion. However, to keep this code simple, + * we decide to just ignore app limited state right now. + */ +void Cubic::onPersistentCongestion() { auto minCwnd = conn_.transportSettings.minCwndInMss * conn_.udpSendPacketLen; ssthresh_ = std::max(cwndBytes_ / 2, minCwnd); cwndBytes_ = minCwnd; @@ -68,7 +75,7 @@ void Cubic::onRTOVerified() { state_ = CubicStates::Hystart; QUIC_TRACE( - cubic_rto, + cubic_persistent_congestion, conn_, cubicStateToString(state_).data(), cwndBytes_, @@ -123,6 +130,10 @@ void Cubic::onPacketLoss(const LossEvent& loss) { } else { QUIC_TRACE(fst_trace, conn_, "cubic_skip_loss"); } + + if (loss.persistentCongestion) { + onPersistentCongestion(); + } } void Cubic::onRemoveBytesFromInflight(uint64_t bytes) { @@ -176,10 +187,11 @@ void Cubic::setAppLimited(bool limited, TimePoint eventTime) noexcept { steadyState_.lastReductionTime->time_since_epoch()) .count() : -1); - if (!quiescenceStart_.hasValue() && limited) { + bool currentAppLimited = isAppLimited(); + if (!currentAppLimited && limited) { quiescenceStart_ = eventTime; } - if (!limited && quiescenceStart_ && *quiescenceStart_ <= eventTime && + if (!limited && currentAppLimited && *quiescenceStart_ <= eventTime && steadyState_.lastReductionTime) { *steadyState_.lastReductionTime += std::chrono::duration_cast( @@ -642,7 +654,7 @@ void Cubic::onPacketAckedInHystart(const AckEvent& ack) { * Theoretically, both paper and Linux/Chromium should result to the same cwnd. */ void Cubic::onPacketAckedInSteady(const AckEvent& ack) { - if (quiescenceStart_.hasValue()) { + if (isAppLimited()) { QUIC_TRACE(fst_trace, conn_, "ack_in_quiescence"); return; } diff --git a/quic/congestion_control/QuicCubic.h b/quic/congestion_control/QuicCubic.h index 8a9147b89..0e509f3a1 100644 --- a/quic/congestion_control/QuicCubic.h +++ b/quic/congestion_control/QuicCubic.h @@ -31,14 +31,11 @@ enum class CubicStates : uint8_t { * | [Ack] [Ack] | * | | | | * -->Hystart------------[Ack]---------->Cubic<--| - * | ^ | ^ - * | | | | - * | |-------[RTOVerified]<---------| | - * | ^ | | - * | | | | - * | | ->[ACK/Loss] | | - * | | | | | | - * | | | | | | + * | | | + * | | | + * | ->[ACK/Loss] | | + * | | | | | + * | | | | | * -[Loss]---->Fast Recovery<--[Loss]- | * | | * | | @@ -96,7 +93,6 @@ class Cubic : public CongestionController { override; void onRemoveBytesFromInflight(uint64_t) override; void onPacketSent(const OutstandingPacket& packet) override; - void onRTOVerified() override; uint64_t getWritableBytes() const noexcept override; uint64_t getCongestionWindow() const noexcept override; @@ -128,6 +124,7 @@ class Cubic : public CongestionController { void onPacketLoss(const LossEvent& loss); void onPacketLossInRecovery(const LossEvent& loss); + void onPersistentCongestion(); float pacingGain() const noexcept; void updatePacing() noexcept; diff --git a/quic/congestion_control/TARGETS b/quic/congestion_control/TARGETS deleted file mode 100644 index 48ea6e6fc..000000000 --- a/quic/congestion_control/TARGETS +++ /dev/null @@ -1,84 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") - -cpp_library( - name = "congestion_control_functions", - srcs = [ - "CongestionControlFunctions.cpp", - ], - headers = [ - "CongestionControlFunctions.h", - ], - deps = [ - "//quic:constants", - "//quic/common:time_util", - "//quic/state:state_machine", - ], -) - -cpp_library( - name = "congestion_controller_factory", - srcs = [ - "CongestionControllerFactory.cpp", - ], - headers = [ - "CongestionControllerFactory.h", - ], - deps = [ - ":copa", - ":cubic", - ":newreno", - "//quic:constants", - ], -) - -cpp_library( - name = "cubic", - srcs = [ - "QuicCubic.cpp", - ], - headers = [ - "QuicCubic.h", - ], - deps = [ - ":congestion_control_functions", - "//quic:exception", - "//quic/logging:logging", - "//quic/state:state_functions", - "//quic/state:state_machine", - ], -) - -cpp_library( - name = "newreno", - srcs = [ - "NewReno.cpp", - ], - headers = [ - "NewReno.h", - ], - deps = [ - ":congestion_control_functions", - "//quic:exception", - "//quic/state:state_machine", - ], -) - -cpp_library( - name = "copa", - srcs = [ - "Copa.cpp", - ], - headers = [ - "Copa.h", - ], - deps = [ - ":congestion_control_functions", - "//folly:optional", - "//quic:exception", - "//quic/common:time_util", - "//quic/congestion_control/third_party:chromium_windowed_filter", - "//quic/state:state_machine", - ], -) diff --git a/quic/congestion_control/test/CopaTest.cpp b/quic/congestion_control/test/CopaTest.cpp index a34d86b80..6616fba51 100644 --- a/quic/congestion_control/test/CopaTest.cpp +++ b/quic/congestion_control/test/CopaTest.cpp @@ -142,7 +142,7 @@ TEST_F(CopaTest, TestWritableBytes) { EXPECT_EQ(copa.getWritableBytes(), 0); } -TEST_F(CopaTest, RTOVerified) { +TEST_F(CopaTest, PersistentCongestion) { QuicServerConnectionState conn; Copa copa(conn); EXPECT_TRUE(copa.inSlowStart()); @@ -150,12 +150,15 @@ TEST_F(CopaTest, RTOVerified) { conn.lossState.largestSent = 5; PacketNum ackPacketNum = 6; uint32_t ackedSize = 10; - copa.onPacketSent(createPacket(ackPacketNum, ackedSize, ackedSize)); - copa.onRTOVerified(); - + auto pkt = createPacket(ackPacketNum, ackedSize, ackedSize); + copa.onPacketSent(pkt); + CongestionController::LossEvent loss; + loss.persistentCongestion = true; + loss.addLostPacket(pkt); + copa.onPacketAckOrLoss(folly::none, loss); EXPECT_EQ( copa.getWritableBytes(), - conn.transportSettings.minCwndInMss * conn.udpSendPacketLen - ackedSize); + conn.transportSettings.minCwndInMss * conn.udpSendPacketLen); EXPECT_TRUE(copa.inSlowStart()); } diff --git a/quic/congestion_control/test/CubicHystartTest.cpp b/quic/congestion_control/test/CubicHystartTest.cpp index 67137e6bf..9561afb09 100644 --- a/quic/congestion_control/test/CubicHystartTest.cpp +++ b/quic/congestion_control/test/CubicHystartTest.cpp @@ -297,25 +297,5 @@ TEST_F(CubicHystartTest, ReduceByCubicReductionFactor) { EXPECT_EQ(initCwnd * 0.9, cubic.getWritableBytes()); EXPECT_EQ(CubicStates::FastRecovery, cubic.state()); } - -TEST_F(CubicHystartTest, RTOVerifiedDuringHystart) { - QuicConnectionStateBase conn(QuicNodeType::Client); - conn.udpSendPacketLen = 100; - Cubic cubic(conn); - - conn.lossState.largestSent = 0; - auto packet = makeTestingWritePacket(0, 700, 700); - cubic.onPacketSent(packet); - cubic.onRTOVerified(); - - EXPECT_EQ(0, cubic.getWritableBytes()); - - CongestionController::LossEvent loss; - loss.addLostPacket(packet); - cubic.onPacketAckOrLoss(folly::none, std::move(loss)); - - // Cwnd can't go less than mincwnd. - EXPECT_EQ(200, cubic.getWritableBytes()); -} } // namespace test } // namespace quic diff --git a/quic/congestion_control/test/CubicStateTest.cpp b/quic/congestion_control/test/CubicStateTest.cpp index 2551a87b7..fc9db6c7f 100644 --- a/quic/congestion_control/test/CubicStateTest.cpp +++ b/quic/congestion_control/test/CubicStateTest.cpp @@ -42,13 +42,6 @@ TEST_F(CubicStateTest, HystartAck) { EXPECT_EQ(CubicStates::Hystart, cubic.state()); } -TEST_F(CubicStateTest, HystartRTOVerified) { - QuicConnectionStateBase conn(QuicNodeType::Client); - Cubic cubic(conn); - cubic.onRTOVerified(); - EXPECT_EQ(CubicStates::Hystart, cubic.state()); -} - TEST_F(CubicStateTest, HystartPace) { QuicConnectionStateBase conn(QuicNodeType::Client); conn.transportSettings.pacingEnabled = true; @@ -104,14 +97,6 @@ TEST_F(CubicStateTest, FastRecoveryLoss) { EXPECT_EQ(CubicStates::FastRecovery, cubic.state()); } -TEST_F(CubicStateTest, FastRecoveryRTOVerified) { - QuicConnectionStateBase conn(QuicNodeType::Client); - TestingCubic cubic(conn); - cubic.setStateForTest(CubicStates::FastRecovery); - cubic.onRTOVerified(); - EXPECT_EQ(CubicStates::Hystart, cubic.state()); -} - TEST_F(CubicStateTest, RecoveryNoPace) { QuicConnectionStateBase conn(QuicNodeType::Client); conn.transportSettings.pacingEnabled = true; @@ -145,14 +130,6 @@ TEST_F(CubicStateTest, SteadyLoss) { EXPECT_EQ(CubicStates::FastRecovery, cubic.state()); } -TEST_F(CubicStateTest, SteadyRTOVerified) { - QuicConnectionStateBase conn(QuicNodeType::Client); - TestingCubic cubic(conn); - cubic.setStateForTest(CubicStates::Steady); - cubic.onRTOVerified(); - EXPECT_EQ(CubicStates::Hystart, cubic.state()); -} - TEST_F(CubicStateTest, SteadyCanPace) { QuicConnectionStateBase conn(QuicNodeType::Client); conn.transportSettings.pacingEnabled = true; diff --git a/quic/congestion_control/test/CubicTest.cpp b/quic/congestion_control/test/CubicTest.cpp index 9ff12d595..43dcc0259 100644 --- a/quic/congestion_control/test/CubicTest.cpp +++ b/quic/congestion_control/test/CubicTest.cpp @@ -41,7 +41,7 @@ TEST_F(CubicTest, AckIncreaseWritable) { EXPECT_EQ(initCwnd, cubic.getWritableBytes()); } -TEST_F(CubicTest, RTOVerified) { +TEST_F(CubicTest, PersistentCongestion) { QuicConnectionStateBase conn(QuicNodeType::Client); Cubic cubic(conn, std::numeric_limits::max(), false); auto initCwnd = cubic.getWritableBytes(); @@ -50,11 +50,8 @@ TEST_F(CubicTest, RTOVerified) { cubic.onPacketSent(packet); CongestionController::LossEvent loss; loss.addLostPacket(packet); + loss.persistentCongestion = true; cubic.onPacketAckOrLoss(folly::none, std::move(loss)); - EXPECT_EQ(CubicStates::FastRecovery, cubic.state()); - - // rto verified: - cubic.onRTOVerified(); EXPECT_EQ(CubicStates::Hystart, cubic.state()); // Cwnd should be dropped to minCwnd: EXPECT_EQ( @@ -68,8 +65,9 @@ TEST_F(CubicTest, RTOVerified) { EXPECT_EQ(CubicStates::Steady, cubic.state()); // Verify both lastMaxCwndBytes and lastReductionTime are also reset in - // RTOVerified. When they are both verified, the first ACK will make both - // timeToOrigin and timeElapsed to be 0 in Ack handling in Steady handler: + // onPersistentCongestion. When they are both verified, the first ACK will + // make both timeToOrigin and timeElapsed to be 0 in Ack handling in Steady + // handler: auto currentCwnd = cubic.getWritableBytes(); // since nothing inflight auto packet3 = makeTestingWritePacket(2, 3000, initCwnd / 2 + 1000 + 3000); cubic.onPacketSent(packet3); @@ -128,19 +126,6 @@ TEST_F(CubicTest, CwndIncreaseAfterReduction) { EXPECT_EQ(CubicStates::Steady, cubic.state()); } -TEST_F(CubicTest, PacketSizeChange) { - QuicConnectionStateBase conn(QuicNodeType::Client); - conn.udpSendPacketLen = 1000; - Cubic cubic(conn); - EXPECT_EQ( - conn.transportSettings.initCwndInMss * 1000, cubic.getWritableBytes()); - - conn.udpSendPacketLen = 50000; - cubic.onRTOVerified(); - EXPECT_EQ( - conn.transportSettings.minCwndInMss * 50000, cubic.getWritableBytes()); -} - TEST_F(CubicTest, AppLimited) { QuicConnectionStateBase conn(QuicNodeType::Client); conn.udpSendPacketLen = 1500; @@ -169,6 +154,7 @@ TEST_F(CubicTest, AppLimited) { cwnd = cubic.getCongestionWindow(); cubic.setAppLimited(true, reductionTime + std::chrono::milliseconds(1100)); + EXPECT_TRUE(cubic.isAppLimited()); auto packet2 = makeTestingWritePacket(2, 1000, 3000); cubic.onPacketSent(packet2); cubic.onPacketAckOrLoss( @@ -178,6 +164,7 @@ TEST_F(CubicTest, AppLimited) { // 1 seconds of quiescence cubic.setAppLimited(false, reductionTime + std::chrono::milliseconds(2100)); + EXPECT_FALSE(cubic.isAppLimited()); auto packet3 = makeTestingWritePacket(3, 1000, 4000); cubic.onPacketSent(packet3); cubic.onPacketAckOrLoss( diff --git a/quic/congestion_control/test/NewRenoTest.cpp b/quic/congestion_control/test/NewRenoTest.cpp index 3d9ab6707..cabdb7d50 100644 --- a/quic/congestion_control/test/NewRenoTest.cpp +++ b/quic/congestion_control/test/NewRenoTest.cpp @@ -178,7 +178,7 @@ TEST_F(NewRenoTest, TestWritableBytes) { EXPECT_EQ(reno.getWritableBytes(), 0); } -TEST_F(NewRenoTest, RTOVerified) { +TEST_F(NewRenoTest, PersistentCongestion) { QuicServerConnectionState conn; NewReno reno(conn); EXPECT_TRUE(reno.inSlowStart()); @@ -186,12 +186,15 @@ TEST_F(NewRenoTest, RTOVerified) { conn.lossState.largestSent = 5; PacketNum ackPacketNum = 6; uint32_t ackedSize = 10; - reno.onPacketSent(createPacket(ackPacketNum, ackedSize)); - reno.onRTOVerified(); - + auto pkt = createPacket(ackPacketNum, ackedSize); + reno.onPacketSent(pkt); + CongestionController::LossEvent loss; + loss.persistentCongestion = true; + loss.addLostPacket(pkt); + reno.onPacketAckOrLoss(folly::none, loss); EXPECT_EQ( reno.getWritableBytes(), - conn.transportSettings.minCwndInMss * conn.udpSendPacketLen - ackedSize); + conn.transportSettings.minCwndInMss * conn.udpSendPacketLen); EXPECT_TRUE(reno.inSlowStart()); } diff --git a/quic/congestion_control/test/TARGETS b/quic/congestion_control/test/TARGETS deleted file mode 100644 index c65b58b70..000000000 --- a/quic/congestion_control/test/TARGETS +++ /dev/null @@ -1,79 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") -load("@fbcode_macros//build_defs:cpp_unittest.bzl", "cpp_unittest") - -cpp_library( - name = "TestingCubic", - headers = [ - "TestingCubic.h", - ], - deps = [ - "//quic/congestion_control:cubic", - ], -) - -cpp_unittest( - name = "CongestionControlFunctionsTest", - srcs = [ - "CongestionControlFunctionsTest.cpp", - ], - deps = [ - "//folly/portability:gtest", - "//quic:constants", - "//quic/congestion_control:congestion_control_functions", - "//quic/state:state_machine", - ], -) - -cpp_unittest( - name = "CubicStateTest", - srcs = [ - "CubicStateTest.cpp", - ], - deps = [ - ":TestingCubic", - "//folly/portability:gtest", - "//quic/common/test:test_utils", - ], -) - -cpp_unittest( - name = "CubicTest", - srcs = [ - "CubicHystartTest.cpp", - "CubicRecoveryTest.cpp", - "CubicSteadyTest.cpp", - "CubicTest.cpp", - ], - deps = [ - ":TestingCubic", - "//folly/portability:gtest", - "//quic/common/test:test_utils", - "//quic/congestion_control:cubic", - ], -) - -cpp_unittest( - name = "NewRenoTest", - srcs = [ - "NewRenoTest.cpp", - ], - deps = [ - "//folly/portability:gtest", - "//quic/common/test:test_utils", - "//quic/congestion_control:newreno", - ], -) - -cpp_unittest( - name = "CopaTest", - srcs = [ - "CopaTest.cpp", - ], - deps = [ - "//folly/portability:gtest", - "//quic/common/test:test_utils", - "//quic/congestion_control:copa", - ], -) diff --git a/quic/congestion_control/third_party/TARGETS b/quic/congestion_control/third_party/TARGETS deleted file mode 100644 index 223461589..000000000 --- a/quic/congestion_control/third_party/TARGETS +++ /dev/null @@ -1,10 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") - -cpp_library( - name = "chromium_windowed_filter", - headers = [ - "windowed_filter.h", - ], -) diff --git a/quic/flowcontrol/TARGETS b/quic/flowcontrol/TARGETS deleted file mode 100644 index d9521b533..000000000 --- a/quic/flowcontrol/TARGETS +++ /dev/null @@ -1,18 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") - -cpp_library( - name = "flow_control", - srcs = [ - "QuicFlowController.cpp", - ], - headers = ["QuicFlowController.h"], - deps = [ - "//quic:constants", - "//quic:exception", - "//quic/codec:types", - "//quic/logging:logging", - "//quic/state:state_machine", - ], -) diff --git a/quic/flowcontrol/test/TARGETS b/quic/flowcontrol/test/TARGETS deleted file mode 100644 index b5b2a645c..000000000 --- a/quic/flowcontrol/test/TARGETS +++ /dev/null @@ -1,19 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_unittest.bzl", "cpp_unittest") - -cpp_unittest( - name = "QuicFlowControlTest", - srcs = [ - "QuicFlowControlTest.cpp", - ], - deps = [ - "//quic/api/test:mocks", - "//quic/client/state:client", - "//quic/common/test:test_utils", - "//quic/flowcontrol:flow_control", - ], - external_deps = [ - ("googletest", None, "gmock"), - ], -) diff --git a/quic/handshake/TARGETS b/quic/handshake/TARGETS deleted file mode 100644 index 6e8e38ed7..000000000 --- a/quic/handshake/TARGETS +++ /dev/null @@ -1,44 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") - -cpp_library( - name = "handshake", - srcs = [ - "HandshakeLayer.cpp", - "QuicFizzFactory.cpp", - ], - headers = [ - "HandshakeLayer.h", - "QuicFizzFactory.h", - ], - deps = [ - "//fizz/crypto:hmac_sha256", - "//fizz/crypto:key_derivation", - "//fizz/crypto/aead:aead", - "//fizz/protocol:exporter", - "//fizz/protocol:factory", - "//fizz/protocol:types", - "//fizz/record:record", - "//folly:expected", - "//quic:constants", - "//quic/codec:packet_number_cipher", - "//quic/codec:types", - ], -) - -cpp_library( - name = "transport_parameters", - srcs = [ - "TransportParameters.cpp", - ], - headers = [ - "TransportParameters.h", - ], - deps = [ - "//fizz/record:record", - "//quic:constants", - "//quic:exception", - "//quic/codec:types", - ], -) diff --git a/quic/handshake/test/TARGETS b/quic/handshake/test/TARGETS deleted file mode 100644 index 1c2d871d5..000000000 --- a/quic/handshake/test/TARGETS +++ /dev/null @@ -1,45 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_unittest.bzl", "cpp_unittest") - -cpp_library( - name = "mocks", - headers = [ - "Mocks.h", - ], - external_deps = [ - ("googletest", None, "gmock"), - ], -) - -cpp_unittest( - name = "TransportParametersTest", - srcs = [ - "TransportParametersTest.cpp", - ], - deps = [ - "//fizz/record/test:extension_tests_base", - "//quic/common/test:test_utils", - "//quic/handshake:transport_parameters", - ], - external_deps = [ - ("googletest", None, "gmock"), - ], -) - -cpp_unittest( - name = "HandshakeLayerTest", - srcs = [ - "HandshakeLayerTest.cpp", - ], - deps = [ - ":mocks", - "//fizz/crypto/aead/test:mocks", - "//quic/codec:types", - "//quic/common/test:test_utils", - "//quic/handshake:handshake", - ], - external_deps = [ - ("googletest", None, "gmock"), - ], -) diff --git a/quic/happyeyeballs/TARGETS b/quic/happyeyeballs/TARGETS deleted file mode 100644 index 8d0ecf46a..000000000 --- a/quic/happyeyeballs/TARGETS +++ /dev/null @@ -1,22 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") - -cpp_library( - name = "happyeyeballs", - srcs = [ - "QuicHappyEyeballsFunctions.cpp", - ], - headers = [ - "QuicHappyEyeballsFunctions.h", - ], - deps = [ - "//folly:network_address", - "//folly/io/async:async_base", - "//folly/io/async:async_udp_socket", - "//folly/net:net_ops", - "//folly/portability:sockets", - "//quic/logging:logging", - "//quic/state:state_machine", - ], -) diff --git a/quic/logging/TARGETS b/quic/logging/TARGETS deleted file mode 100644 index d25d61905..000000000 --- a/quic/logging/TARGETS +++ /dev/null @@ -1,15 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") - -cpp_library( - name = "logging", - headers = [ - "QuicLogger.h", - ], - deps = [ - "//folly/tracing:static_tracepoint", - "//quic/codec:types", - "//quic/state:state_machine", - ], -) diff --git a/quic/logging/test/TARGETS b/quic/logging/test/TARGETS deleted file mode 100644 index 37f2cd449..000000000 --- a/quic/logging/test/TARGETS +++ /dev/null @@ -1,14 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_unittest.bzl", "cpp_unittest") - -cpp_unittest( - name = "MacroTest", - srcs = [ - "QuicLoggingMacroTest.cpp", - ], - deps = [ - "//folly/portability:gtest", - "//quic/logging:logging", - ], -) diff --git a/quic/loss/QuicLossFunctions.cpp b/quic/loss/QuicLossFunctions.cpp index a790b4156..586fad553 100644 --- a/quic/loss/QuicLossFunctions.cpp +++ b/quic/loss/QuicLossFunctions.cpp @@ -17,16 +17,24 @@ std::chrono::microseconds calculateRTO(const QuicConnectionStateBase& conn) { conn.lossState.maxAckDelay; } +bool isPersistentCongestion( + const QuicConnectionStateBase& conn, + TimePoint lostPeriodStart, + TimePoint lostPeriodEnd) noexcept { + if (conn.lossState.srtt == std::chrono::microseconds::zero()) { + return false; + } + auto rto = calculateRTO(conn); + return (lostPeriodEnd - lostPeriodStart) >= + rto * kPersistentCongestionPeriodFactor; +} + void onRTOAlarm(QuicConnectionStateBase& conn) { VLOG(10) << __func__ << " " << conn; - if (conn.lossState.rtoCount == 0) { - conn.lossState.largestSentBeforeRto = conn.lossState.largestSent; - } QUIC_TRACE( rto_alarm, conn, conn.lossState.largestSent, - *conn.lossState.largestSentBeforeRto, conn.lossState.rtoCount, (uint64_t)conn.outstandingPackets.size()); QUIC_STATS(conn.infoCallback, onRTO); diff --git a/quic/loss/QuicLossFunctions.h b/quic/loss/QuicLossFunctions.h index 30f9fa114..2c8fe971e 100644 --- a/quic/loss/QuicLossFunctions.h +++ b/quic/loss/QuicLossFunctions.h @@ -28,6 +28,18 @@ bool hasNonAckDataToWrite(const QuicConnectionStateBase& conn); std::chrono::microseconds calculateRTO(const QuicConnectionStateBase& conn); +/** + * Whether conn is having persistent congestion. + * + * TODO: This currently doesn't check if packets within this window from other + * pnSpace are lost too. + * Pending https://github.com/quicwg/base-drafts/issues/2649 + */ +bool isPersistentCongestion( + const QuicConnectionStateBase& conn, + TimePoint lostPeriodStart, + TimePoint lostPeriodEnd) noexcept; + inline std::ostream& operator<<( std::ostream& os, const LossState::AlarmMethod& alarmMethod) { @@ -50,16 +62,15 @@ std::pair calculateAlarmDuration(const QuicConnectionStateBase& conn) { std::chrono::microseconds alarmDuration; folly::Optional alarmMethod; - TimePoint lastSentPacketTime = conn.outstandingPackets.back().time; + TimePoint lastSentPacketTime = + conn.lossState.lastRetransmittablePacketSentTime; if (conn.outstandingHandshakePacketsCount > 0) { if (conn.lossState.srtt == std::chrono::microseconds(0)) { alarmDuration = kDefaultInitialRtt * 2; } else { alarmDuration = conn.lossState.srtt * 2; } - // TODO: kMinTLPTimeout will be gone in later diff - alarmDuration = - timeMax(conn.lossState.maxAckDelay + alarmDuration, kMinTLPTimeout); + alarmDuration += conn.lossState.maxAckDelay; alarmDuration *= 1 << std::min(conn.lossState.handshakeAlarmCount, (uint16_t)15); alarmMethod = LossState::AlarmMethod::Handshake; @@ -68,6 +79,7 @@ calculateAlarmDuration(const QuicConnectionStateBase& conn) { DCHECK_NE(lastSentPacketTime.time_since_epoch().count(), 0); } else if (conn.lossState.lossTime) { if (*conn.lossState.lossTime > lastSentPacketTime) { + // We do this so that lastSentPacketTime + alarmDuration = lossTime alarmDuration = std::chrono::duration_cast( *conn.lossState.lossTime - lastSentPacketTime); } else { @@ -136,7 +148,6 @@ void setLossDetectionAlarm(QuicConnectionStateBase& conn, Timeout& timeout) { timeout.cancelLossTimeout(); return; } - // TODO: updating outstandingClonedPacketsCount will be in followup diffs. /* * We have this condition to disambiguate the case where we have. * (1) All outstanding packets that are clones that are processed and there @@ -189,32 +200,24 @@ void setLossDetectionAlarm(QuicConnectionStateBase& conn, Timeout& timeout) { * This function should be invoked after some event that is possible to * trigger loss detection, for example: packets are acked */ -template +template folly::Optional detectLossPackets( QuicConnectionStateBase& conn, PacketNum largestAcked, const LossVisitor& lossVisitor, TimePoint lossTime, - folly::Optional rtoVerifiedPacket, PacketNumberSpace pnSpace) { - DCHECK(!rtoVerifiedPacket || *rtoVerifiedPacket <= largestAcked); conn.lossState.lossTime.clear(); - folly::Optional delayUntilLost; - // TODO: maybe cache these values for efficiency - if (conn.lossState.lossMode == LossState::LossMode::TimeLossDetection) { - delayUntilLost = std::chrono::duration_cast( - std::max(conn.lossState.srtt, conn.lossState.lrtt) * - (1 + conn.lossState.timeReorderingFraction)); - } else if (largestAcked == conn.lossState.largestSent) { - delayUntilLost = std::max(conn.lossState.srtt, conn.lossState.lrtt) * 9 / 8; - } + std::chrono::microseconds delayUntilLost = + std::max(conn.lossState.srtt, conn.lossState.lrtt) * 9 / 8; VLOG(10) << __func__ << " outstanding=" << conn.outstandingPackets.size() - << " largestAcked=" << largestAcked << " delayUntilLost=" - << delayUntilLost.value_or(std::chrono::microseconds::zero()).count() - << "us" + << " largestAcked=" << largestAcked + << " delayUntilLost=" << delayUntilLost.count() << "us" << " " << conn; CongestionController::LossEvent lossEvent(lossTime); + // Note that time based loss detection is also within the same PNSpace. auto iter = getFirstOutstandingPacket(conn, pnSpace); + bool shouldSetTimer = false; while (iter != conn.outstandingPackets.end()) { auto& pkt = *iter; auto currentPacketNum = folly::variant_match( @@ -230,21 +233,13 @@ folly::Optional detectLossPackets( iter++; continue; } - bool lost = false; - if (delayUntilLost) { - lost = lost || ((ClockType::now() - pkt.time) > *delayUntilLost); - } - if (conn.lossState.lossMode == LossState::LossMode::ReorderingThreshold) { - lost = lost || - (largestAcked - currentPacketNum) > - conn.lossState.reorderingThreshold; - } - if (rtoVerifiedPacket && currentPacketNum <= *rtoVerifiedPacket) { - lost = true; - } + bool lost = (lossTime - pkt.time) > delayUntilLost; + lost = lost || + (largestAcked - currentPacketNum) > conn.lossState.reorderingThreshold; if (!lost) { // We can exit early here because if packet N doesn't meet the // threshold, then packet N + 1 will not either. + shouldSetTimer = true; break; } if (!pkt.pureAck) { @@ -276,19 +271,27 @@ folly::Optional detectLossPackets( iter = conn.outstandingPackets.erase(iter); } - if (!conn.outstandingPackets.empty() && !conn.lossState.lossTime && - delayUntilLost) { + // Because we handle handshake timer before lossTime, lossTime is used by + // AppData space only. So this is fine. + auto earliest = getFirstOutstandingPacket(conn, pnSpace); + for (; earliest != conn.outstandingPackets.end(); earliest++) { + if (!earliest->pureAck && + (!earliest->associatedEvent || + conn.outstandingPacketEvents.count(*earliest->associatedEvent))) { + break; + } + } + if (shouldSetTimer && earliest != conn.outstandingPackets.end()) { // We are eligible to set a loss timer and there are a few packets which // are unacked, so we can set the early retransmit timer for them. VLOG(10) << __func__ << " early retransmit timer outstanding=" << conn.outstandingPackets.empty() << " delayUntilLost" - << delayUntilLost->count() << "us" + << delayUntilLost.count() << "us" << " " << conn; - conn.lossState.lossTime = - *delayUntilLost + conn.outstandingPackets.front().time; + conn.lossState.lossTime = delayUntilLost + earliest->time; } - // TODO(yangchi): pass the event time in to elimite the process delay if (lossEvent.largestLostPacketNum.hasValue()) { + DCHECK(lossEvent.largestLostSentTime && lossEvent.smallestLostSentTime); QUIC_TRACE( packets_lost, conn, @@ -370,19 +373,24 @@ void onLossDetectionAlarm( VLOG(10) << "Transmission alarm fired with no outstanding packets " << conn; return; } + // TODO: The specs prioritize EarlyRetransmitOrReordering over crypto timer. if (conn.lossState.currentAlarmMethod == LossState::AlarmMethod::Handshake) { onHandshakeAlarm(conn, lossVisitor); } else if ( conn.lossState.currentAlarmMethod == LossState::AlarmMethod::EarlyRetransmitOrReordering) { - auto lossEvent = detectLossPackets( + auto lossEvent = detectLossPackets( conn, getAckState(conn, PacketNumberSpace::AppData).largestAckedByPeer, lossVisitor, now, - folly::none, PacketNumberSpace::AppData); if (conn.congestionController && lossEvent) { + DCHECK(lossEvent->largestLostSentTime && lossEvent->smallestLostSentTime); + lossEvent->persistentCongestion = isPersistentCongestion( + conn, + *lossEvent->smallestLostSentTime, + *lossEvent->largestLostSentTime); conn.congestionController->onPacketAckOrLoss( folly::none, std::move(lossEvent)); } @@ -416,28 +424,12 @@ folly::Optional handleAckForLoss( const LossVisitor& lossVisitor, CongestionController::AckEvent& ack, PacketNumberSpace pnSpace) { - folly::Optional rtoVerifiedPacket; auto& largestAcked = getAckState(conn, pnSpace).largestAckedByPeer; if (ack.largestAckedPacket.hasValue()) { - // LargestAcked is larger than largest one in largestSentBeforeRto, notify - // onRTOVerified: - if (conn.lossState.rtoCount > 0 && conn.lossState.largestSentBeforeRto && - *ack.largestAckedPacket > *conn.lossState.largestSentBeforeRto) { - QUIC_TRACE( - rto_verified, - conn, - *ack.largestAckedPacket, - *conn.lossState.largestSentBeforeRto, - (uint64_t)conn.outstandingPackets.size()); - if (conn.congestionController) { - conn.congestionController->onRTOVerified(); - } - rtoVerifiedPacket = *ack.largestAckedPacket; - } - // TODO: Should we NOT reset these counters if the received Ack - // frame doesn't ack anything that's in OP list? + // TODO: Should we NOT reset these counters if the received Ack frame + // doesn't ack anything that's in OP list? conn.lossState.rtoCount = 0; - conn.lossState.largestSentBeforeRto = folly::none; + conn.lossState.handshakeAlarmCount = 0; largestAcked = std::max(largestAcked, *ack.largestAckedPacket); } auto lossEvent = detectLossPackets( @@ -445,7 +437,6 @@ folly::Optional handleAckForLoss( getAckState(conn, pnSpace).largestAckedByPeer, lossVisitor, ack.ackTime, - std::move(rtoVerifiedPacket), pnSpace); conn.pendingEvents.setLossDetectionAlarm = (conn.outstandingPackets.size() > conn.outstandingPureAckPacketsCount); diff --git a/quic/loss/TARGETS b/quic/loss/TARGETS deleted file mode 100644 index 5c4340fa3..000000000 --- a/quic/loss/TARGETS +++ /dev/null @@ -1,25 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") - -cpp_library( - name = "loss", - srcs = [ - "QuicLossFunctions.cpp", - ], - headers = [ - "QuicLossFunctions.h", - ], - deps = [ - "//folly:overload", - "//folly/io/async:async_base", - "//quic:constants", - "//quic/codec:types", - "//quic/common:time_util", - "//quic/flowcontrol:flow_control", - "//quic/logging:logging", - "//quic/state:simple_frame_functions", - "//quic/state:state_functions", - "//quic/state:state_machine", - ], -) diff --git a/quic/loss/test/QuicLossFunctionsTest.cpp b/quic/loss/test/QuicLossFunctionsTest.cpp index 9e287d1da..1ba4cc59b 100644 --- a/quic/loss/test/QuicLossFunctionsTest.cpp +++ b/quic/loss/test/QuicLossFunctionsTest.cpp @@ -189,6 +189,8 @@ PacketNum QuicLossFunctionsTest::sendPacket( } if (pureAck) { conn.outstandingPureAckPacketsCount++; + } else { + conn.lossState.lastRetransmittablePacketSentTime = time; } if (!pureAck && conn.congestionController) { conn.congestionController->onPacketSent(outstandingPacket); @@ -295,17 +297,15 @@ TEST_F(QuicLossFunctionsTest, TestOnLossDetectionAlarm) { EXPECT_CALL(*rawCongestionController, onPacketSent(_)) .WillRepeatedly(Return()); + sendPacket(*conn, Clock::now(), false, folly::none, PacketType::OneRtt); MockClock::mockNow = []() { return TimePoint(123ms); }; std::vector lostPacket; MockClock::mockNow = []() { return TimePoint(23ms); }; EXPECT_CALL(*transportInfoCb_, onRTO()); - auto expectedLargestSentBeforeRto = - sendPacket(*conn, TimePoint(), false, folly::none, PacketType::OneRtt); setLossDetectionAlarm(*conn, timeout); + EXPECT_EQ(LossState::AlarmMethod::RTO, conn->lossState.currentAlarmMethod); onLossDetectionAlarm( *conn, testingLossMarkFunc(lostPacket)); - EXPECT_EQ( - *conn->lossState.largestSentBeforeRto, expectedLargestSentBeforeRto); EXPECT_EQ(conn->lossState.rtoCount, 1); EXPECT_TRUE(conn->pendingEvents.setLossDetectionAlarm); // RTO shouldn't mark loss @@ -318,8 +318,6 @@ TEST_F(QuicLossFunctionsTest, TestOnLossDetectionAlarm) { EXPECT_CALL(*rawCongestionController, onPacketAckOrLoss(_, _)).Times(0); onLossDetectionAlarm( *conn, testingLossMarkFunc(lostPacket)); - EXPECT_EQ( - *conn->lossState.largestSentBeforeRto, expectedLargestSentBeforeRto); EXPECT_EQ(conn->lossState.rtoCount, 2); // RTO doesn't take anything out of outstandingPackets EXPECT_FALSE(conn->outstandingPackets.empty()); @@ -430,47 +428,6 @@ TEST_F(QuicLossFunctionsTest, TestMarkCryptoLostAfterCancelRetransmission) { EXPECT_EQ(conn->cryptoState->handshakeStream.lossBuffer.size(), 0); } -TEST_F(QuicLossFunctionsTest, RTOVerifiedMarksAllPacketsBelowAckedLost) { - auto conn = createConn(); - auto mockCongestionController = std::make_unique(); - auto rawCongestionController = mockCongestionController.get(); - conn->congestionController = std::move(mockCongestionController); - - std::vector packets; - for (int i = 0; i < 6; ++i) { - packets.emplace_back(sendPacket( - *conn, Clock::now(), false, folly::none, PacketType::OneRtt)); - } - - ASSERT_EQ(conn->outstandingPackets.size(), packets.size()); - PacketNum largestPacket = packets.back(); - conn->lossState.largestSentBeforeRto = packets.at(packets.size() - 2); - conn->lossState.rtoCount = 1; - - CongestionController::AckEvent ackEvent; - ackEvent.ackTime = Clock::now(); - ackEvent.largestAckedPacket = packets.at(packets.size() - 1); - - std::set lostPackets; - auto markingLossVisitor = - [&](auto& /* conn */, auto&, bool, PacketNum packetNum) { - lostPackets.emplace(packetNum); - }; - - EXPECT_CALL(*rawCongestionController, onRTOVerified()).Times(1); - handleAckForLoss( - *conn, markingLossVisitor, ackEvent, PacketNumberSpace::AppData); - EXPECT_EQ(conn->lossState.rtoCount, 0); - EXPECT_FALSE(conn->lossState.largestSentBeforeRto.hasValue()); - EXPECT_EQ(lostPackets.size(), packets.size() - 1); - ASSERT_EQ(1, conn->outstandingPackets.size()); - auto outstandingPacketNum = folly::variant_match( - getFirstOutstandingPacket(*conn, PacketNumberSpace::AppData) - ->packet.header, - [](auto& h) { return h.getPacketSequenceNum(); }); - EXPECT_EQ(outstandingPacketNum, largestPacket); -} - TEST_F(QuicLossFunctionsTest, TestMarkPacketLossAfterStreamReset) { folly::EventBase evb; MockAsyncUDPSocket socket(&evb); @@ -487,7 +444,9 @@ TEST_F(QuicLossFunctionsTest, TestMarkPacketLossAfterStreamReset) { *buf, true); invokeStreamStateMachine( - *conn, *stream1, StreamEvents::SendReset(ApplicationErrorCode::STOPPING)); + *conn, + *stream1, + StreamEvents::SendReset(GenericApplicationErrorCode::UNKNOWN)); markPacketLoss( *conn, @@ -543,13 +502,11 @@ TEST_F(QuicLossFunctionsTest, TestReorderingThreshold) { conn->outstandingPackets.erase( firstHandshakeOpIter + 2, firstHandshakeOpIter + 5); // Ack for packet 9 arrives - MockClock::mockNow = []() { return TimePoint(70ms); }; - auto lossEvent = detectLossPackets( + auto lossEvent = detectLossPackets( *conn, 9, testingLossMarkFunc, TimePoint(90ms), - folly::none, PacketNumberSpace::Handshake); EXPECT_EQ(2, lossEvent->largestLostPacketNum.value()); EXPECT_EQ(TimePoint(90ms), lossEvent->lossTime); @@ -607,6 +564,7 @@ TEST_F(QuicLossFunctionsTest, TestHandleAckForLoss) { TEST_F(QuicLossFunctionsTest, TestHandleAckedPacket) { auto conn = createConn(); conn->lossState.rtoCount = 10; + conn->lossState.handshakeAlarmCount = 5; conn->lossState.reorderingThreshold = 10; sendPacket(*conn, TimePoint(), false, folly::none, PacketType::OneRtt); @@ -633,6 +591,7 @@ TEST_F(QuicLossFunctionsTest, TestHandleAckedPacket) { Clock::now()); EXPECT_EQ(0, conn->lossState.rtoCount); + EXPECT_EQ(0, conn->lossState.handshakeAlarmCount); EXPECT_TRUE(conn->outstandingPackets.empty()); EXPECT_EQ(0, conn->outstandingPureAckPacketsCount); EXPECT_FALSE(conn->pendingEvents.setLossDetectionAlarm); @@ -651,7 +610,7 @@ TEST_F(QuicLossFunctionsTest, TestMarkRstLoss) { auto stream = conn->streamManager->createNextBidirectionalStream().value(); auto currentOffset = stream->currentWriteOffset; RstStreamFrame rstFrame( - stream->id, ApplicationErrorCode::STOPPING, currentOffset); + stream->id, GenericApplicationErrorCode::UNKNOWN, currentOffset); conn->pendingEvents.resets.insert({stream->id, rstFrame}); writeQuicDataToSocket( socket, @@ -679,7 +638,7 @@ TEST_F(QuicLossFunctionsTest, TestMarkRstLoss) { EXPECT_EQ(1, conn->pendingEvents.resets.count(stream->id)); auto& retxRstFrame = conn->pendingEvents.resets.at(stream->id); EXPECT_EQ(stream->id, retxRstFrame.streamId); - EXPECT_EQ(ApplicationErrorCode::STOPPING, retxRstFrame.errorCode); + EXPECT_EQ(GenericApplicationErrorCode::UNKNOWN, retxRstFrame.errorCode); EXPECT_EQ(currentOffset, retxRstFrame.offset); // write again: @@ -698,7 +657,7 @@ TEST_F(QuicLossFunctionsTest, TestMarkRstLoss) { bool rstFound = false; for (auto& frame : all_frames(packet2.frames)) { EXPECT_EQ(stream->id, frame.streamId); - EXPECT_EQ(ApplicationErrorCode::STOPPING, frame.errorCode); + EXPECT_EQ(GenericApplicationErrorCode::UNKNOWN, frame.errorCode); EXPECT_EQ(currentOffset, frame.offset); rstFound = true; } @@ -727,7 +686,6 @@ TEST_F(QuicLossFunctionsTest, ReorderingThresholdChecksSamePacketNumberSpace) { latestSent + 1, countingLossVisitor, Clock::now(), - folly::none, PacketNumberSpace::AppData); EXPECT_EQ(0, lossVisitorCount); @@ -736,7 +694,6 @@ TEST_F(QuicLossFunctionsTest, ReorderingThresholdChecksSamePacketNumberSpace) { latestSent + 1, countingLossVisitor, Clock::now(), - folly::none, PacketNumberSpace::Handshake); EXPECT_GT(lossVisitorCount, 0); } @@ -772,32 +729,29 @@ TEST_F(QuicLossFunctionsTest, TestMarkWindowUpdateLoss) { TEST_F(QuicLossFunctionsTest, TestTimeReordering) { std::vector lostPacket; auto conn = createConn(); - conn->lossState.lossMode = LossState::LossMode::TimeLossDetection; auto mockCongestionController = std::make_unique(); auto rawCongestionController = mockCongestionController.get(); conn->congestionController = std::move(mockCongestionController); EXPECT_CALL(*rawCongestionController, onPacketSent(_)) .WillRepeatedly(Return()); + PacketNum largestSent = 0; for (int i = 0; i < 7; ++i) { - sendPacket( + largestSent = sendPacket( *conn, TimePoint(i * 100ms), false, folly::none, PacketType::OneRtt); } // Some packets are already acked - conn->lossState.srtt = 200ms; - conn->lossState.lrtt = 150ms; + conn->lossState.srtt = 400ms; + conn->lossState.lrtt = 350ms; conn->outstandingPackets.erase( getFirstOutstandingPacket(*conn, PacketNumberSpace::AppData) + 2, getFirstOutstandingPacket(*conn, PacketNumberSpace::AppData) + 5); - MockClock::mockNow = []() { return TimePoint(700ms); }; - auto lossEvent = - detectLossPackets( - *conn, - 9, - testingLossMarkFunc(lostPacket), - TimePoint(900ms), - folly::none, - PacketNumberSpace::AppData); + auto lossEvent = detectLossPackets( + *conn, + largestSent, + testingLossMarkFunc(lostPacket), + TimePoint(900ms), + PacketNumberSpace::AppData); EXPECT_EQ(2, lossEvent->largestLostPacketNum.value()); EXPECT_EQ(TimePoint(900ms), lossEvent->lossTime); // Packet 1,2 should be marked as loss @@ -813,7 +767,6 @@ TEST_F(QuicLossFunctionsTest, TestTimeReordering) { [](const auto& h) { return h.getPacketSequenceNum(); }); EXPECT_EQ(packetNum, 6); EXPECT_TRUE(conn->lossState.lossTime); - EXPECT_EQ(*conn->lossState.lossTime, TimePoint(725ms)); } TEST_F(QuicLossFunctionsTest, RTONoLongerMarksPacketsToBeRetransmitted) { @@ -926,41 +879,18 @@ TEST_F(QuicLossFunctionsTest, PureAckSkipsCongestionControl) { }); // Ack for packet 9 arrives - MockClock::mockNow = []() { return TimePoint(70ms); }; - auto lossEvent = - detectLossPackets( - *conn, - 9, - testingLossMarkFunc(lostPacket), - TimePoint(80ms), - folly::none, - PacketNumberSpace::Handshake); + auto lossEvent = detectLossPackets( + *conn, + 9, + testingLossMarkFunc(lostPacket), + TimePoint(80ms), + PacketNumberSpace::Handshake); EXPECT_EQ(5, lossEvent->largestLostPacketNum.value()); EXPECT_EQ(sumNoPureAckBytes, lossEvent->lostBytes); EXPECT_EQ(TimePoint(80ms), lossEvent->lossTime); EXPECT_EQ(conn->outstandingHandshakePacketsCount, 0); } -TEST_F(QuicLossFunctionsTest, DetectPacketLossWithRTOVerification) { - auto conn = createConn(); - sendPacket(*conn, TimePoint(), false, folly::none, PacketType::OneRtt); - sendPacket(*conn, TimePoint(), false, folly::none, PacketType::OneRtt); - sendPacket(*conn, TimePoint(), false, folly::none, PacketType::OneRtt); - auto packet4 = - sendPacket(*conn, TimePoint(), false, folly::none, PacketType::OneRtt); - auto packet5 = - sendPacket(*conn, TimePoint(), false, folly::none, PacketType::OneRtt); - std::vector lostPackets; - detectLossPackets( - *conn, - packet5, - testingLossMarkFunc(lostPackets), - TimePoint(100ms), - packet4, - PacketNumberSpace::AppData); - EXPECT_EQ(4, lostPackets.size()); -} - TEST_F(QuicLossFunctionsTest, EmptyOutstandingNoTimeout) { auto conn = createConn(); EXPECT_CALL(timeout, cancelLossTimeout()).Times(1); @@ -980,31 +910,24 @@ TEST_F(QuicLossFunctionsTest, AlarmDurationHandshakeOutstanding) { MockClock::mockNow = [=]() { return thisMoment; }; auto duration = calculateAlarmDuration(*conn); EXPECT_EQ( - std::max( - kDefaultInitialRtt * 2 + conn->lossState.maxAckDelay, - kMinTLPTimeout) - - packetSentDelay, + kDefaultInitialRtt * 2 - packetSentDelay + std::chrono::milliseconds(25), duration.first); EXPECT_EQ(duration.second, LossState::AlarmMethod::Handshake); - conn->lossState.srtt = std::chrono::microseconds(100); + conn->lossState.srtt = std::chrono::milliseconds(100); duration = calculateAlarmDuration(*conn); EXPECT_EQ( - std::chrono::duration_cast(std::max( - std::chrono::microseconds(200 + 25 * 1000), - std::chrono::duration_cast( - kMinTLPTimeout))) - + std::chrono::duration_cast( + std::chrono::milliseconds(225)) - packetSentDelay, duration.first); + conn->lossState.maxAckDelay = std::chrono::milliseconds(45); conn->lossState.handshakeAlarmCount = 2; duration = calculateAlarmDuration(*conn); EXPECT_EQ( - std::chrono::duration_cast(std::max( - std::chrono::microseconds(200 + 25 * 1000), - std::chrono::duration_cast( - kMinTLPTimeout))) * - 4 - + std::chrono::duration_cast( + std::chrono::milliseconds(980)) - packetSentDelay, duration.first); } @@ -1088,7 +1011,6 @@ TEST_F(QuicLossFunctionsTest, NoSkipLossVisitor) { lastSent, countingLossVisitor, TimePoint(100ms), - folly::none, PacketNumberSpace::AppData); EXPECT_EQ(1, lossVisitorCount); } @@ -1118,7 +1040,6 @@ TEST_F(QuicLossFunctionsTest, SkipLossVisitor) { lastSent, countingLossVisitor, TimePoint(100ms), - folly::none, PacketNumberSpace::AppData); EXPECT_EQ(0, lossVisitorCount); } @@ -1156,7 +1077,6 @@ TEST_F(QuicLossFunctionsTest, NoDoubleProcess) { lastSent, countingLossVisitor, TimePoint(100ms), - folly::none, PacketNumberSpace::AppData); EXPECT_EQ(1, lossVisitorCount); EXPECT_EQ(4, conn->outstandingPackets.size()); @@ -1172,12 +1092,11 @@ TEST_F(QuicLossFunctionsTest, DetectPacketLossClonedPacketsCounter) { auto ackedPacket = sendPacket(*conn, Clock::now(), false, folly::none, PacketType::OneRtt); auto noopLossMarker = [](auto&, auto&, bool, PacketNum) {}; - detectLossPackets( + detectLossPackets( *conn, ackedPacket, noopLossMarker, Clock::now(), - folly::none, PacketNumberSpace::AppData); EXPECT_EQ(0, conn->outstandingClonedPacketsCount); } @@ -1274,7 +1193,6 @@ TEST_F(QuicLossFunctionsTest, TotalLossCount) { largestSent, countingLossVisitor, TimePoint(100ms), - folly::none, PacketNumberSpace::AppData); EXPECT_EQ(135 + lostPackets, conn->lossState.rtxCount); } @@ -1374,42 +1292,35 @@ TEST_F(QuicLossFunctionsTest, RTOLargerThanMaxDelay) { EXPECT_GE(calculateRTO(conn), std::chrono::seconds(20)); } -TEST_P(QuicLossFunctionsTest, RTOVerificationNotKickInWithoutRTOCount) { +TEST_F(QuicLossFunctionsTest, TimeThreshold) { auto conn = createConn(); - auto mockCongestionController = std::make_unique(); - auto rawCongestionController = mockCongestionController.get(); - conn->congestionController = std::move(mockCongestionController); - conn->lossState.largestSentBeforeRto = 100; - - CongestionController::AckEvent ackEvent; - ackEvent.ackTime = Clock::now(); - ackEvent.largestAckedPacket = 101; - auto noopLossVisitor = [&](auto& /* conn */, auto&, bool, PacketNum) {}; - - EXPECT_CALL(*transportInfoCb_, onRTO()).Times(0); - EXPECT_CALL(*rawCongestionController, onRTOVerified()).Times(0); - handleAckForLoss(*conn, noopLossVisitor, ackEvent, GetParam()); - EXPECT_EQ(0, conn->lossState.rtoCount); - EXPECT_FALSE(conn->lossState.largestSentBeforeRto.hasValue()); -} - -TEST_P(QuicLossFunctionsTest, RTOVerification) { - auto conn = createConn(); - auto mockCongestionController = std::make_unique(); - auto rawCongestionController = mockCongestionController.get(); - conn->congestionController = std::move(mockCongestionController); - conn->lossState.largestSentBeforeRto = 100; - conn->lossState.rtoCount = 1; - - CongestionController::AckEvent ackEvent; - ackEvent.ackTime = Clock::now(); - ackEvent.largestAckedPacket = 101; - auto noopLossVisitor = [&](auto& /* conn */, auto&, bool, PacketNum) {}; - - EXPECT_CALL(*rawCongestionController, onRTOVerified()).Times(1); - handleAckForLoss(*conn, noopLossVisitor, ackEvent, GetParam()); - EXPECT_EQ(0, conn->lossState.rtoCount); - EXPECT_FALSE(conn->lossState.largestSentBeforeRto.hasValue()); + conn->lossState.srtt = std::chrono::milliseconds(10); + auto referenceTime = Clock::now(); + auto packet1 = sendPacket( + *conn, + referenceTime - std::chrono::milliseconds(10), + false, + folly::none, + PacketType::OneRtt); + auto packet2 = sendPacket( + *conn, + referenceTime + conn->lossState.srtt / 2, + false, + folly::none, + PacketType::OneRtt); + auto lossVisitor = [&](const auto& /*conn*/, + const auto& /*packet*/, + bool, + PacketNum packetNum) { + EXPECT_EQ(packet1, packetNum); + }; + detectLossPackets( + *conn, + packet2, + lossVisitor, + referenceTime + conn->lossState.srtt * 9 / 8 + + std::chrono::milliseconds(5), + PacketNumberSpace::AppData); } TEST_P(QuicLossFunctionsTest, CappedShiftNoCrash) { @@ -1429,6 +1340,17 @@ TEST_P(QuicLossFunctionsTest, CappedShiftNoCrash) { calculateAlarmDuration(*conn); } +TEST_F(QuicLossFunctionsTest, PersistentCongestion) { + auto conn = createConn(); + EXPECT_FALSE(isPersistentCongestion( + *conn, Clock::now() - std::chrono::seconds(10), Clock::now())); + conn->lossState.srtt = std::chrono::seconds(1); + EXPECT_TRUE(isPersistentCongestion( + *conn, Clock::now() - std::chrono::seconds(10), Clock::now())); + EXPECT_FALSE(isPersistentCongestion( + *conn, Clock::now() - std::chrono::milliseconds(100), Clock::now())); +} + INSTANTIATE_TEST_CASE_P( QuicLossFunctionsTests, QuicLossFunctionsTest, diff --git a/quic/loss/test/TARGETS b/quic/loss/test/TARGETS deleted file mode 100644 index a31ad5edf..000000000 --- a/quic/loss/test/TARGETS +++ /dev/null @@ -1,23 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_unittest.bzl", "cpp_unittest") - -cpp_unittest( - name = "QuicLossFunctionsTest", - srcs = [ - "QuicLossFunctionsTest.cpp", - ], - deps = [ - "//folly/io/async/test:mocks", - "//folly/portability:gmock", - "//folly/portability:gtest", - "//quic/api:transport", - "//quic/api/test:mocks", - "//quic/client/state:client", - "//quic/codec:types", - "//quic/common/test:test_utils", - "//quic/loss:loss", - "//quic/server/state:server", - "//quic/state/test:mocks", - ], -) diff --git a/quic/samples/echo/TARGETS b/quic/samples/echo/TARGETS deleted file mode 100644 index 26031bb4d..000000000 --- a/quic/samples/echo/TARGETS +++ /dev/null @@ -1,40 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_binary.bzl", "cpp_binary") -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") - -cpp_library( - name = "EchoHandler", - headers = [ - "EchoClient.h", - "EchoHandler.h", - "EchoServer.h", - ], - deps = [ - "//folly/io:iobuf", - "//folly/io/async:async_base", - "//folly/io/async:scoped_event_base_thread", - "//quic/api:transport", - "//quic/client:client", - "//quic/common/test:test_utils", - "//quic/server:server", - ], - external_deps = [ - "glog", - ], -) - -cpp_binary( - name = "echo", - srcs = [ - "main.cpp", - ], - deps = [ - ":EchoHandler", - "//folly/init:init", - ], - external_deps = [ - "gflags", - "glog", - ], -) diff --git a/quic/server/TARGETS b/quic/server/TARGETS deleted file mode 100644 index c8edb44be..000000000 --- a/quic/server/TARGETS +++ /dev/null @@ -1,43 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") - -cpp_library( - name = "server", - srcs = [ - "QuicServer.cpp", - "QuicServerPacketRouter.cpp", - "QuicServerTransport.cpp", - "QuicServerWorker.cpp", - ], - headers = [ - "QuicReusePortUDPSocketFactory.h", - "QuicServer.h", - "QuicServerPacketRouter.h", - "QuicServerTransport.h", - "QuicServerTransportFactory.h", - "QuicServerWorker.h", - "QuicSharedUDPSocketFactory.h", - "QuicUDPSocketFactory.h", - ], - deps = [ - "//fizz/server:fizz_server_context", - "//folly:thread_local", - "//folly/io:iobuf", - "//folly/io/async:async_udp_socket", - "//folly/io/async:event_base_manager", - "//folly/io/async:scoped_event_base_thread", - "//folly/system:thread_id", - "//quic:constants", - "//quic/api:transport", - "//quic/codec:header_codec", - "//quic/codec:types", - "//quic/common:timers", - "//quic/congestion_control:congestion_controller_factory", - "//quic/server/handshake:app_token", - "//quic/server/handshake:default_app_token_validator", - "//quic/server/handshake:server_extension", - "//quic/server/state:server", - "//quic/state:stats_callback", - ], -) diff --git a/quic/server/handshake/TARGETS b/quic/server/handshake/TARGETS deleted file mode 100644 index d80af3df3..000000000 --- a/quic/server/handshake/TARGETS +++ /dev/null @@ -1,101 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") - -cpp_library( - name = "server_handshake", - srcs = [ - "ServerHandshake.cpp", - ], - headers = [ - "ServerHandshake.h", - ], - deps = [ - ":app_token", - ":server_extension", - "//fizz/protocol:default_certificate_verifier", - "//fizz/protocol:protocol", - "//fizz/server:fizz_server", - "//fizz/server:fizz_server_context", - "//fizz/server:protocol", - "//folly/io:iobuf", - "//folly/io/async:delayed_destruction", - "//quic:constants", - "//quic:exception", - "//quic/handshake:handshake", - "//quic/state:state_machine", - "//quic/state:stream_functions", - ], -) - -cpp_library( - name = "server_extension", - headers = [ - "ServerTransportParametersExtension.h", - ], - deps = [ - "//fizz/server:server_extensions", - "//quic/handshake:transport_parameters", - ], -) - -cpp_library( - name = "app_token", - srcs = [ - "AppToken.cpp", - ], - headers = [ - "AppToken.h", - ], - deps = [ - "//fizz/record:record", - "//fizz/server:protocol", - "//folly:network_address", - "//folly:optional", - "//folly/io:iobuf", - "//quic:constants", - "//quic/handshake:transport_parameters", - ], - external_deps = [ - "glog", - ], -) - -cpp_library( - name = "default_app_token_validator", - srcs = [ - "DefaultAppTokenValidator.cpp", - ], - headers = [ - "DefaultAppTokenValidator.h", - ], - deps = [ - "//fizz/server:protocol", - "//fizz/server:resumption_state", - "//folly:network_address", - "//folly:optional", - "//quic:constants", - "//quic/api:transport", - "//quic/handshake:transport_parameters", - "//quic/server/state:server", - ], - external_deps = [ - "glog", - ], -) - -cpp_library( - name = "stateless_reset_generator", - srcs = [ - "StatelessResetGenerator.cpp", - ], - headers = [ - "StatelessResetGenerator.h", - ], - deps = [ - "//fizz/crypto:hkdf", - "//fizz/crypto:hmac_sha256", - "//folly:range", - "//quic/codec:types", - ], -) diff --git a/quic/server/handshake/test/TARGETS b/quic/server/handshake/test/TARGETS deleted file mode 100644 index 287866843..000000000 --- a/quic/server/handshake/test/TARGETS +++ /dev/null @@ -1,90 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_unittest.bzl", "cpp_unittest") - -cpp_unittest( - name = "ServerHandshakeTest", - srcs = [ - "ServerHandshakeTest.cpp", - ], - deps = [ - "//fizz/client/test:mocks", - "//fizz/crypto/test:TestUtil", - "//fizz/protocol/test:mocks", - "//fizz/server/test:mocks", - "//folly/io/async:scoped_event_base_thread", - "//folly/io/async:ssl_context", - "//folly/io/async/test:mocks", - "//folly/ssl:init", - "//quic/client/handshake:client_extension", - "//quic/common/test:test_utils", - "//quic/handshake:handshake", - "//quic/server/handshake:app_token", - "//quic/server/handshake:server_handshake", - "//quic/state:state_machine", - ], - external_deps = [ - ("googletest", None, "gmock"), - ], -) - -cpp_unittest( - name = "ServerTransportParametersTest", - srcs = [ - "ServerTransportParametersTest.cpp", - ], - deps = [ - "//fizz/protocol/test:test_messages", - "//quic:constants", - "//quic/common/test:test_utils", - "//quic/server/handshake:server_extension", - ], - external_deps = [ - ("googletest", None, "gmock"), - ], -) - -cpp_unittest( - name = "AppTokenTest", - srcs = [ - "AppTokenTest.cpp", - ], - deps = [ - "//fizz/server:resumption_state", - "//folly:optional", - "//quic:constants", - "//quic/server/handshake:app_token", - "//quic/server/state:server", - ], -) - -cpp_unittest( - name = "DefaultAppTokenValidatorTest", - srcs = [ - "DefaultAppTokenValidatorTest.cpp", - ], - deps = [ - "//fizz/server:resumption_state", - "//folly:optional", - "//quic:constants", - "//quic/api/test:mocks", - "//quic/server/handshake:default_app_token_validator", - "//quic/server/state:server", - ], - external_deps = [ - ("googletest", None, "gmock"), - ], -) - -cpp_unittest( - name = "StatelessResetGeneratorTest", - srcs = [ - "StatelessResetGeneratorTest.cpp", - ], - deps = [ - "//folly:network_address", - "//folly:random", - "//folly/portability:gtest", - "//quic/server/handshake:stateless_reset_generator", - ], -) diff --git a/quic/server/state/TARGETS b/quic/server/state/TARGETS deleted file mode 100644 index eefa3ebd1..000000000 --- a/quic/server/state/TARGETS +++ /dev/null @@ -1,43 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") - -cpp_library( - name = "server", - srcs = [ - "ServerStateMachine.cpp", - ], - headers = [ - "ServerStateMachine.h", - "ServerStateMachine-inl.h", - ], - deps = [ - "//folly:exception_wrapper", - "//folly:network_address", - "//folly:overload", - "//folly:random", - "//folly/io/async:async_socket_exception", - "//quic:exception", - "//quic/codec:types", - "//quic/congestion_control:congestion_controller_factory", - "//quic/congestion_control:cubic", - "//quic/flowcontrol:flow_control", - "//quic/handshake:transport_parameters", - "//quic/logging:logging", - "//quic/loss:loss", - "//quic/server/handshake:server_handshake", - "//quic/state:ack_handler", - "//quic/state:pacing_functions", - "//quic/state:qpr_functions", - "//quic/state:simple_frame_functions", - "//quic/state:state_functions", - "//quic/state:state_machine", - "//quic/state:stats_callback", - "//quic/state:stream_functions", - "//quic/state/stream:stream", - ], - external_deps = [ - "boost", - "glog", - ], -) diff --git a/quic/server/test/QuicServerTransportTest.cpp b/quic/server/test/QuicServerTransportTest.cpp index 5a8121b0b..d4fa16910 100644 --- a/quic/server/test/QuicServerTransportTest.cpp +++ b/quic/server/test/QuicServerTransportTest.cpp @@ -808,14 +808,16 @@ TEST_F(QuicServerTransportTest, RecvDataAfterIdleTimeout) { TEST_F(QuicServerTransportTest, TestCloseConnectionWithError) { server->close(std::make_pair( - QuicErrorCode(ApplicationErrorCode::STOPPING), std::string("stopping"))); + QuicErrorCode(GenericApplicationErrorCode::UNKNOWN), + std::string("stopping"))); EXPECT_TRUE(verifyFramePresent( serverWrites, *makeClientEncryptedCodec())); } TEST_F(QuicServerTransportTest, TestCloseConnectionWithNoError) { server->close(std::make_pair( - QuicErrorCode(ApplicationErrorCode::STOPPING), std::string("stopping"))); + QuicErrorCode(GenericApplicationErrorCode::UNKNOWN), + std::string("stopping"))); EXPECT_TRUE(verifyFramePresent( serverWrites, *makeClientEncryptedCodec())); } @@ -854,7 +856,8 @@ TEST_F(QuicServerTransportTest, TestCloseConnectionWithNoErrorPendingStreams) { acks, PacketNumberSpace::AppData))); server->close(std::make_pair( - QuicErrorCode(ApplicationErrorCode::STOPPING), std::string("stopping"))); + QuicErrorCode(GenericApplicationErrorCode::UNKNOWN), + std::string("stopping"))); EXPECT_THROW( recvEncryptedStream(streamId, *IOBuf::copyBuffer("hello")), @@ -876,7 +879,7 @@ TEST_F(QuicServerTransportTest, ReceivePacketAfterLocalError) { // Deliver a reset to non existent stream to trigger a local conn error StreamId streamId = 0x01; - RstStreamFrame rstFrame(streamId, ApplicationErrorCode::STOPPING, 0); + RstStreamFrame rstFrame(streamId, GenericApplicationErrorCode::UNKNOWN, 0); writeFrame(std::move(rstFrame), builder); auto packet = std::move(builder).buildPacket(); deliverDataWithoutErrorCheck(packetToBuf(packet)); @@ -892,7 +895,7 @@ TEST_F(QuicServerTransportTest, ReceivePacketAfterLocalError) { server->getConn().udpSendPacketLen, std::move(header2), 0 /* largestAcked */); - RstStreamFrame rstFrame2(streamId, ApplicationErrorCode::STOPPING, 0); + RstStreamFrame rstFrame2(streamId, GenericApplicationErrorCode::UNKNOWN, 0); writeFrame(std::move(rstFrame2), builder2); auto packet2 = std::move(builder2).buildPacket(); deliverDataWithoutErrorCheck(packetToBuf(packet2)); @@ -912,7 +915,7 @@ TEST_F(QuicServerTransportTest, ReceiveCloseAfterLocalError) { // Deliver a reset to non existent stream to trigger a local conn error StreamId streamId = 0x01; - RstStreamFrame rstFrame(streamId, ApplicationErrorCode::STOPPING, 0); + RstStreamFrame rstFrame(streamId, GenericApplicationErrorCode::UNKNOWN, 0); writeFrame(std::move(rstFrame), builder); auto packet = std::move(builder).buildPacket(); deliverDataWithoutErrorCheck(packetToBuf(packet)); @@ -981,7 +984,8 @@ TEST_F(QuicServerTransportTest, NoDataExceptCloseProcessedAfterClosing) { auto packet = std::move(builder).buildPacket(); server->close(std::make_pair( - QuicErrorCode(ApplicationErrorCode::STOPPING), std::string("hello"))); + QuicErrorCode(GenericApplicationErrorCode::UNKNOWN), + std::string("hello"))); EXPECT_TRUE(verifyFramePresent( serverWrites, *makeClientEncryptedCodec())); EXPECT_TRUE(hasNotReceivedNewPacketsSinceLastCloseSent(server->getConn())); @@ -1134,7 +1138,7 @@ TEST_F(QuicServerTransportTest, RecvRstStreamFrameNonexistClientStream) { 0 /* largestAcked */); ASSERT_TRUE(builder.canBuildPacket()); - RstStreamFrame rstFrame(streamId, ApplicationErrorCode::STOPPING, 0); + RstStreamFrame rstFrame(streamId, GenericApplicationErrorCode::UNKNOWN, 0); writeFrame(std::move(rstFrame), builder); auto packet = std::move(builder).buildPacket(); deliverData(packetToBuf(packet)); @@ -1155,7 +1159,7 @@ TEST_F(QuicServerTransportTest, RecvRstStreamFrameNonexistServerStream) { ASSERT_TRUE(builder.canBuildPacket()); StreamId streamId = 0x01; - RstStreamFrame rstFrame(streamId, ApplicationErrorCode::STOPPING, 0); + RstStreamFrame rstFrame(streamId, GenericApplicationErrorCode::UNKNOWN, 0); writeFrame(std::move(rstFrame), builder); auto packet = std::move(builder).buildPacket(); EXPECT_THROW(deliverData(packetToBuf(packet)), std::runtime_error); @@ -1193,7 +1197,7 @@ TEST_F(QuicServerTransportTest, RecvRstStreamFrame) { RstStreamFrame rstFrame( streamId, - ApplicationErrorCode::STOPPING, + GenericApplicationErrorCode::UNKNOWN, words.at(0).length() + words.at(1).length()); ASSERT_TRUE(builder.canBuildPacket()); writeFrame(std::move(rstFrame), builder); @@ -1214,7 +1218,7 @@ TEST_F(QuicServerTransportTest, RecvRstStreamFrame) { } } ASSERT_NE(rstStreamFrame, nullptr); - EXPECT_EQ(ApplicationErrorCode::STOPPING, rstStreamFrame->errorCode); + EXPECT_EQ(GenericApplicationErrorCode::NO_ERROR, rstStreamFrame->errorCode); EXPECT_EQ(streamId, rstStreamFrame->streamId); EXPECT_EQ( words.at(2).length() + words.at(3).length(), rstStreamFrame->offset); @@ -1265,12 +1269,14 @@ TEST_F(QuicServerTransportTest, RecvStopSendingFrame) { std::move(header), 0 /* largestAcked */); - StopSendingFrame stopSendingFrame(streamId, ApplicationErrorCode::STOPPING); + StopSendingFrame stopSendingFrame( + streamId, GenericApplicationErrorCode::UNKNOWN); ASSERT_TRUE(builder.canBuildPacket()); writeFrame(std::move(stopSendingFrame), builder); auto packet = std::move(builder).buildPacket(); EXPECT_CALL( - connCallback, onStopSending(streamId, ApplicationErrorCode::STOPPING)); + connCallback, + onStopSending(streamId, GenericApplicationErrorCode::UNKNOWN)); deliverData(packetToBuf(packet)); } @@ -1305,11 +1311,12 @@ TEST_F(QuicServerTransportTest, RecvStopSendingFrameAfterCloseStream) { std::move(header), 0 /* largestAcked */); - StopSendingFrame stopSendingFrame(streamId, ApplicationErrorCode::STOPPING); + StopSendingFrame stopSendingFrame( + streamId, GenericApplicationErrorCode::UNKNOWN); ASSERT_TRUE(builder.canBuildPacket()); writeFrame(std::move(stopSendingFrame), builder); auto packet = std::move(builder).buildPacket(); - server->resetStream(streamId, ApplicationErrorCode::STOPPING); + server->resetStream(streamId, GenericApplicationErrorCode::UNKNOWN); EXPECT_CALL(connCallback, onStopSending(_, _)).Times(0); deliverData(packetToBuf(packet)); } @@ -1382,7 +1389,8 @@ TEST_F(QuicServerTransportTest, RecvStopSendingFrameAfterHalfCloseRemote) { std::move(header), 0 /* largestAcked */); - StopSendingFrame stopSendingFrame(streamId, ApplicationErrorCode::STOPPING); + StopSendingFrame stopSendingFrame( + streamId, GenericApplicationErrorCode::UNKNOWN); ASSERT_TRUE(builder.canBuildPacket()); StreamFrameMetaData streamMeta; streamMeta.hasMoreFrames = true; @@ -1394,7 +1402,8 @@ TEST_F(QuicServerTransportTest, RecvStopSendingFrameAfterHalfCloseRemote) { writeFrame(std::move(stopSendingFrame), builder); auto packet = std::move(builder).buildPacket(); EXPECT_CALL( - connCallback, onStopSending(streamId, ApplicationErrorCode::STOPPING)); + connCallback, + onStopSending(streamId, GenericApplicationErrorCode::UNKNOWN)); deliverData(packetToBuf(packet)); } @@ -1410,13 +1419,15 @@ TEST_F(QuicServerTransportTest, RecvStopSendingBeforeStream) { std::move(header), 0 /* largestAcked */); - StopSendingFrame stopSendingFrame(streamId, ApplicationErrorCode::STOPPING); + StopSendingFrame stopSendingFrame( + streamId, GenericApplicationErrorCode::UNKNOWN); ASSERT_TRUE(builder.canBuildPacket()); writeFrame(std::move(stopSendingFrame), builder); auto packet = std::move(builder).buildPacket(); EXPECT_CALL(connCallback, onNewBidirectionalStream(streamId)); EXPECT_CALL( - connCallback, onStopSending(streamId, ApplicationErrorCode::STOPPING)); + connCallback, + onStopSending(streamId, GenericApplicationErrorCode::UNKNOWN)); deliverData(packetToBuf(packet)); } @@ -1460,13 +1471,16 @@ TEST_F(QuicServerTransportTest, RecvStopSendingFrameAfterReset) { std::move(header), 0 /* largestAcked */); - StopSendingFrame stopSendingFrame1(streamId1, ApplicationErrorCode::STOPPING); - StopSendingFrame stopSendingFrame2(streamId2, ApplicationErrorCode::STOPPING); + StopSendingFrame stopSendingFrame1( + streamId1, GenericApplicationErrorCode::UNKNOWN); + StopSendingFrame stopSendingFrame2( + streamId2, GenericApplicationErrorCode::UNKNOWN); ASSERT_TRUE(builder.canBuildPacket()); writeFrame(std::move(stopSendingFrame1), builder); writeFrame(std::move(stopSendingFrame2), builder); auto packet = std::move(builder).buildPacket(); - EXPECT_CALL(connCallback, onStopSending(_, ApplicationErrorCode::STOPPING)) + EXPECT_CALL( + connCallback, onStopSending(_, GenericApplicationErrorCode::UNKNOWN)) .WillOnce(Invoke([&](StreamId /*sid*/, ApplicationErrorCode /*e*/) { server->close(folly::none); })); @@ -1485,7 +1499,8 @@ TEST_F(QuicServerTransportTest, StopSendingLoss) { server->getConn().udpSendPacketLen, std::move(header), server->getConn().ackStates.appDataAckState.largestAckedByPeer); - StopSendingFrame stopSendingFrame(streamId, ApplicationErrorCode::STOPPING); + StopSendingFrame stopSendingFrame( + streamId, GenericApplicationErrorCode::UNKNOWN); ASSERT_TRUE(builder.canBuildPacket()); writeFrame(stopSendingFrame, builder); auto packet = std::move(builder).buildPacket(); @@ -1513,7 +1528,8 @@ TEST_F(QuicServerTransportTest, StopSendingLossAfterStreamClosed) { server->getConn().udpSendPacketLen, std::move(header), server->getConn().ackStates.appDataAckState.largestAckedByPeer); - StopSendingFrame stopSendingFrame(streamId, ApplicationErrorCode::STOPPING); + StopSendingFrame stopSendingFrame( + streamId, GenericApplicationErrorCode::UNKNOWN); ASSERT_TRUE(builder.canBuildPacket()); writeFrame(stopSendingFrame, builder); auto packet = std::move(builder).buildPacket(); @@ -1538,7 +1554,7 @@ TEST_F(QuicServerTransportTest, TestCloneStopSending) { server->getNonConstConn().outstandingPackets.clear(); server->getNonConstConn().lossState.lossTime.clear(); - server->stopSending(streamId, ApplicationErrorCode::STOPPING); + server->stopSending(streamId, GenericApplicationErrorCode::UNKNOWN); loopForWrites(); // Find the outstanding StopSending. auto packetItr = std::find_if( @@ -1589,7 +1605,7 @@ TEST_F(QuicServerTransportTest, TestCloneStopSending) { TEST_F(QuicServerTransportTest, TestAckStopSending) { auto streamId = server->createBidirectionalStream().value(); server->getNonConstConn().streamManager->getStream(streamId); - server->stopSending(streamId, ApplicationErrorCode::STOPPING); + server->stopSending(streamId, GenericApplicationErrorCode::UNKNOWN); loopForWrites(); auto match = [](OutstandingPacket& packet) { return std::find_if( @@ -1651,7 +1667,7 @@ TEST_F(QuicServerTransportTest, TestAckRstStream) { server->getNonConstConn(), server->getSocket(), *stream, - ApplicationErrorCode::STOPPING); + GenericApplicationErrorCode::UNKNOWN); IntervalSet acks = {{packetNum, packetNum}}; auto packet1 = createAckPacket( @@ -1704,12 +1720,13 @@ TEST_F(QuicServerTransportTest, ReceiveApplicationClose) { std::move(header), 0 /* largestAcked */); std::string errMsg = "Stand clear of the closing doors, please"; - ApplicationCloseFrame appClose(ApplicationErrorCode::STOPPING, errMsg); + ApplicationCloseFrame appClose(GenericApplicationErrorCode::UNKNOWN, errMsg); writeFrame(std::move(appClose), builder); auto packet = std::move(builder).buildPacket(); EXPECT_CALL( - connCallback, onConnectionError(IsError(ApplicationErrorCode::STOPPING))); + connCallback, + onConnectionError(IsAppError(GenericApplicationErrorCode::UNKNOWN))); deliverDataWithoutErrorCheck(packetToBuf(packet)); // Now the transport should be closed EXPECT_EQ( @@ -1717,7 +1734,7 @@ TEST_F(QuicServerTransportTest, ReceiveApplicationClose) { server->getConn().localConnectionError->first); EXPECT_EQ( server->getConn().peerConnectionError->first, - QuicErrorCode(ApplicationErrorCode::STOPPING)); + QuicErrorCode(GenericApplicationErrorCode::UNKNOWN)); auto closedMsg = folly::to("Server closed by peer reason=", errMsg); EXPECT_EQ(server->getConn().peerConnectionError->second, closedMsg); @@ -2926,7 +2943,8 @@ TEST_F(QuicUnencryptedServerTransportTest, TestCloseWhileAsyncPending) { recvClientFinished(); server->close(std::make_pair( - QuicErrorCode(ApplicationErrorCode::STOPPING), std::string("hello"))); + QuicErrorCode(GenericApplicationErrorCode::UNKNOWN), + std::string("hello"))); EXPECT_TRUE(server->isClosed()); testLooper.loop(); diff --git a/quic/server/test/TARGETS b/quic/server/test/TARGETS deleted file mode 100644 index bcd8c31c5..000000000 --- a/quic/server/test/TARGETS +++ /dev/null @@ -1,61 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") -load("@fbcode_macros//build_defs:cpp_unittest.bzl", "cpp_unittest") - -cpp_library( - name = "mocks", - headers = [ - "Mocks.h", - ], - deps = [ - "//quic/server:server", - ], -) - -cpp_unittest( - name = "QuicServerTest", - srcs = [ - "QuicServerTest.cpp", - "QuicSocketTest.cpp", - ], - deps = [ - ":mocks", - "//folly/futures:core", - "//folly/io:iobuf", - "//folly/io/async:async_base", - "//folly/io/async/test:mocks", - "//folly/portability:gmock", - "//folly/portability:gtest", - "//quic/api:transport", - "//quic/api/test:mocks", - "//quic/codec:header_codec", - "//quic/codec:types", - "//quic/common/test:test_utils", - "//quic/samples/echo:EchoHandler", - "//quic/server:server", - ], -) - -cpp_unittest( - name = "QuicServerTransportTest", - srcs = [ - "QuicServerTransportTest.cpp", - ], - deps = [ - ":mocks", - "//folly/io/async/test:mocks", - "//quic/api:transport", - "//quic/api/test:mocks", - "//quic/codec:pktbuilder", - "//quic/codec:types", - "//quic/common/test:test_utils", - "//quic/congestion_control:congestion_controller_factory", - "//quic/server:server", - "//quic/server/handshake:server_handshake", - "//quic/state:stream_functions", - ], - external_deps = [ - ("googletest", None, "gmock"), - ], -) diff --git a/quic/state/AckHandlers.cpp b/quic/state/AckHandlers.cpp index 6481279ef..63e48d96d 100644 --- a/quic/state/AckHandlers.cpp +++ b/quic/state/AckHandlers.cpp @@ -73,12 +73,6 @@ void processAckFrame( if (currentPacketNum > ackBlockIt->endPacket) { break; } - // If we are tracking RTO verification, and a packet sent before RTO is - // acked, then the RTO was spurious and we should stop tracking. - if (conn.lossState.largestSentBeforeRto && - currentPacketNum <= *conn.lossState.largestSentBeforeRto) { - conn.lossState.largestSentBeforeRto = folly::none; - } VLOG(10) << __func__ << " acked packetNum=" << currentPacketNum << " space=" << currentPacketNumberSpace << " handshake=" << (int)packetItEnd->isHandshake @@ -99,8 +93,8 @@ void processAckFrame( ackReceiveTime > packetItEnd->time ? ackReceiveTime : Clock::now(); auto rttSample = std::chrono::duration_cast( ackReceiveTimeOrNow - packetItEnd->time); - if (currentPacketNum == frame.largestAcked) { - updateRtt(conn, rttSample, frame.ackDelay, packetItEnd->pureAck); + if (currentPacketNum == frame.largestAcked && !packetItEnd->pureAck) { + updateRtt(conn, rttSample, frame.ackDelay); } QUIC_TRACE( packet_acked, @@ -149,6 +143,13 @@ void processAckFrame( auto lossEvent = handleAckForLoss(conn, lossVisitor, ack, pnSpace); if (conn.congestionController && (ack.largestAckedPacket.hasValue() || lossEvent)) { + if (lossEvent) { + DCHECK(lossEvent->largestLostSentTime && lossEvent->smallestLostSentTime); + lossEvent->persistentCongestion = isPersistentCongestion( + conn, + *lossEvent->smallestLostSentTime, + *lossEvent->largestLostSentTime); + } conn.congestionController->onPacketAckOrLoss( std::move(ack), std::move(lossEvent)); } diff --git a/quic/state/QuicStateFunctions.cpp b/quic/state/QuicStateFunctions.cpp index ecc2072c4..16b4c916d 100644 --- a/quic/state/QuicStateFunctions.cpp +++ b/quic/state/QuicStateFunctions.cpp @@ -16,16 +16,11 @@ namespace quic { void updateRtt( QuicConnectionStateBase& conn, std::chrono::microseconds rttSample, - std::chrono::microseconds ackDelay, - bool pureAck) { + std::chrono::microseconds ackDelay) { conn.lossState.mrtt = timeMin(conn.lossState.mrtt, rttSample); - // This interesting condition is from the specs. + conn.lossState.maxAckDelay = timeMax(conn.lossState.maxAckDelay, ackDelay); if (rttSample > conn.lossState.mrtt + ackDelay) { rttSample -= ackDelay; - if (!pureAck) { - conn.lossState.maxAckDelay = - timeMax(conn.lossState.maxAckDelay, ackDelay); - } } conn.lossState.lrtt = rttSample; if (conn.lossState.srtt == std::chrono::microseconds::zero()) { @@ -251,5 +246,4 @@ bool hasReceivedPackets(const QuicConnectionStateBase& conn) noexcept { conn.ackStates.handshakeAckState.largestReceivedPacketNum || conn.ackStates.appDataAckState.largestReceivedPacketNum; } - } // namespace quic diff --git a/quic/state/QuicStateFunctions.h b/quic/state/QuicStateFunctions.h index 580d902a6..9672d7a29 100644 --- a/quic/state/QuicStateFunctions.h +++ b/quic/state/QuicStateFunctions.h @@ -33,8 +33,7 @@ void updateAckSendStateOnSentPacketWithAcks( void updateRtt( QuicConnectionStateBase& conn, std::chrono::microseconds rttSample, - std::chrono::microseconds ackDelay, - bool pureAck); + std::chrono::microseconds ackDelay); template void invokeStreamStateMachine( diff --git a/quic/state/StateData.h b/quic/state/StateData.h index 97114948f..8fd1929d6 100644 --- a/quic/state/StateData.h +++ b/quic/state/StateData.h @@ -131,9 +131,20 @@ struct CongestionController { uint64_t lostBytes; uint32_t lostPackets; const TimePoint lossTime; + // The packet sent time of the lost packet with largest packet sent time in + // this LossEvent + folly::Optional largestLostSentTime; + // The packet sent time of the lost packet with smallest packet sent time in + // the LossEvent + folly::Optional smallestLostSentTime; + // Whether this LossEvent also indicates persistent congestion + bool persistentCongestion; explicit LossEvent(TimePoint time = Clock::now()) - : lostBytes(0), lostPackets(0), lossTime(time) {} + : lostBytes(0), + lostPackets(0), + lossTime(time), + persistentCongestion(false) {} void addLostPacket(const OutstandingPacket& packet) { if (UNLIKELY( @@ -150,6 +161,10 @@ struct CongestionController { std::max(packetNum, largestLostPacketNum.value_or(packetNum)); lostBytes += packet.encodedSize; lostPackets++; + largestLostSentTime = + std::max(packet.time, largestLostSentTime.value_or(packet.time)); + smallestLostSentTime = + std::min(packet.time, smallestLostSentTime.value_or(packet.time)); } }; @@ -185,7 +200,6 @@ struct CongestionController { virtual void onPacketAckOrLoss( folly::Optional, folly::Optional) = 0; - virtual void onRTOVerified() = 0; /** * Return the number of bytes that the congestion controller @@ -273,13 +287,7 @@ using Resets = std::unordered_map; using FrameList = std::vector; struct LossState { - enum LossMode { - ReorderingThreshold, - TimeLossDetection, - }; enum class AlarmMethod { EarlyRetransmitOrReordering, Handshake, RTO }; - // Current loss mode we are running on - LossMode lossMode{LossState::ReorderingThreshold}; // Smooth rtt std::chrono::microseconds srtt{std::chrono::microseconds::zero()}; // Latest rtt @@ -293,16 +301,12 @@ struct LossState { uint16_t handshakeAlarmCount{0}; // The time when last handshake packet was sent TimePoint lastHandshakePacketSentTime; - // Latest packet number sent before latest RTO - folly::Optional largestSentBeforeRto; // Latest packet number sent // TODO: 0 is a legit PacketNum now, we need to make this optional: // TODO: this also needs to be 3 numbers now... PacketNum largestSent{0}; // Reordering threshold used uint32_t reorderingThreshold{kReorderingThreshold}; - // Time reordering fraction used - double timeReorderingFraction{kTimeReorderingFraction}; // Timer for time reordering detection or early retransmit alarm. folly::Optional lossTime; // Current method by which the loss detection alarm is set. @@ -341,6 +345,8 @@ struct LossState { folly::Optional lastAckedPacketSentTime; // The latest time a packet is acked folly::Optional lastAckedTime; + // The time when last retranmittable packet is sent + TimePoint lastRetransmittablePacketSentTime; }; class Logger; diff --git a/quic/state/TARGETS b/quic/state/TARGETS deleted file mode 100644 index d739f5ae0..000000000 --- a/quic/state/TARGETS +++ /dev/null @@ -1,173 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") - -cpp_library( - name = "transport_settings", - headers = [ - "TransportSettings.h", - ], - deps = [ - "//quic:constants", - ], -) - -cpp_library( - name = "ack_states", - headers = [ - "AckStates.h", - ], - deps = [ - "//quic/codec:types", - "//quic/common:interval_set", - ], -) - -cpp_library( - name = "state_machine", - srcs = [ - "QuicStreamManager.cpp", - "QuicStreamUtilities.cpp", - "StateData.cpp", - ], - headers = [ - "QuicStreamManager.h", - "QuicStreamUtilities.h", - "StateData.h", - "StateMachine.h", - "StreamData.h", - ], - deps = [ - ":stats_callback", - ":transport_settings", - "//folly:optional", - "//folly:overload", - "//folly/io:iobuf", - "//folly/io/async:async_base", - "//folly/io/async:async_udp_socket", - "//quic:constants", - "//quic/codec:codec", - "//quic/codec:types", - "//quic/handshake:handshake", - "//quic/state:ack_states", - ], - external_deps = [ - "boost", - ], -) - -cpp_library( - name = "ack_handler", - srcs = [ - "AckHandlers.cpp", - ], - headers = [ - "AckHandlers.h", - ], - deps = [ - ":state_functions", - ":state_machine", - "//folly:overload", - "//quic:constants", - "//quic/codec:types", - "//quic/logging:logging", - "//quic/loss:loss", - ], -) - -cpp_library( - name = "stream_functions", - srcs = [ - "QuicStreamFunctions.cpp", - ], - headers = [ - "QuicStreamFunctions.h", - ], - deps = [ - ":state_machine", - "//folly/io:iobuf", - "//quic:exception", - "//quic/flowcontrol:flow_control", - "//quic/logging:logging", - ], -) - -cpp_library( - name = "state_functions", - srcs = [ - "QuicStateFunctions.cpp", - ], - headers = [ - "QuicStateFunctions.h", - ], - deps = [ - ":state_machine", - "//folly:overload", - "//quic/codec:pktbuilder", - "//quic/codec:types", - "//quic/common:time_util", - "//quic/logging:logging", - "//quic/state/stream:stream", - ], -) - -cpp_library( - name = "stats_callback", - headers = [ - "QuicTransportStatsCallback.h", - ], - deps = [ - "//folly:optional", - "//folly/functional:invoke", - "//folly/io/async:async_base", - ], -) - -cpp_library( - name = "pacing_functions", - srcs = [ - "QuicPacingFunctions.cpp", - ], - headers = [ - "QuicPacingFunctions.h", - ], - deps = [ - ":state_machine", - ], -) - -cpp_library( - name = "simple_frame_functions", - srcs = [ - "SimpleFrameFunctions.cpp", - ], - headers = [ - "SimpleFrameFunctions.h", - ], - deps = [ - ":qpr_functions", - ":state_functions", - ":state_machine", - ":stream_functions", - "//quic/codec:types", - ], - external_deps = [ - "boost", - ], -) - -cpp_library( - name = "qpr_functions", - srcs = [ - "QPRFunctions.cpp", - ], - headers = [ - "QPRFunctions.h", - ], - deps = [ - ":state_machine", - ":stream_functions", - "//quic/codec:types", - "//quic/flowcontrol:flow_control", - ], -) diff --git a/quic/state/stream/StreamHalfClosedRemoteHandlers.h b/quic/state/stream/StreamHalfClosedRemoteHandlers.h index 4e1f89cbe..8590e2373 100644 --- a/quic/state/stream/StreamHalfClosedRemoteHandlers.h +++ b/quic/state/stream/StreamHalfClosedRemoteHandlers.h @@ -74,7 +74,8 @@ Handler:: transit(stream); onResetQuicStream(stream, std::move(rst)); // TODO: remove. - appendPendingStreamReset(stream.conn, stream, ApplicationErrorCode::STOPPING); + appendPendingStreamReset( + stream.conn, stream, GenericApplicationErrorCode::NO_ERROR); } inline void Handler< diff --git a/quic/state/stream/StreamOpenHandlers.h b/quic/state/stream/StreamOpenHandlers.h index ee238abc4..23daa5037 100644 --- a/quic/state/stream/StreamOpenHandlers.h +++ b/quic/state/stream/StreamOpenHandlers.h @@ -70,7 +70,7 @@ Handler::handle( if (isBidirectionalStream(stream.id)) { // TODO: remove. appendPendingStreamReset( - stream.conn, stream, ApplicationErrorCode::STOPPING); + stream.conn, stream, GenericApplicationErrorCode::NO_ERROR); } else { transit(stream); } diff --git a/quic/state/stream/TARGETS b/quic/state/stream/TARGETS deleted file mode 100644 index 18b50cadb..000000000 --- a/quic/state/stream/TARGETS +++ /dev/null @@ -1,26 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") - -cpp_library( - name = "stream", - srcs = [ - "StreamStateFunctions.cpp", - ], - headers = [ - "StreamClosedHandlers.h", - "StreamHalfClosedLocalHandlers.h", - "StreamHalfClosedRemoteHandlers.h", - "StreamOpenHandlers.h", - "StreamStateFunctions.h", - "StreamStateMachine.h", - "StreamWaitingForRstAckHandlers.h", - ], - deps = [ - "//quic:exception", - "//quic/codec:types", - "//quic/flowcontrol:flow_control", - "//quic/state:state_machine", - "//quic/state:stream_functions", - ], -) diff --git a/quic/state/stream/test/StreamStateFunctionsTest.cpp b/quic/state/stream/test/StreamStateFunctionsTest.cpp index 8b0253fa9..2058289f3 100644 --- a/quic/state/stream/test/StreamStateFunctionsTest.cpp +++ b/quic/state/stream/test/StreamStateFunctionsTest.cpp @@ -39,7 +39,7 @@ TEST_F(StreamStateFunctionsTests, SanityTest) { EXPECT_TRUE(stream.writable()); invokeHandler( - stream, StreamEvents::SendReset(ApplicationErrorCode::STOPPING)); + stream, StreamEvents::SendReset(GenericApplicationErrorCode::UNKNOWN)); // Something are cleared: EXPECT_TRUE(stream.writeBuffer.empty()); EXPECT_TRUE(stream.retransmissionBuffer.empty()); @@ -156,7 +156,7 @@ TEST_F(StreamStateFunctionsTests, SendReset) { stream, StreamBuffer(folly::IOBuf::copyBuffer("hi"), 0)); EXPECT_FALSE(stream.writeBuffer.empty()); EXPECT_FALSE(stream.readBuffer.empty()); - resetQuicStream(stream, ApplicationErrorCode::STOPPING); + resetQuicStream(stream, GenericApplicationErrorCode::UNKNOWN); EXPECT_TRUE(stream.writeBuffer.empty()); EXPECT_TRUE(stream.readBuffer.empty()); @@ -169,7 +169,7 @@ TEST_F(StreamStateFunctionsTests, ResetNoFlowControlGenerated) { QuicStreamState stream(id, conn); writeDataToQuicStream(stream, folly::IOBuf::copyBuffer("hello"), true); EXPECT_GT(conn.flowControlState.sumCurStreamBufferLen, 0); - RstStreamFrame rst(id, ApplicationErrorCode::STOPPING, 90); + RstStreamFrame rst(id, GenericApplicationErrorCode::UNKNOWN, 90); stream.currentReadOffset = 80; stream.maxOffsetObserved = 90; @@ -193,7 +193,7 @@ TEST_F(StreamStateFunctionsTests, ResetFlowControlGenerated) { QuicStreamState stream(id, conn); writeDataToQuicStream(stream, folly::IOBuf::copyBuffer("hello"), true); EXPECT_GT(conn.flowControlState.sumCurStreamBufferLen, 0); - RstStreamFrame rst(id, ApplicationErrorCode::STOPPING, 100); + RstStreamFrame rst(id, GenericApplicationErrorCode::UNKNOWN, 100); stream.currentReadOffset = 80; stream.maxOffsetObserved = 90; stream.flowControlState.advertisedMaxOffset = 100; @@ -214,7 +214,7 @@ TEST_F(StreamStateFunctionsTests, ResetOffsetNotMatch) { QuicServerConnectionState conn; StreamId id = 1; QuicStreamState stream(id, conn); - RstStreamFrame rst(id, ApplicationErrorCode::STOPPING, 10); + RstStreamFrame rst(id, GenericApplicationErrorCode::UNKNOWN, 10); stream.currentReadOffset = 20; stream.maxOffsetObserved = 100; stream.finalReadOffset = 100; @@ -227,7 +227,7 @@ TEST_F(StreamStateFunctionsTests, ResetOffsetLessThanMaxObserved) { QuicServerConnectionState conn; StreamId id = 1; QuicStreamState stream(id, conn); - RstStreamFrame rst(id, ApplicationErrorCode::STOPPING, 30); + RstStreamFrame rst(id, GenericApplicationErrorCode::UNKNOWN, 30); stream.currentReadOffset = 20; stream.maxOffsetObserved = 100; stream.flowControlState.advertisedMaxOffset = 300; @@ -239,7 +239,7 @@ TEST_F(StreamStateFunctionsTests, ResetOffsetGreaterThanStreamFlowControl) { QuicServerConnectionState conn; StreamId id = 1; QuicStreamState stream(id, conn); - RstStreamFrame rst(id, ApplicationErrorCode::STOPPING, 200); + RstStreamFrame rst(id, GenericApplicationErrorCode::UNKNOWN, 200); stream.currentReadOffset = 20; stream.maxOffsetObserved = 30; stream.flowControlState.advertisedMaxOffset = 100; @@ -251,7 +251,7 @@ TEST_F(StreamStateFunctionsTests, ResetOffsetGreaterThanConnFlowControl) { QuicServerConnectionState conn; StreamId id = 1; QuicStreamState stream(id, conn); - RstStreamFrame rst(id, ApplicationErrorCode::STOPPING, 200); + RstStreamFrame rst(id, GenericApplicationErrorCode::UNKNOWN, 200); stream.currentReadOffset = 20; stream.maxOffsetObserved = 30; @@ -270,7 +270,7 @@ TEST_F(StreamStateFunctionsTests, ResetAfterReadingAllBytesTillFin) { QuicServerConnectionState conn; StreamId id = 1; QuicStreamState stream(id, conn); - RstStreamFrame rst(id, ApplicationErrorCode::STOPPING, 100); + RstStreamFrame rst(id, GenericApplicationErrorCode::UNKNOWN, 100); stream.currentReadOffset = 101; stream.finalReadOffset = 100; stream.maxOffsetObserved = 100; diff --git a/quic/state/stream/test/StreamStateMachineTest.cpp b/quic/state/stream/test/StreamStateMachineTest.cpp index ff35682a2..1a491843d 100644 --- a/quic/state/stream/test/StreamStateMachineTest.cpp +++ b/quic/state/stream/test/StreamStateMachineTest.cpp @@ -97,7 +97,7 @@ TEST_F(QuicOpenStateTest, InvalidEvent) { auto conn = createConn(); StreamId id = 5; QuicStreamState stream(id, *conn); - RstStreamFrame frame(1, ApplicationErrorCode::STOPPING, 0); + RstStreamFrame frame(1, GenericApplicationErrorCode::UNKNOWN, 0); EXPECT_THROW( invokeHandler(stream, StreamEvents::RstAck(frame)), QuicTransportException); @@ -157,7 +157,7 @@ TEST_F(QuicWaitingForRstAckStateTest, RstAck) { stream.finalWriteOffset = 0xACDC; stream.readBuffer.emplace_back( folly::IOBuf::copyBuffer("One more thing"), 0xABCD, false); - RstStreamFrame frame(id, ApplicationErrorCode::STOPPING, 0); + RstStreamFrame frame(id, GenericApplicationErrorCode::UNKNOWN, 0); invokeHandler(stream, StreamEvents::RstAck(frame)); EXPECT_TRUE(isState(stream)); @@ -172,7 +172,7 @@ TEST_F(QuicClosedStateTest, RstAck) { StreamId id = 5; QuicStreamState stream(id, *conn); stream.state = StreamStates::Closed(); - RstStreamFrame frame(id, ApplicationErrorCode::STOPPING, 0); + RstStreamFrame frame(id, GenericApplicationErrorCode::UNKNOWN, 0); invokeHandler(stream, StreamEvents::RstAck(frame)); EXPECT_TRUE(isState(stream)); } @@ -306,7 +306,7 @@ TEST_F(QuicSendResetTest, FromOpen) { StreamId id = 5; QuicStreamState stream(id, *conn); invokeHandler( - stream, StreamEvents::SendReset(ApplicationErrorCode::STOPPING)); + stream, StreamEvents::SendReset(GenericApplicationErrorCode::UNKNOWN)); EXPECT_TRUE(isState(stream)); } @@ -316,7 +316,7 @@ TEST_F(QuicSendResetTest, FromHalfCloseRemote) { QuicStreamState stream(id, *conn); stream.state = StreamStates::HalfClosedRemote(); invokeHandler( - stream, StreamEvents::SendReset(ApplicationErrorCode::STOPPING)); + stream, StreamEvents::SendReset(GenericApplicationErrorCode::UNKNOWN)); EXPECT_TRUE(isState(stream)); } @@ -326,7 +326,7 @@ TEST_F(QuicSendResetTest, FromHalfCloseLocal) { QuicStreamState stream(id, *conn); stream.state = StreamStates::HalfClosedLocal(); invokeHandler( - stream, StreamEvents::SendReset(ApplicationErrorCode::STOPPING)); + stream, StreamEvents::SendReset(GenericApplicationErrorCode::UNKNOWN)); EXPECT_TRUE(isState(stream)); } @@ -336,7 +336,7 @@ TEST_F(QuicSendResetTest, FromClosed) { QuicStreamState stream(id, *conn); stream.state = StreamStates::Closed(); invokeHandler( - stream, StreamEvents::SendReset(ApplicationErrorCode::STOPPING)); + stream, StreamEvents::SendReset(GenericApplicationErrorCode::UNKNOWN)); } TEST_F(QuicSendResetTest, FromWaitingForRstAck) { @@ -345,7 +345,7 @@ TEST_F(QuicSendResetTest, FromWaitingForRstAck) { QuicStreamState stream(id, *conn); stream.state = StreamStates::WaitingForRstAck(); invokeHandler( - stream, StreamEvents::SendReset(ApplicationErrorCode::STOPPING)); + stream, StreamEvents::SendReset(GenericApplicationErrorCode::UNKNOWN)); } class QuicRecvResetTest : public Test {}; @@ -355,7 +355,7 @@ TEST_F(QuicRecvResetTest, FromOpen) { StreamId id = 5; StreamId rstStream = 1; QuicStreamState stream(id, *conn); - RstStreamFrame rst(rstStream, ApplicationErrorCode::STOPPING, 100); + RstStreamFrame rst(rstStream, GenericApplicationErrorCode::UNKNOWN, 100); invokeHandler(stream, std::move(rst)); EXPECT_TRUE(isState(stream)); verifyStreamReset(stream, 100); @@ -366,7 +366,7 @@ TEST_F(QuicRecvResetTest, FromOpenReadEOFMismatch) { StreamId id = 5; QuicStreamState stream(id, *conn); - RstStreamFrame rst(1, ApplicationErrorCode::STOPPING, 100); + RstStreamFrame rst(1, GenericApplicationErrorCode::UNKNOWN, 100); stream.finalReadOffset = 1024; EXPECT_THROW( invokeHandler(stream, std::move(rst)), @@ -379,7 +379,7 @@ TEST_F(QuicRecvResetTest, FromHalfClosedRemoteNoReadOffsetYet) { QuicStreamState stream(id, *conn); stream.state = StreamStates::HalfClosedRemote(); invokeHandler( - stream, RstStreamFrame(1, ApplicationErrorCode::STOPPING, 100)); + stream, RstStreamFrame(1, GenericApplicationErrorCode::UNKNOWN, 100)); EXPECT_TRUE(isState(stream)); verifyStreamReset(stream, 100); } @@ -391,7 +391,7 @@ TEST_F(QuicRecvResetTest, FromHalfClosedRemoteReadOffsetMatch) { stream.state = StreamStates::HalfClosedRemote(); stream.finalReadOffset = 1024; invokeHandler( - stream, RstStreamFrame(1, ApplicationErrorCode::STOPPING, 1024)); + stream, RstStreamFrame(1, GenericApplicationErrorCode::UNKNOWN, 1024)); EXPECT_TRUE(isState(stream)); verifyStreamReset(stream, 1024); } @@ -404,7 +404,7 @@ TEST_F(QuicRecvResetTest, FromHalfClosedRemoteReadOffsetMismatch) { stream.finalReadOffset = 1024; EXPECT_THROW( invokeHandler( - stream, RstStreamFrame(1, ApplicationErrorCode::STOPPING, 100)), + stream, RstStreamFrame(1, GenericApplicationErrorCode::UNKNOWN, 100)), QuicTransportException); } @@ -414,7 +414,7 @@ TEST_F(QuicRecvResetTest, FromHalfClosedLocal) { QuicStreamState stream(id, *conn); stream.state = StreamStates::HalfClosedLocal(); invokeHandler( - stream, RstStreamFrame(1, ApplicationErrorCode::STOPPING, 200)); + stream, RstStreamFrame(1, GenericApplicationErrorCode::UNKNOWN, 200)); EXPECT_TRUE(isState(stream)); verifyStreamReset(stream, 200); } @@ -427,7 +427,7 @@ TEST_F(QuicRecvResetTest, FromHalfClosedLocalReadEOFMismatch) { stream.finalReadOffset = 2014; EXPECT_THROW( invokeHandler( - stream, RstStreamFrame(1, ApplicationErrorCode::STOPPING, 200)), + stream, RstStreamFrame(1, GenericApplicationErrorCode::UNKNOWN, 200)), QuicTransportException); } @@ -437,7 +437,7 @@ TEST_F(QuicRecvResetTest, FromWaitingForRstAckNoReadOffsetYet) { QuicStreamState stream(id, *conn); stream.state = StreamStates::WaitingForRstAck(); invokeHandler( - stream, RstStreamFrame(1, ApplicationErrorCode::STOPPING, 200)); + stream, RstStreamFrame(1, GenericApplicationErrorCode::UNKNOWN, 200)); EXPECT_TRUE(isState(stream)); verifyStreamReset(stream, 200); } @@ -449,7 +449,7 @@ TEST_F(QuicRecvResetTest, FromWaitingForRstAckOffsetMatch) { stream.state = StreamStates::WaitingForRstAck(); stream.finalReadOffset = 200; invokeHandler( - stream, RstStreamFrame(1, ApplicationErrorCode::STOPPING, 200)); + stream, RstStreamFrame(1, GenericApplicationErrorCode::UNKNOWN, 200)); EXPECT_TRUE(isState(stream)); verifyStreamReset(stream, 200); } @@ -462,7 +462,7 @@ TEST_F(QuicRecvResetTest, FromWaitingForRstAckOffsetMismatch) { stream.finalReadOffset = 300; EXPECT_THROW( invokeHandler( - stream, RstStreamFrame(1, ApplicationErrorCode::STOPPING, 200)), + stream, RstStreamFrame(1, GenericApplicationErrorCode::UNKNOWN, 200)), QuicTransportException); } @@ -472,7 +472,7 @@ TEST_F(QuicRecvResetTest, FromClosedNoReadOffsetYet) { QuicStreamState stream(id, *conn); stream.state = StreamStates::Closed(); invokeHandler( - stream, RstStreamFrame(1, ApplicationErrorCode::STOPPING, 200)); + stream, RstStreamFrame(1, GenericApplicationErrorCode::UNKNOWN, 200)); EXPECT_TRUE(isState(stream)); verifyStreamReset(stream, 200); } @@ -484,7 +484,7 @@ TEST_F(QuicRecvResetTest, FromClosedOffsetMatch) { stream.state = StreamStates::Closed(); stream.finalReadOffset = 1234; invokeHandler( - stream, RstStreamFrame(1, ApplicationErrorCode::STOPPING, 1234)); + stream, RstStreamFrame(1, GenericApplicationErrorCode::UNKNOWN, 1234)); EXPECT_TRUE(isState(stream)); verifyStreamReset(stream, 1234); } @@ -497,7 +497,8 @@ TEST_F(QuicRecvResetTest, FromClosedOffsetMismatch) { stream.finalReadOffset = 123; EXPECT_THROW( invokeHandler( - stream, RstStreamFrame(1, ApplicationErrorCode::STOPPING, 1234)), + stream, + RstStreamFrame(1, GenericApplicationErrorCode::UNKNOWN, 1234)), QuicTransportException); } @@ -520,7 +521,8 @@ TEST_F(QuicUnidirectionalStreamTest, OpenInvalidRstStream) { stream.state = StreamStates::Open(); EXPECT_THROW( invokeHandler( - stream, RstStreamFrame(1, ApplicationErrorCode::STOPPING, 1234)), + stream, + RstStreamFrame(1, GenericApplicationErrorCode::UNKNOWN, 1234)), QuicTransportException); } @@ -531,7 +533,8 @@ TEST_F(QuicUnidirectionalStreamTest, OpenInvalidSendReset) { stream.state = StreamStates::Open(); EXPECT_THROW( invokeHandler( - stream, StreamEvents::SendReset(ApplicationErrorCode::STOPPING)), + stream, + StreamEvents::SendReset(GenericApplicationErrorCode::UNKNOWN)), QuicTransportException); } @@ -552,7 +555,7 @@ TEST_F(QuicUnidirectionalStreamTest, OpenInvalidStopSending) { stream.state = StreamStates::Open(); EXPECT_THROW( invokeHandler( - stream, StopSendingFrame(id, ApplicationErrorCode::STOPPING)), + stream, StopSendingFrame(id, GenericApplicationErrorCode::UNKNOWN)), QuicTransportException); } @@ -573,7 +576,8 @@ TEST_F(QuicUnidirectionalStreamTest, ClosedInvalidRstStream) { stream.state = StreamStates::Closed(); EXPECT_THROW( invokeHandler( - stream, RstStreamFrame(1, ApplicationErrorCode::STOPPING, 1234)), + stream, + RstStreamFrame(1, GenericApplicationErrorCode::UNKNOWN, 1234)), QuicTransportException); } @@ -584,7 +588,8 @@ TEST_F(QuicUnidirectionalStreamTest, ClosedInvalidSendReset) { stream.state = StreamStates::Closed(); EXPECT_THROW( invokeHandler( - stream, StreamEvents::SendReset(ApplicationErrorCode::STOPPING)), + stream, + StreamEvents::SendReset(GenericApplicationErrorCode::UNKNOWN)), QuicTransportException); } @@ -605,7 +610,7 @@ TEST_F(QuicUnidirectionalStreamTest, ClosedInvalidStopSending) { stream.state = StreamStates::Closed(); EXPECT_THROW( invokeHandler( - stream, StopSendingFrame(id, ApplicationErrorCode::STOPPING)), + stream, StopSendingFrame(id, GenericApplicationErrorCode::UNKNOWN)), QuicTransportException); } @@ -628,7 +633,7 @@ TEST_F(QuicUnidirectionalStreamTest, OpenRstStream) { QuicStreamState stream(id, *conn); stream.state = StreamStates::Open(); invokeHandler( - stream, RstStreamFrame(1, ApplicationErrorCode::STOPPING, 1234)); + stream, RstStreamFrame(1, GenericApplicationErrorCode::UNKNOWN, 1234)); EXPECT_TRUE(isState(stream)); } @@ -665,7 +670,8 @@ TEST_F(QuicUnidirectionalStreamTest, WaitingForRstAckInvalidRstStream) { stream.state = StreamStates::WaitingForRstAck(); EXPECT_THROW( invokeHandler( - stream, RstStreamFrame(1, ApplicationErrorCode::STOPPING, 1234)), + stream, + RstStreamFrame(1, GenericApplicationErrorCode::UNKNOWN, 1234)), QuicTransportException); } @@ -676,7 +682,7 @@ TEST_F(QuicUnidirectionalStreamTest, WaitingForRstAckInvalidStopSending) { stream.state = StreamStates::WaitingForRstAck(); EXPECT_THROW( invokeHandler( - stream, StopSendingFrame(id, ApplicationErrorCode::STOPPING)), + stream, StopSendingFrame(id, GenericApplicationErrorCode::UNKNOWN)), QuicTransportException); } } // namespace test diff --git a/quic/state/stream/test/TARGETS b/quic/state/stream/test/TARGETS deleted file mode 100644 index 0d661d245..000000000 --- a/quic/state/stream/test/TARGETS +++ /dev/null @@ -1,33 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_unittest.bzl", "cpp_unittest") - -cpp_unittest( - name = "StreamStateMachineTest", - srcs = [ - "StreamStateMachineTest.cpp", - ], - deps = [ - "//folly:overload", - "//quic/api:transport", - "//quic/codec:types", - "//quic/common/test:test_utils", - "//quic/state:stream_functions", - "//quic/state/stream:stream", - ], - external_deps = [ - ("googletest", None, "gmock"), - ], -) - -cpp_unittest( - name = "StreamStateFunctionsTest", - srcs = [ - "StreamStateFunctionsTest.cpp", - ], - deps = [ - "//quic/server/state:server", - "//quic/state:stream_functions", - "//quic/state/stream:stream", - ], -) diff --git a/quic/state/test/AckHandlersTest.cpp b/quic/state/test/AckHandlersTest.cpp index 2c93fa3f0..14b3b5e3b 100644 --- a/quic/state/test/AckHandlersTest.cpp +++ b/quic/state/test/AckHandlersTest.cpp @@ -64,6 +64,8 @@ TEST_P(AckHandlersTest, TestAckMultipleSequentialBlocks) { auto mockCongestionController = std::make_unique(); auto rawCongestionController = mockCongestionController.get(); conn.congestionController = std::move(mockCongestionController); + // Get the time based loss detection out of the way + conn.lossState.srtt = std::chrono::seconds(10); StreamId current = 10; auto sentTime = Clock::now(); @@ -136,6 +138,8 @@ TEST_P(AckHandlersTest, TestAckBlocksWithGaps) { auto mockCongestionController = std::make_unique(); auto rawCongestionController = mockCongestionController.get(); conn.congestionController = std::move(mockCongestionController); + // Get the time based loss detection out of the way + conn.lossState.srtt = std::chrono::seconds(10); StreamId current = 10; for (PacketNum packetNum = 10; packetNum < 51; packetNum++) { @@ -239,6 +243,8 @@ TEST_P(AckHandlersTest, TestNonSequentialPacketNumbers) { auto mockCongestionController = std::make_unique(); auto rawCongestionController = mockCongestionController.get(); conn.congestionController = std::move(mockCongestionController); + // Get the time based loss detection out of the way + conn.lossState.srtt = std::chrono::seconds(10); StreamId current = 10; for (PacketNum packetNum = 10; packetNum < 20; packetNum++) { @@ -413,139 +419,13 @@ TEST_P(AckHandlersTest, AckVisitorForAckTest) { EXPECT_TRUE(conn.ackStates.appDataAckState.acks.empty()); } -TEST_P(AckHandlersTest, NotifyVerifiedRTO) { - QuicServerConnectionState conn; - auto mockController = std::make_unique(); - auto rawController = mockController.get(); - conn.congestionController = std::move(mockController); - - conn.lossState.largestSentBeforeRto = 8; - conn.lossState.rtoCount = 1; - - PacketNum packetAfterRtoNum = 10; - auto packetAfterRto = createNewPacket(packetAfterRtoNum, GetParam()); - conn.outstandingPackets.emplace_back(OutstandingPacket( - std::move(packetAfterRto), Clock::now(), 0, false, false, 0)); - - ReadAckFrame ackFrame; - ackFrame.largestAcked = 10; - ackFrame.ackBlocks.emplace_back(5, 7); - ackFrame.ackBlocks.emplace_back(9, 10); - EXPECT_CALL(*rawController, onPacketAckOrLoss(_, _)) - .Times(1) - .WillOnce(Invoke([&](auto ackEvent, auto) { - EXPECT_EQ(1, ackEvent->ackedPackets.size()); - EXPECT_EQ(0, ackEvent->ackedPackets.front().encodedSize); - EXPECT_EQ(0, ackEvent->ackedPackets.front().totalBytesSent); - EXPECT_FALSE(ackEvent->ackedPackets.front().isHandshake); - EXPECT_FALSE(ackEvent->ackedPackets.front().pureAck); - })); - EXPECT_CALL(*rawController, onRTOVerified()).Times(1); - processAckFrame( - conn, - GetParam(), - ackFrame, - [](const auto&, const auto&, const auto&) {}, - [](auto&, auto&, bool, PacketNum) {}, - Clock::now()); - EXPECT_FALSE(conn.lossState.largestSentBeforeRto.hasValue()); - // Nothing is outstanding any more - EXPECT_TRUE(conn.outstandingPackets.empty()); - EXPECT_EQ(0, conn.outstandingPureAckPacketsCount); - EXPECT_FALSE(conn.pendingEvents.setLossDetectionAlarm); -} - -TEST_P(AckHandlersTest, RTOTrackerIsAcked) { - QuicServerConnectionState conn; - auto mockController = std::make_unique(); - auto rawController = mockController.get(); - conn.congestionController = std::move(mockController); - - conn.lossState.largestSentBeforeRto = 8; - conn.lossState.rtoCount = 1; - - PacketNum rtoPacket = 8; - auto packetAfterRto = createNewPacket(rtoPacket, GetParam()); - conn.outstandingPackets.emplace_back(OutstandingPacket( - std::move(packetAfterRto), Clock::now(), 0, false, false, 0)); - - ReadAckFrame ackFrame; - ackFrame.largestAcked = 10; - ackFrame.ackBlocks.emplace_back(5, 10); - EXPECT_CALL(*rawController, onPacketAckOrLoss(_, _)) - .Times(1) - .WillOnce(Invoke([&](auto ackEvent, auto) { - EXPECT_EQ(1, ackEvent->ackedPackets.size()); - EXPECT_EQ(0, ackEvent->ackedPackets.front().encodedSize); - EXPECT_EQ(0, ackEvent->ackedPackets.front().totalBytesSent); - EXPECT_FALSE(ackEvent->ackedPackets.front().isHandshake); - EXPECT_FALSE(ackEvent->ackedPackets.front().pureAck); - })); - EXPECT_CALL(*rawController, onRTOVerified()).Times(0); - processAckFrame( - conn, - GetParam(), - ackFrame, - [](const auto&, const auto&, const auto&) {}, - [](auto&, auto&, bool, PacketNum) {}, - Clock::now()); - EXPECT_FALSE(conn.lossState.largestSentBeforeRto.hasValue()); - // Nothing is outstanding any more - EXPECT_TRUE(conn.outstandingPackets.empty()); - EXPECT_EQ(0, conn.outstandingPureAckPacketsCount); - EXPECT_FALSE(conn.pendingEvents.setLossDetectionAlarm); -} - -TEST_P(AckHandlersTest, PacketBeforeRTOTrackerIsAcked) { - QuicServerConnectionState conn; - auto mockController = std::make_unique(); - auto rawController = mockController.get(); - conn.congestionController = std::move(mockController); - - conn.lossState.largestSentBeforeRto = 8; - conn.lossState.rtoCount = 1; - - PacketNum packetBeforeRtoTracker = 7; - auto packetAfterRto = createNewPacket(packetBeforeRtoTracker, GetParam()); - conn.outstandingPackets.emplace_back(OutstandingPacket( - std::move(packetAfterRto), Clock::now(), 0, false, false, 0)); - - ReadAckFrame ackFrame; - ackFrame.largestAcked = 7; - ackFrame.ackBlocks.emplace_back(5, 7); - EXPECT_CALL(*rawController, onPacketAckOrLoss(_, _)) - .Times(1) - .WillOnce(Invoke([&](auto ackEvent, auto) { - EXPECT_EQ(1, ackEvent->ackedPackets.size()); - EXPECT_EQ(0, ackEvent->ackedPackets.front().encodedSize); - EXPECT_EQ(0, ackEvent->ackedPackets.front().totalBytesSent); - EXPECT_FALSE(ackEvent->ackedPackets.front().isHandshake); - EXPECT_FALSE(ackEvent->ackedPackets.front().pureAck); - })); - EXPECT_CALL(*rawController, onRTOVerified()).Times(0); - processAckFrame( - conn, - GetParam(), - ackFrame, - [](const auto&, const auto&, const auto&) {}, - [](auto&, auto&, bool, PacketNum) {}, - Clock::now()); - EXPECT_FALSE(conn.lossState.largestSentBeforeRto.hasValue()); - // Nothing is outstanding any more - EXPECT_TRUE(conn.outstandingPackets.empty()); - EXPECT_EQ(0, conn.outstandingPureAckPacketsCount); - EXPECT_FALSE(conn.pendingEvents.setLossDetectionAlarm); -} - TEST_P(AckHandlersTest, NoNewAckedPacket) { QuicServerConnectionState conn; auto mockController = std::make_unique(); auto rawController = mockController.get(); conn.congestionController = std::move(mockController); - conn.lossState.largestSentBeforeRto = 5; conn.lossState.rtoCount = 1; - PacketNum packetAfterRtoNum = 10; auto packetAfterRto = createNewPacket(packetAfterRtoNum, GetParam()); conn.outstandingPackets.emplace_back(OutstandingPacket( @@ -554,7 +434,6 @@ TEST_P(AckHandlersTest, NoNewAckedPacket) { ReadAckFrame ackFrame; ackFrame.largestAcked = 5; EXPECT_CALL(*rawController, onPacketAckOrLoss(_, _)).Times(0); - EXPECT_CALL(*rawController, onRTOVerified()).Times(0); processAckFrame( conn, GetParam(), @@ -562,7 +441,6 @@ TEST_P(AckHandlersTest, NoNewAckedPacket) { [](const auto&, const auto&, const auto&) {}, [](auto&, auto&, bool, PacketNum) {}, Clock::now()); - EXPECT_TRUE(conn.lossState.largestSentBeforeRto.hasValue()); EXPECT_TRUE(conn.pendingEvents.setLossDetectionAlarm); EXPECT_EQ(conn.lossState.rtoCount, 1); EXPECT_EQ(conn.ackStates.appDataAckState.largestAckedByPeer, 0); @@ -583,13 +461,14 @@ TEST_P(AckHandlersTest, LossByAckedRecovered) { [](const auto&, const auto&, const auto&) {}, [](auto&, auto&, bool, PacketNum) {}, Clock::now()); - EXPECT_FALSE(conn.lossState.largestSentBeforeRto.hasValue()); } TEST_P(AckHandlersTest, AckPacketNumDoesNotExist) { QuicServerConnectionState conn; auto mockController = std::make_unique(); conn.congestionController = std::move(mockController); + // Get the time based loss detection out of the way + conn.lossState.srtt = std::chrono::seconds(10); PacketNum packetNum1 = 9; auto regularPacket1 = createNewPacket(packetNum1, GetParam()); @@ -614,7 +493,6 @@ TEST_P(AckHandlersTest, AckPacketNumDoesNotExist) { [](const auto&, const auto&, const auto&) {}, [](auto&, auto&, bool, PacketNum) {}, Clock::now()); - EXPECT_FALSE(conn.lossState.largestSentBeforeRto.hasValue()); EXPECT_EQ(1, conn.outstandingPackets.size()); } @@ -1077,6 +955,65 @@ TEST_P(AckHandlersTest, UpdatePendingAckStates) { EXPECT_EQ(111 + 1357, conn.lossState.totalBytesAcked); } +TEST_F(AckHandlersTest, PureAckDoesNotUpdateRtt) { + QuicServerConnectionState conn; + conn.congestionController = nullptr; + PacketNum packetNum = 0; + auto regularPacket = createNewPacket(packetNum, PacketNumberSpace::AppData); + conn.outstandingPackets.emplace_back(OutstandingPacket( + std::move(regularPacket), + Clock::now(), + 111, + false /* handshake */, + true /* pureAck */, + 111)); + conn.outstandingPureAckPacketsCount++; + ASSERT_FALSE(conn.outstandingPackets.empty()); + ReadAckFrame ackFrame; + ackFrame.largestAcked = 0; + ackFrame.ackDelay = std::chrono::microseconds(100); + ackFrame.ackBlocks.emplace_back(0, 0); + processAckFrame( + conn, + PacketNumberSpace::AppData, + ackFrame, + [&](const auto&, const auto&, const auto&) {}, + [&](auto&, auto&, bool, auto) {}, + Clock::now()); + EXPECT_EQ(std::chrono::microseconds::max(), conn.lossState.mrtt); + EXPECT_EQ(std::chrono::microseconds::zero(), conn.lossState.srtt); + EXPECT_EQ(std::chrono::microseconds::zero(), conn.lossState.lrtt); + EXPECT_EQ(std::chrono::microseconds::zero(), conn.lossState.rttvar); + EXPECT_TRUE(conn.outstandingPackets.empty()); + + packetNum++; + regularPacket = createNewPacket(packetNum, PacketNumberSpace::AppData); + conn.outstandingPackets.emplace_back(OutstandingPacket( + std::move(regularPacket), + Clock::now(), + 111, + false /* handshake */, + false /* pureAck */, + 111)); + ASSERT_FALSE(conn.outstandingPackets.empty()); + ackFrame.largestAcked = 1; + ackFrame.ackDelay = std::chrono::microseconds(100); + ackFrame.ackBlocks.clear(); + ackFrame.ackBlocks.emplace_back(1, 1); + processAckFrame( + conn, + PacketNumberSpace::AppData, + ackFrame, + [&](const auto&, const auto&, const auto&) {}, + [&](auto&, auto&, bool, auto) {}, + Clock::now()); + EXPECT_NE(std::chrono::microseconds::max(), conn.lossState.mrtt); + EXPECT_NE(std::chrono::microseconds::zero(), conn.lossState.srtt); + EXPECT_NE(std::chrono::microseconds::zero(), conn.lossState.lrtt); + EXPECT_NE(std::chrono::microseconds::zero(), conn.lossState.rttvar); + EXPECT_TRUE(conn.outstandingPackets.empty()); +} + INSTANTIATE_TEST_CASE_P( AckHandlersTests, AckHandlersTest, diff --git a/quic/state/test/Mocks.h b/quic/state/test/Mocks.h index 649fb4ee5..0a29bc205 100644 --- a/quic/state/test/Mocks.h +++ b/quic/state/test/Mocks.h @@ -22,7 +22,6 @@ class MockCongestionController : public CongestionController { MOCK_METHOD2( onPacketAckOrLoss, void(folly::Optional, folly::Optional)); - MOCK_METHOD0(onRTOVerified, void()); MOCK_CONST_METHOD0(getWritableBytes, uint64_t()); MOCK_CONST_METHOD0(getCongestionWindow, uint64_t()); MOCK_METHOD0(onSpuriousLoss, void()); diff --git a/quic/state/test/QuicStateFunctionsTest.cpp b/quic/state/test/QuicStateFunctionsTest.cpp index f72557bae..fef79998c 100644 --- a/quic/state/test/QuicStateFunctionsTest.cpp +++ b/quic/state/test/QuicStateFunctionsTest.cpp @@ -313,75 +313,36 @@ class QuicStateFunctionsTest : public TestWithParam {}; TEST_F(QuicStateFunctionsTest, RttCalculationNoAckDelay) { QuicServerConnectionState conn; - auto packet = makeTestShortPacket(); - auto thisMoment = quic::Clock::now(); - MockClock::mockNow = [=]() { return thisMoment; }; - conn.outstandingPackets.emplace_back(OutstandingPacket( - packet, - MockClock::now() - std::chrono::microseconds(1000), - 0, - false, - false, - 0)); - - auto rttSample = std::chrono::duration_cast( - thisMoment + std::chrono::microseconds(100) - - conn.outstandingPackets.back().time); - updateRtt(conn, rttSample, std::chrono::microseconds::zero(), false); + auto rttSample = std::chrono::microseconds(1100); + updateRtt(conn, rttSample, std::chrono::microseconds::zero()); EXPECT_EQ(1100, conn.lossState.srtt.count()); EXPECT_EQ(1100 / 2, conn.lossState.rttvar.count()); + EXPECT_EQ(std::chrono::microseconds(0), conn.lossState.maxAckDelay); } TEST_F(QuicStateFunctionsTest, RttCalculationWithAckDelay) { QuicServerConnectionState conn; - auto packet = makeTestShortPacket(); - auto thisMoment = quic::Clock::now(); - MockClock::mockNow = [=]() { return thisMoment; }; - conn.outstandingPackets.emplace_back(OutstandingPacket( - packet, - MockClock::now() - std::chrono::microseconds(900), - 0, - false, - false, - 0)); - - // We only deduct by ackDelay if mrtt is smaller than current rtt sample - conn.lossState.mrtt = std::chrono::microseconds::zero(); - auto rttSample = std::chrono::duration_cast( - thisMoment + std::chrono::microseconds(100) - - conn.outstandingPackets.back().time); - updateRtt(conn, rttSample, std::chrono::microseconds(300), false); - EXPECT_EQ(700, conn.lossState.srtt.count()); - EXPECT_EQ(350, conn.lossState.rttvar.count()); -} - -TEST_F(QuicStateFunctionsTest, RttCalculationWithAckDelayAndLargeMinRtt) { - QuicServerConnectionState conn; - auto packet = makeTestShortPacket(); - auto thisMoment = quic::Clock::now(); - MockClock::mockNow = [=]() { return thisMoment; }; - conn.outstandingPackets.emplace_back(OutstandingPacket( - packet, - MockClock::now() - std::chrono::microseconds(900), - 0, - false, - false, - 0)); - - // make mrtt so large that we won't use this ackDelay - conn.lossState.mrtt = std::chrono::microseconds::max(); - auto rttSample = std::chrono::duration_cast( - thisMoment + std::chrono::microseconds(100) - - conn.outstandingPackets.back().time); - updateRtt(conn, rttSample, std::chrono::microseconds(300), false); + auto rttSample = std::chrono::microseconds(1000); + updateRtt(conn, rttSample, std::chrono::microseconds(300)); EXPECT_EQ(1000, conn.lossState.srtt.count()); EXPECT_EQ(500, conn.lossState.rttvar.count()); + EXPECT_EQ(std::chrono::microseconds(300), conn.lossState.maxAckDelay); +} + +TEST_F(QuicStateFunctionsTest, RttCalculationWithMrttAckDelay) { + QuicServerConnectionState conn; + conn.lossState.mrtt = std::chrono::microseconds(100); + auto rttSample = std::chrono::microseconds(1000); + updateRtt(conn, rttSample, std::chrono::microseconds(300)); + EXPECT_EQ(700, conn.lossState.srtt.count()); + EXPECT_EQ(350, conn.lossState.rttvar.count()); + EXPECT_EQ(std::chrono::microseconds(300), conn.lossState.maxAckDelay); } TEST_F(QuicStateFunctionsTest, TestInvokeStreamStateMachineConnectionError) { QuicServerConnectionState conn; QuicStreamState stream(1, conn); - RstStreamFrame rst(1, ApplicationErrorCode::STOPPING, 100); + RstStreamFrame rst(1, GenericApplicationErrorCode::UNKNOWN, 100); stream.finalReadOffset = 1024; EXPECT_THROW( invokeStreamStateMachine(conn, stream, std::move(rst)), @@ -394,7 +355,7 @@ TEST_F(QuicStateFunctionsTest, TestInvokeStreamStateMachineConnectionError) { TEST_F(QuicStateFunctionsTest, InvokeResetDoesNotSendFlowControl) { QuicServerConnectionState conn; QuicStreamState stream(1, conn); - RstStreamFrame rst(1, ApplicationErrorCode::STOPPING, 90); + RstStreamFrame rst(1, GenericApplicationErrorCode::UNKNOWN, 90); // this would normally trigger a flow control update. stream.flowControlState.advertisedMaxOffset = 100; stream.flowControlState.windowSize = 100; @@ -413,7 +374,7 @@ TEST_F(QuicStateFunctionsTest, TestInvokeStreamStateMachineStreamError) { // a good idea? We'll find out. QuicServerConnectionState conn; QuicStreamState stream(1, conn); - RstStreamFrame rst(1, ApplicationErrorCode::STOPPING, 100); + RstStreamFrame rst(1, GenericApplicationErrorCode::UNKNOWN, 100); try { invokeStreamStateMachine(conn, stream, StreamEvents::RstAck(rst)); ADD_FAILURE(); @@ -427,92 +388,37 @@ TEST_F(QuicStateFunctionsTest, TestInvokeStreamStateMachineStreamError) { TEST_F(QuicStateFunctionsTest, UpdateMinRtt) { QuicServerConnectionState conn; - auto packet0 = makeTestShortPacket(); - auto thisMoment = quic::Clock::now(); - conn.outstandingPackets.emplace_back( - OutstandingPacket(packet0, thisMoment, 0, false, false, 0)); // First rtt sample, will be assign to both srtt and mrtt - auto rttSample = std::chrono::duration_cast( - thisMoment + std::chrono::microseconds(100) - - conn.outstandingPackets.back().time); - updateRtt(conn, rttSample, std::chrono::microseconds::zero(), false); + auto rttSample = std::chrono::microseconds(100); + updateRtt(conn, rttSample, std::chrono::microseconds::zero()); EXPECT_EQ(std::chrono::microseconds(100), conn.lossState.lrtt); EXPECT_EQ(conn.lossState.lrtt, conn.lossState.mrtt); EXPECT_EQ(conn.lossState.lrtt, conn.lossState.srtt); auto oldMrtt = conn.lossState.mrtt; // Slower packet - auto packet1 = makeTestShortPacket(); - conn.outstandingPackets.emplace_back(OutstandingPacket( - packet1, - thisMoment + std::chrono::microseconds(150), - 0, - false, - false, - 0)); - rttSample = std::chrono::duration_cast( - thisMoment + std::chrono::microseconds(700) - - conn.outstandingPackets.back().time); - updateRtt(conn, rttSample, std::chrono::microseconds::zero(), false); + rttSample = std::chrono::microseconds(550); + updateRtt(conn, rttSample, std::chrono::microseconds::zero()); EXPECT_EQ(oldMrtt, conn.lossState.mrtt); // Faster packet - auto packet2 = makeTestShortPacket(); - conn.outstandingPackets.emplace_back(OutstandingPacket( - packet2, - thisMoment + std::chrono::microseconds(800), - 0, - false, - false, - 0)); - rttSample = std::chrono::duration_cast( - thisMoment + std::chrono::microseconds(820) - - conn.outstandingPackets.back().time); - updateRtt(conn, rttSample, std::chrono::microseconds::zero(), false); + rttSample = std::chrono::microseconds(20); + updateRtt(conn, rttSample, std::chrono::microseconds::zero()); EXPECT_EQ(std::chrono::microseconds(20), conn.lossState.mrtt); } TEST_F(QuicStateFunctionsTest, UpdateMaxAckDelay) { QuicServerConnectionState conn; - conn.lossState.mrtt = std::chrono::microseconds::zero(); - auto thisMoment = quic::Clock::now(); + EXPECT_EQ(std::chrono::microseconds::zero(), conn.lossState.maxAckDelay); + auto rttSample = std::chrono::microseconds(100); - auto packet0 = makeTestShortPacket(); - conn.outstandingPackets.emplace_back( - OutstandingPacket(packet0, thisMoment, 0, false, false, 0)); - auto rttSample = std::chrono::duration_cast( - thisMoment + std::chrono::microseconds(100) - - conn.outstandingPackets.back().time); - updateRtt(conn, rttSample, std::chrono::microseconds(30), false); + // update maxAckDelay + updateRtt(conn, rttSample, std::chrono::microseconds(30)); EXPECT_EQ(std::chrono::microseconds(30), conn.lossState.maxAckDelay); - auto packet1 = makeTestShortPacket(); - conn.outstandingPackets.emplace_back(OutstandingPacket( - packet1, - thisMoment + std::chrono::microseconds(200), - 0, - false, - false, - 0)); - rttSample = std::chrono::duration_cast( - thisMoment + std::chrono::microseconds(300) - - conn.outstandingPackets.back().time); - updateRtt(conn, rttSample, std::chrono::microseconds(3), false); + // smaller ackDelay + updateRtt(conn, rttSample, std::chrono::microseconds(3)); EXPECT_EQ(std::chrono::microseconds(30), conn.lossState.maxAckDelay); - - auto packet2 = makeTestShortPacket(); - conn.outstandingPackets.emplace_back(OutstandingPacket( - packet2, - thisMoment + std::chrono::microseconds(400), - 0, - false, - false, - 0)); - rttSample = std::chrono::duration_cast( - thisMoment + std::chrono::microseconds(600) - - conn.outstandingPackets.back().time); - updateRtt(conn, rttSample, std::chrono::microseconds(100), false); - EXPECT_EQ(std::chrono::microseconds(100), conn.lossState.maxAckDelay); } TEST_F(QuicStateFunctionsTest, IsConnectionPaced) { diff --git a/quic/state/test/QuicStreamFunctionsTest.cpp b/quic/state/test/QuicStreamFunctionsTest.cpp index 84adefc73..dd34b5108 100644 --- a/quic/state/test/QuicStreamFunctionsTest.cpp +++ b/quic/state/test/QuicStreamFunctionsTest.cpp @@ -1517,7 +1517,7 @@ TEST_F(QuicStreamFunctionsTest, RemovedClosedState) { conn.streamManager->addLoss(streamId); conn.streamManager->queueWindowUpdate(streamId); conn.streamManager->addStopSending( - streamId, ApplicationErrorCode::HTTP_NO_ERROR); + streamId, GenericApplicationErrorCode::UNKNOWN); stream->state = StreamStates::Closed{}; conn.streamManager->removeClosedStream(streamId); EXPECT_FALSE(conn.streamManager->streamExists(streamId)); @@ -1760,9 +1760,9 @@ TEST_F(QuicServerStreamFunctionsTest, TestAppendPendingStreamResetAllData) { stream.currentWriteOffset = len + 1; stream.retransmissionBuffer.clear(); - appendPendingStreamReset(conn, stream, ApplicationErrorCode::STOPPING); + appendPendingStreamReset(conn, stream, GenericApplicationErrorCode::UNKNOWN); auto rst = conn.pendingEvents.resets.at(id); - EXPECT_EQ(rst.errorCode, ApplicationErrorCode::STOPPING); + EXPECT_EQ(rst.errorCode, GenericApplicationErrorCode::UNKNOWN); EXPECT_EQ(rst.offset, len); } @@ -1779,9 +1779,9 @@ TEST_F( stream.currentWriteOffset = len; stream.retransmissionBuffer.clear(); - appendPendingStreamReset(conn, stream, ApplicationErrorCode::STOPPING); + appendPendingStreamReset(conn, stream, GenericApplicationErrorCode::UNKNOWN); auto rst = conn.pendingEvents.resets.at(id); - EXPECT_EQ(rst.errorCode, ApplicationErrorCode::STOPPING); + EXPECT_EQ(rst.errorCode, GenericApplicationErrorCode::UNKNOWN); EXPECT_EQ(rst.offset, len); } diff --git a/quic/state/test/TARGETS b/quic/state/test/TARGETS deleted file mode 100644 index dfdb3e111..000000000 --- a/quic/state/test/TARGETS +++ /dev/null @@ -1,104 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") -load("@fbcode_macros//build_defs:cpp_unittest.bzl", "cpp_unittest") - -cpp_library( - name = "mocks", - headers = [ - "Mocks.h", - ], - deps = [ - "//folly/portability:gmock", - "//quic:constants", - ], -) - -cpp_unittest( - name = "StateMachineTest", - srcs = [ - "StateDataTest.cpp", - "StateMachineTest.cpp", - ], - deps = [ - "//quic/common/test:test_utils", - "//quic/state:state_machine", - ], - external_deps = [ - ("googletest", None, "gmock"), - ], -) - -cpp_unittest( - name = "QuicStreamFunctionsTest", - srcs = [ - "QuicStreamFunctionsTest.cpp", - ], - deps = [ - "//quic/client/state:client", - "//quic/common/test:test_utils", - "//quic/server/state:server", - "//quic/state:stream_functions", - ], - external_deps = [ - ("googletest", None, "gmock"), - ], -) - -cpp_unittest( - name = "AckHandlersTest", - srcs = [ - "AckHandlersTest.cpp", - ], - deps = [ - ":mocks", - "//quic/common/test:test_utils", - "//quic/server/state:server", - "//quic/state:ack_handler", - "//quic/state:state_machine", - ], - external_deps = [ - ("googletest", None, "gmock"), - ], -) - -cpp_unittest( - name = "QuicStateFunctionsTest", - srcs = [ - "QuicStateFunctionsTest.cpp", - ], - deps = [ - ":mocks", - "//quic/common/test:test_utils", - "//quic/server/state:server", - "//quic/state:state_functions", - ], - external_deps = [ - ("googletest", None, "gmock"), - ], -) - -cpp_unittest( - name = "QuicPacingFunctionsTest", - srcs = [ - "QuicPacingFunctionsTest.cpp", - ], - deps = [ - "//folly/portability:gtest", - "//quic/state:pacing_functions", - ], -) - -cpp_unittest( - name = "QPRFunctionsTest", - srcs = [ - "QPRFunctionsTest.cpp", - ], - deps = [ - "//quic/server/state:server", - "//quic/state:qpr_functions", - ], - external_deps = [ - ("googletest", None, "gmock"), - ], -) diff --git a/quic/tools/TARGETS b/quic/tools/TARGETS deleted file mode 100644 index 7f3ad3361..000000000 --- a/quic/tools/TARGETS +++ /dev/null @@ -1,33 +0,0 @@ -# @autodeps - -load("@fbcode_macros//build_defs:python_binary.bzl", "python_binary") -load("@fbcode_macros//build_defs:python_library.bzl", "python_library") - -python_library( - name = "base", - srcs = [ - "base.py", - ], - base_module = "mvfst", - external_deps = [ - ("bcc-cpp", None, "bcc"), - ("bcc-py", None), - ], -) - -python_binary( - name = "print", - srcs = [ - "print.py", - ], - base_module = "mvfst", - main_module = "mvfst.print", - strip_libpar = False, - deps = [ - ":base", - ], - external_deps = [ - ("bcc-cpp", None, "bcc"), - ("bcc-py", None), - ], -)