diff --git a/.drone.jsonnet b/.drone.jsonnet index 87cd0e6a6..0f0e7b0a4 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -1,27 +1,24 @@ local events = ['pull_request', 'cron']; local platforms = { - develop: ['opensuse/leap:15', 'centos:7', 'centos:8', 'debian:10', 'ubuntu:18.04', 'ubuntu:20.04'], - 'develop-7': ['opensuse/leap:15', 'centos:7', 'centos:8', 'debian:10', 'ubuntu:18.04', 'ubuntu:20.04'], - 'develop-6': ['opensuse/leap:15', 'centos:7', 'centos:8', 'debian:10', 'ubuntu:18.04', 'ubuntu:20.04'], - 'develop-5': ['opensuse/leap:15', 'centos:7', 'centos:8', 'debian:10', 'ubuntu:18.04', 'ubuntu:20.04'], + develop: ['centos:7', 'rockylinux:8', 'debian:10', 'ubuntu:20.04'], + 'develop-6': ['centos:7', 'rockylinux:8', 'debian:10', 'ubuntu:20.04'], + 'develop-5': ['centos:7', 'rockylinux:8', 'debian:10', 'ubuntu:20.04'], }; local platforms_arm = { - develop: ['centos:8'], - 'develop-7': ['centos:8'], - 'develop-6': ['centos:8'], + develop: ['rockylinux:8'], + 'develop-6': ['rockylinux:8'], }; local any_branch = '**'; -local platforms_custom = ['opensuse/leap:15', 'centos:7', 'centos:8', 'debian:10', 'ubuntu:18.04', 'ubuntu:20.04']; -local platforms_arm_custom = ['centos:8']; +local platforms_custom = ['centos:7', 'rockylinux:8', 'debian:10', 'ubuntu:20.04']; +local platforms_arm_custom = ['rockylinux:8']; -local platforms_mtr = ['centos:7', 'centos:8', 'ubuntu:20.04']; +local platforms_mtr = ['centos:7', 'rockylinux:8', 'ubuntu:20.04']; local server_ref_map = { develop: '10.8', - 'develop-7': '10.7', 'develop-6': '10.6-enterprise', 'develop-5': '10.5', '**': '10.8', @@ -32,31 +29,31 @@ local builddir = 'verylongdirnameforverystrangecpackbehavior'; local cmakeflags = '-DCMAKE_BUILD_TYPE=RelWithDebInfo -DPLUGIN_COLUMNSTORE=YES -DPLUGIN_XPAND=NO -DPLUGIN_MROONGA=NO -DPLUGIN_ROCKSDB=NO ' + '-DPLUGIN_TOKUDB=NO -DPLUGIN_CONNECT=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_SPHINX=NO ' + '-DWITH_EMBEDDED_SERVER=OFF -DWITH_WSREP=OFF ' + - '-DBUILD_CONFIG=mysql_release -DWITH_UNITTESTS=YES'; + '-DBUILD_CONFIG=mysql_release -DWITH_UNITTESTS=YES -DCMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE=PRE_TEST'; local gcc_update_alternatives = 'update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 '; -local clang12_update_alternatives = 'update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-12 '; +local clang12_update_alternatives = 'update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-12 && update-alternatives --install /usr/bin/cc cc /usr/bin/clang 100 && update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 100 '; -local yum_vault_mirror = "sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-*; sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-Linux-*; "; local rpm_build_deps = 'install -y lz4 systemd-devel git make libaio-devel openssl-devel boost-devel bison snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect createrepo '; local centos7_build_deps = 'yum install -y epel-release centos-release-scl && yum install -y pcre2-devel devtoolset-10 devtoolset-10-gcc cmake3 lz4-devel && ln -s /usr/bin/cmake3 /usr/bin/cmake && . /opt/rh/devtoolset-10/enable '; -local centos8_build_deps = yum_vault_mirror + ' yum install -y gcc-toolset-10 libarchive dnf-plugins-core cmake lz4-devel && . /opt/rh/gcc-toolset-10/enable && yum config-manager --set-enabled powertools '; +local centos8_build_deps = 'dnf install -y gcc-toolset-10 libarchive cmake lz4-devel && . /opt/rh/gcc-toolset-10/enable '; +local rockylinux8_powertools = "dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools "; local ubuntu18_04_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-12 main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-12 &&' + clang12_update_alternatives; local debian10_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/buster/ llvm-toolchain-buster-12 main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-12 &&' + clang12_update_alternatives; -local opensuse_build_deps = 'zypper install -y liblz4-devel cmake libboost_system-devel pcre2-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel gcc-fortran gcc10 gcc10-c++ && ' + gcc_update_alternatives; -local deb_build_deps = 'apt update --yes && apt install --yes --no-install-recommends build-essential devscripts ccache equivs eatmydata dh-systemd && mk-build-deps debian/control -t "apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends" -r -i '; -local ubuntu20_04_deps = 'apt update --yes && apt install -y g++-10 && ' + gcc_update_alternatives; +local opensuse_build_deps = 'zypper install -y clang12 liblz4-devel cmake libboost_system-devel pcre2-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel gcc-fortran gcc10 gcc10-c++ && ' + clang12_update_alternatives; +local deb_build_deps = 'apt update --yes && apt install --yes --no-install-recommends build-essential devscripts git ccache equivs eatmydata dh-systemd && mk-build-deps debian/control -t "apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends" -r -i '; +local ubuntu20_04_deps = 'apt update --yes && apt install -y g++-10 git && ' + gcc_update_alternatives; - -local platformMap(platform) = +local platformMap(platform, arch) = + local clang_force = if (arch == 'arm64') then ' && export CXX=/usr/bin/clang++ && export CC=/usr/bin/clang ' else ''; local platform_map = { - 'opensuse/leap:15': opensuse_build_deps + ' && zypper ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=sles15 && make -j$(nproc) package', + 'opensuse/leap:15': opensuse_build_deps + ' && zypper ' + rpm_build_deps + clang_force + ' && cmake ' + cmakeflags + ' -DRPM=sles15 && make -j$(nproc) package', 'centos:7': centos7_build_deps + ' && yum ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=centos7 && make -j$(nproc) package', - 'centos:8': centos8_build_deps + ' && yum ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=centos8 && make -j$(nproc) package', + 'centos:8': centos8_build_deps + ' && dnf ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=centos8 && make -j$(nproc) package', + 'rockylinux:8': rockylinux8_powertools + ' && ' + centos8_build_deps + ' && dnf ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=rockylinux8 && make -j$(nproc) package', 'debian:10': deb_build_deps + " && CMAKEFLAGS='" + cmakeflags + " -DDEB=buster' debian/autobake-deb.sh", - 'ubuntu:18.04': ubuntu18_04_deps + ' && ' + deb_build_deps + " && CMAKEFLAGS='" + cmakeflags + " -DDEB=bionic' debian/autobake-deb.sh", 'ubuntu:20.04': ubuntu20_04_deps + ' && ' + deb_build_deps + " && CMAKEFLAGS='" + cmakeflags + " -DDEB=focal' debian/autobake-deb.sh", }; platform_map[platform]; @@ -67,8 +64,8 @@ local testRun(platform) = 'opensuse/leap:15': 'ctest -R columnstore: -j $(nproc) --output-on-failure', 'centos:7': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure', 'centos:8': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure', + 'rockylinux:8': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure', 'debian:10': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure', - 'ubuntu:18.04': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure', 'ubuntu:20.04': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure', }; platform_map[platform]; @@ -76,26 +73,30 @@ local testRun(platform) = local testPreparation(platform) = local platform_map = { - 'opensuse/leap:15': 'zypper install -y gtest boost-devel libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel cppunit-devel snappy-devel cmake', - 'centos:7': 'yum -y install epel-release && yum install -y gtest-devel cppunit-devel cmake3 boost-devel snappy-devel', - 'centos:8': yum_vault_mirror + ' yum install -y dnf-plugins-core libarchive && yum config-manager --set-enabled powertools && yum install -y lz4 gtest-devel cppunit-devel cmake3 boost-devel snappy-devel', - 'debian:10': 'apt update && apt install --yes libboost-all-dev libgtest-dev libcppunit-dev libsnappy-dev googletest cmake', - 'ubuntu:18.04': 'apt update && apt install --yes libboost-all-dev libgtest-dev libcppunit-dev googletest libsnappy-dev cmake g++ && cd /usr/src/googletest; cmake . && cmake --build . --target install; cd -', - 'ubuntu:20.04': 'apt update && apt install --yes libboost-all-dev libgtest-dev libcppunit-dev googletest libsnappy-dev cmake', + 'opensuse/leap:15': 'zypper install -y git boost-devel libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel cppunit-devel snappy-devel cmake', + 'centos:7': 'yum -y install epel-release && yum install -y git cppunit-devel cmake3 boost-devel snappy-devel', + 'centos:8': 'dnf install -y git lz4 cppunit-devel cmake3 boost-devel snappy-devel', + 'rockylinux:8': rockylinux8_powertools + ' && dnf install -y git lz4 cppunit-devel cmake3 boost-devel snappy-devel', + 'debian:10': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake', + 'ubuntu:18.04': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake g++', + 'ubuntu:20.04': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake', }; platform_map[platform]; local Pipeline(branch, platform, event, arch='amd64') = { - local pkg_format = if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'opensuse/leap') then 'rpm' else 'deb', + local pkg_format = if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'opensuse/leap' || std.split(platform, ':')[0] == 'rockylinux') then 'rpm' else 'deb', local init = if (pkg_format == 'rpm') then '/usr/lib/systemd/systemd' else 'systemd', local mtr_path = if (pkg_format == 'rpm') then '/usr/share/mysql-test' else '/usr/share/mysql/mysql-test', local socket_path = if (pkg_format == 'rpm') then '/var/lib/mysql/mysql.sock' else '/run/mysqld/mysqld.sock', local config_path_prefix = if (pkg_format == 'rpm') then '/etc/my.cnf.d/' else '/etc/mysql/mariadb.conf.d/50-', - local img = if (std.split(platform, ':')[0] == 'centos') then platform else 'romcheck/' + std.strReplace(platform, '/', '-'), + local img = if (platform == 'centos:7' || std.split(platform, ':')[0] == 'rockylinux') then platform else 'romcheck/' + std.strReplace(platform, '/', '-'), local regression_ref = if (std.split(branch, '-')[0] == 'develop') then branch else 'develop-6', local branchp = if (branch == '**') then '' else branch, + local container_tags = if (event == 'cron') then [branch, branch + '-' + std.strReplace(event, '_', '-') + '-${DRONE_BUILD_NUMBER}'] else [branch + '-' + std.strReplace(event, '_', '-') + '-${DRONE_BUILD_NUMBER}'], + local container_version = branch + '/' + event + '/${DRONE_BUILD_NUMBER}/' + arch, + local server_remote = if (std.split(branch, '-')[0] == 'columnstore' || branch == 'develop-6') then 'https://github.com/mariadb-corporation/MariaDBEnterprise' else 'https://github.com/MariaDB/server', local pipeline = self, @@ -119,6 +120,7 @@ local Pipeline(branch, platform, event, arch='amd64') = { delete: 'true', }, }, + _volumes:: { mdb: { name: 'mdb', @@ -136,9 +138,7 @@ local Pipeline(branch, platform, event, arch='amd64') = { commands: [ 'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name smoke$${DRONE_BUILD_NUMBER} --privileged --detach ' + img + ' ' + init + ' --unit=basic.target', 'docker cp result smoke$${DRONE_BUILD_NUMBER}:/', - if (platform == 'centos:8') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "' + yum_vault_mirror + '"' else '', - if (std.split(platform, ':')[0] == 'centos') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "yum install -y epel-release which rsyslog hostname && yum install -y /result/*.' + pkg_format + '"' else '', - if (platform == 'centos:8') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "sed -i s/enabled=0/enabled=1/ /etc/yum.repos.d/*PowerTools.repo && yum install -y gtest"' else '', + if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "yum install -y epel-release which rsyslog hostname procps-ng && yum install -y /result/*.' + pkg_format + '"' else '', if (pkg_format == 'deb') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d', if (pkg_format == 'deb') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y rsyslog hostname && apt install -y -f /result/*.' + pkg_format + '"' else '', if (std.split(platform, '/')[0] == 'opensuse') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "zypper install -y which hostname rsyslog && zypper install -y --allow-unsigned-rpm /result/*.' + pkg_format + '"' else '', @@ -161,9 +161,8 @@ local Pipeline(branch, platform, event, arch='amd64') = { commands: [ 'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env MYSQL_TEST_DIR=' + mtr_path + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name mtr$${DRONE_BUILD_NUMBER} --privileged --detach ' + img + ' ' + init + ' --unit=basic.target', 'docker cp result mtr$${DRONE_BUILD_NUMBER}:/', - if (platform == 'centos:8') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "' + yum_vault_mirror + '"' else '', if (std.split(platform, '/')[0] == 'opensuse') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "zypper install -y which hostname rsyslog patch perl-Data-Dumper-Concise perl-Memoize-ExpireLRU && zypper install -y --allow-unsigned-rpm /result/*.' + pkg_format + '"' else '', - if (std.split(platform, ':')[0] == 'centos') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "yum install -y epel-release diffutils which rsyslog hostname patch perl-Data-Dumper perl-Getopt-Long perl-Memoize perl-Time-HiRes cracklib-dicts && yum install -y /result/*.' + pkg_format + '"' else '', + if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "yum install -y epel-release diffutils which rsyslog hostname patch perl-Data-Dumper perl-Getopt-Long perl-Memoize perl-Time-HiRes cracklib-dicts procps-ng && yum install -y /result/*.' + pkg_format + '"' else '', if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d', if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y rsyslog hostname patch && apt install -y -f /result/*.' + pkg_format + '"' else '', 'docker cp mysql-test/columnstore mtr$${DRONE_BUILD_NUMBER}:' + mtr_path + '/suite/', @@ -224,8 +223,7 @@ local Pipeline(branch, platform, event, arch='amd64') = { 'docker cp /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager regression$${DRONE_BUILD_NUMBER}:/', // check storage-manager unit test binary file 'docker exec -t regression$${DRONE_BUILD_NUMBER} ls -l /storage-manager', - if (platform == 'centos:8') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "' + yum_vault_mirror + '"' else '', - if (std.split(platform, ':')[0] == 'centos') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "yum install -y epel-release diffutils tar lz4 wget which rsyslog hostname && yum install -y /result/*.' + pkg_format + '"' else '', + if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "yum install -y epel-release diffutils tar lz4 wget which rsyslog hostname procps-ng && yum install -y /result/*.' + pkg_format + '"' else '', if (pkg_format == 'deb') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d', if (pkg_format == 'deb') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y tar liblz4-tool wget rsyslog hostname && apt install -y -f /result/*.' + pkg_format + '"' else '', if (std.split(platform, '/')[0] == 'opensuse') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "zypper install -y gzip tar lz4 wget which hostname rsyslog && zypper install -y --allow-unsigned-rpm /result/*.' + pkg_format + '"' else '', @@ -282,46 +280,34 @@ local Pipeline(branch, platform, event, arch='amd64') = { }, dockerfile:: { name: 'dockerfile', - image: 'docker:git', - volumes: [pipeline._volumes.docker], + image: 'alpine/git', commands: [ - 'git clone --depth 1 https://github.com/mariadb-corporation/mariadb-community-columnstore-docker.git', - 'cd mariadb-community-columnstore-docker', - 'apk add --no-cache patch', - 'patch Dockerfile ../Dockerfile.patch', - 'cp ../result/MariaDB-common-10* ../result/MariaDB-client-10* ../result/MariaDB-server-10* ../result/MariaDB-shared-10* ../result/MariaDB-columnstore-engine-10* ./', + 'git clone --depth 1 https://github.com/mariadb-corporation/mariadb-skysql-columnstore-docker docker', + "sed -i 's|dlm.mariadb.com/enterprise-release-helpers/mariadb_es_repo_setup|cspkg.s3.amazonaws.com/cs_repo|' docker/Dockerfile", ], }, - ecr:: { - name: 'ecr', - image: 'plugins/ecr', - settings: { - registry: '866067714787.dkr.ecr.us-east-1.amazonaws.com', - repo: 'columnstore/engine', - context: 'mariadb-community-columnstore-docker', - dockerfile: 'mariadb-community-columnstore-docker/Dockerfile', - access_key: { - from_secret: 'aws_access_key_id', - }, - secret_key: { - from_secret: 'aws_secret_access_key', - }, - }, - }, - docker:: { - name: 'docker', + dockerhub:: { + name: 'dockerhub', image: 'plugins/docker', + environment: { + VERSION: container_version, + }, settings: { - repo: 'romcheck/columnstore', - context: '/drone/src/mariadb-community-columnstore-docker', - dockerfile: 'mariadb-community-columnstore-docker/Dockerfile', - username: 'romcheck', + repo: 'mariadb/enterprise-columnstore-dev', + context: 'docker', + dockerfile: 'docker/Dockerfile', + build_args_from_env: ['VERSION'], + tags: container_tags, + username: { + from_secret: 'dockerhub_user', + }, password: { - from_secret: 'dockerhub_token', + from_secret: 'dockerhub_password', }, }, }, + kind: 'pipeline', type: 'docker', name: std.join(' ', [branch, platform, event, arch]), @@ -362,7 +348,7 @@ local Pipeline(branch, platform, event, arch='amd64') = { }, { name: 'build', - image: platform, + image: img, volumes: [pipeline._volumes.mdb], environment: { DEBIAN_FRONTEND: 'noninteractive', @@ -370,7 +356,6 @@ local Pipeline(branch, platform, event, arch='amd64') = { }, commands: [ 'cd /mdb/' + builddir, - if (platform == 'centos:8') then yum_vault_mirror else '', // Remove Debian build flags that could prevent ColumnStore from building "sed '/-DPLUGIN_COLUMNSTORE=NO/d' -i debian/rules", // Tweak debian packaging stuff @@ -392,22 +377,18 @@ local Pipeline(branch, platform, event, arch='amd64') = { // Leave test package for mtr "sed -i '/(mariadb|mysql)-test/d;/-test/d' debian/autobake-deb.sh", "sed -i '/test-embedded/d' debian/mariadb-test.install", - // From Debian Bullseye/Ubuntu Groovy, liburing replaces libaio - "apt-cache madison liburing-dev | grep 'liburing-dev' || sed 's/liburing-dev/libaio-dev/g' -i debian/control && sed '/-DIGNORE_AIO_CHECK=YES/d' -i debian/rules && sed '/-DWITH_URING=yes/d' -i debian/rules", - // From Debian Buster/Ubuntu Focal onwards libpmem-dev is available - "apt-cache madison libpmem-dev | grep 'libpmem-dev' || sed '/libpmem-dev/d' -i debian/control && sed '/-DWITH_PMEM/d' -i debian/rules", + // Deb dependencies from server scripts + if (pkg_format == 'deb') then "apt-cache madison liburing-dev | grep liburing-dev || sed 's/liburing-dev/libaio-dev/g' -i debian/control && sed '/-DIGNORE_AIO_CHECK=YES/d' -i debian/rules && sed '/-DWITH_URING=yes/d' -i debian/rules && apt-cache madison libpmem-dev | grep 'libpmem-dev' || sed '/libpmem-dev/d' -i debian/control && sed '/-DWITH_PMEM/d' -i debian/rules && sed '/libfmt-dev/d' -i debian/control" else '', // Change plugin_maturity level // "sed -i 's/BETA/GAMMA/' storage/columnstore/CMakeLists.txt", - // Workaround till upstream removes 4535 workaround (workaround for workaround!) - "sed -i '/MCOL-4535/,/^$/d' debian/autobake-deb.sh", testPreparation(platform), - platformMap(platform), + platformMap(platform, arch), if (pkg_format == 'rpm') then 'createrepo .' else 'dpkg-scanpackages ../ | gzip > ../Packages.gz', ], }, { name: 'unittests', - image: platform, + image: img, volumes: [pipeline._volumes.mdb], environment: { DEBIAN_FRONTEND: 'noninteractive', @@ -437,7 +418,7 @@ local Pipeline(branch, platform, event, arch='amd64') = { ] + [pipeline.publish()] + (if (event == 'cron') || (event == 'push') then [pipeline.publish('pkg latest', 'latest')] else []) + - // (if (platform == 'centos:8' && event == 'cron') then [pipeline.dockerfile] + [pipeline.docker] + [pipeline.ecr] else []) + + (if (event != 'custom') && (platform == 'rockylinux:8') && (arch == 'amd64') && (branch != 'develop-5') then [pipeline.dockerfile] + [pipeline.dockerhub] else []) + [pipeline.smoke] + [pipeline.smokelog] + (if (std.member(platforms_mtr, platform)) then [pipeline.mtr] + [pipeline.mtrlog] + [pipeline.publish('mtr')] else []) + diff --git a/.gitignore b/.gitignore index c35a65112..3ed72171a 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,10 @@ *.so.* *.vpj .libs/ +.cache/ +compile_commands.json +*.code-workspace +settings.json config.log *~ *kdev* @@ -172,3 +176,4 @@ mariadb-columnstore-regression-test/ build/Testing/ tests/*\[1\]_tests.cmake tests/*\[1\]_include.cmake +.boost diff --git a/CMakeLists.txt b/CMakeLists.txt index 079d9abfc..ca8542fbf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,7 +28,7 @@ endif() INCLUDE(ExternalProject) INCLUDE(CheckCXXSourceCompiles) -SET(CMAKE_CXX_STANDARD 17) +SET(CMAKE_CXX_STANDARD 20) SET(CMAKE_CXX_STANDARD_REQUIRED TRUE) SET(CMAKE_CXX_EXTENSIONS FALSE) SET(CMAKE_EXPORT_COMPILE_COMMANDS TRUE) @@ -95,12 +95,7 @@ SET (ENGINE_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) INCLUDE(columnstore_version) INCLUDE(misc) - -FIND_PACKAGE(Boost 1.53.0 COMPONENTS system filesystem thread regex date_time chrono atomic) -IF (NOT Boost_FOUND) - MESSAGE_ONCE(CS_NO_BOOST "Required Boost libraries not found!") - return() -ENDIF() +INCLUDE(boost) FIND_PACKAGE(BISON) IF (NOT BISON_FOUND) @@ -149,14 +144,6 @@ if (NOT CURL_FOUND) return() endif() -IF (WITH_UNITTESTS OR WITH_GTEST) - INCLUDE (FindGTest) - IF (NOT GTEST_FOUND) - MESSAGE(FATAL_ERROR "GSuite libs not found but are requested. Please install them or build.") - ENDIF() - SET (GTEST_LIBRARIES ${GTEST_LIBRARY} ${GTESTMAIN_LIBRARY} ${PTHREAD_LIBRARY}) -ENDIF() - FIND_PROGRAM(AWK_EXECUTABLE awk DOC "path to the awk executable") if(NOT AWK_EXECUTABLE) MESSAGE_ONCE(CS_NO_AWK "awk not found!") @@ -222,8 +209,13 @@ IF (MASK_LONGDOUBLE) MY_CHECK_AND_SET_COMPILER_FLAG("-DMASK_LONGDOUBLE") ENDIF() +if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-deprecated-enum-enum-conversion -Wno-register") +elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-register") +endif() -SET (CMAKE_REQUIRED_FLAGS "-Werror -Wall") +MY_CHECK_AND_SET_COMPILER_FLAG("-Werror -Wall") SET (ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed") SET (ENGINE_DT_LIB datatypes) SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt libmysql_client ${ENGINE_DT_LIB}) @@ -346,7 +338,6 @@ ADD_SUBDIRECTORY(dbcon/ddlpackage) ADD_SUBDIRECTORY(dbcon/ddlpackageproc) ADD_SUBDIRECTORY(dbcon/dmlpackage) ADD_SUBDIRECTORY(dbcon/dmlpackageproc) -ADD_SUBDIRECTORY(exemgr) ADD_SUBDIRECTORY(ddlproc) ADD_SUBDIRECTORY(dmlproc) ADD_SUBDIRECTORY(oamapps) diff --git a/build/README.md b/build/README.md index 9ee3599bc..1a4bbaa75 100644 --- a/build/README.md +++ b/build/README.md @@ -4,13 +4,15 @@ This is MariaDB Columnstore To build MCS from source you will need: * modern linux distribution. e.g. Ubuntu 20 + * modern C++ compiler: clang version greater than 12 or gcc version greater than 11 Clone or download this repository. git clone https://github.com/MariaDB/server -Edit a bootstrap script to fix paths to MariaDB repo cloned. - vim /some/path/server/storage/columnstore/columnstore/build/bootstrap_mcs.sh +Update Sumbodules + git submodule update --init --recursive Run the bootstrap /some/path/server/storage/columnstore/columnstore/build/bootstrap_mcs.sh +bootstrap_mcs.sh could be run with --help for list of flags diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index 8187e5e0c..02ac3477c 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -1,179 +1,304 @@ -#!/bin/sh +#!/bin/bash # This script compiles/installs MCS from scratch and it makes some assumptions: # - the server's source code is two directories above the MCS engine source. # - the script is to be run under root. -# - it is run MDB_MDB_SOURCE_PATH/storage/columnstore/columnstore, e.g. ./build/bootstrap_mcs.sh -DISTRO=$1 -MDB_BUILD_TYPE=$2 -MDB_SOURCE_PATH=`pwd`/../../../ -MCS_CONFIG_DIR=/etc/columnstore -# Needs systemd to be installed obviously. -# Feel free to ask in MariaDB Zulip how to bootstrap MCS in containers or other systemd-free environments. -systemctl stop mariadb-columnstore -MDB_GIT_URL=https://github.com/MariaDB/server.git -MDB_GIT_TAG=10.8 -if [ -z "$DISTRO" ]; then - echo "Choose a distro" + +SCRIPT_LOCATION=$(dirname "$0") +MDB_SOURCE_PATH=$(realpath $SCRIPT_LOCATION/../../../..) + +source $SCRIPT_LOCATION/utils.sh + + +if [ "$EUID" -ne 0 ] + then error "Please run script as root to install MariaDb to system paths" exit 1 fi -if [ $DISTRO = 'bionic' ]; then - sudo apt-get -y update - apt-get -y install build-essential automake libboost-all-dev bison cmake \ - libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev \ - libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev git \ - libsnappy-dev libcurl4-openssl-dev -elif [ $DISTRO = 'focal' ]; then - sudo apt-get -y update - apt-get -y install build-essential automake libboost-all-dev bison cmake \ - libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev \ - libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev git \ - libsnappy-dev libcurl4-openssl-dev -elif [ $DISTRO = 'centos' ]; then - yum -y install epel-release \ - && yum -y groupinstall "Development Tools" \ - && yum -y install bison ncurses-devel readline-devel perl-devel openssl-devel cmake libxml2-devel gperf libaio-devel libevent-devel python-devel ruby-devel tree wget pam-devel snappy-devel libicu \ - && yum -y install vim wget strace ltrace gdb rsyslog net-tools openssh-server expect boost perl-DBI libicu boost-devel initscripts jemalloc-devel libcurl-devel -elif [ $DISTRO = 'leap' ]; then - zypper install -y bison ncurses-devel readline-devel libopenssl-devel cmake libxml2-devel gperf libaio-devel libevent-devel python-devel ruby-devel tree wget pam-devel snappy-devel libicu-devel \ - && zypper install -y libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel \ - && zypper install -y vim wget strace ltrace gdb rsyslog net-tools expect perl-DBI libicu boost-devel jemalloc-devel libcurl-devel \ - && zypper install -y gcc gcc-c++ git automake libtool +message "Building Mariadb Server from $color_yellow$MDB_SOURCE_PATH$color_normal" + +BUILD_TYPE_OPTIONS=("Debug" "RelWithDebInfo") +DISTRO_OPTIONS=("Ubuntu" "CentOS" "Debian" "openSUSE" "Rocky") +BRANCHES=($(git branch --list --no-color| grep "[^* ]+" -Eo)) + +optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE +optparse.define short=d long=distro desc="Choouse your OS: ${DISTRO_OPTIONS[*]}" variable=OS +optparse.define short=s long=skip-deps desc="Skip install dependences" variable=SKIP_DEPS default=false value=true +optparse.define short=C long=force-cmake-reconfig desc="Force cmake reconfigure" variable=FORCE_CMAKE_CONFIG default=false value=true +optparse.define short=S long=skip-columnstore-submodules desc="Skip columnstore submodules initialization" variable=SKIP_SUBMODULES default=false value=true +optparse.define short=b long=branch desc="Choouse git branch ('none' for menu)" variable=BRANCH + +source $( optparse.build ) + +if [[ ! " ${BUILD_TYPE_OPTIONS[*]} " =~ " ${MCS_BUILD_TYPE} " ]]; then + getChoice -q "Select your Build Type" -o BUILD_TYPE_OPTIONS + MCS_BUILD_TYPE=$selectedChoice fi -if [ ! -d $MDB_SOURCE_PATH ]; then - git clone $MDB_GIT_URL $MDB_SOURCE_PATH -b $MDB_GIT_TAG +if [[ ! " ${DISTRO_OPTIONS[*]} " =~ " ${OS} " || $OS = "CentOS" ]]; then + detect_distro fi -if [ ! -d $MCS_CONFIG_DIR ]; then - mkdir $MCS_CONFIG_DIR -fi +INSTALL_PREFIX="/usr/" +DATA_DIR="/var/lib/mysql/data" +CMAKE_BIN_NAME=cmake -if [ -z "$(grep mysql /etc/passwd)" ]; then - echo "Adding user mysql into /etc/passwd" - useradd -r -U mysql -d /var/lib/mysql - exit 1 -fi +select_branch() +{ + if [[ ! " ${BRANCHES[*]} " =~ " ${BRANCH} " ]]; then + if [[ $BRANCH = 'none' ]]; then + getChoice -q "Select your branch" -o BRANCHES + BRANCH=$selectedChoice + fi + cd $SCRIPT_LOCATION + message "Selecting $BRANCH branch for Columnstore" + git checkout $BRANCH + cd - -if [ -z "$(grep mysql /etc/group)" ]; then - echo "You need to manually add mysql group into /etc/group, e.g. mysql:x:999" - exit 1 -fi + message "Turning off Columnstore submodule auto update via gitconfig" + cd $MDB_SOURCE_PATH + git config submodule.storage/columnstore/columnstore.update none + cd - + fi -MCS_INSTALL_PREFIX=/var/lib/ -rm -rf /var/lib/columnstore/data1/* -rm -rf /var/lib/columnstore/data/ -rm -rf /var/lib/columnstore/local/ -rm -f /var/lib/columnstore/storagemanager/storagemanager-lock -rm -f /var/lib/columnstore/storagemanager/cs-initialized + cd $SCRIPT_LOCATION + CURRENT_BRANCH=$(git branch --show-current) + cd - + message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_normal branch" +} -MCS_TMP_DIR=/tmp/columnstore_tmp_files -TMP_PATH=/tmp -CPUS=$(getconf _NPROCESSORS_ONLN) +install_deps() +{ + message "Installing deps" + if [[ $OS = 'Ubuntu' || $OS = 'Debian' ]]; then + sudo apt-get -y update + apt-get -y install build-essential automake libboost-all-dev bison cmake \ + libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev \ + libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev git \ + libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest libsnappy-dev libjemalloc-dev + elif [[ $OS = 'CentOS' || $OS = 'Rocky' ]]; then + yum -y install epel-release \ + && yum -y groupinstall "Development Tools" \ + && yum config-manager --set-enabled powertools \ + && yum -y install bison ncurses-devel readline-devel perl-devel openssl-devel libxml2-devel gperf libaio-devel libevent-devel tree wget pam-devel snappy-devel libicu \ + && yum -y install vim wget strace ltrace gdb rsyslog net-tools openssh-server expect boost perl-DBI libicu boost-devel initscripts jemalloc-devel libcurl-devel gtest-devel cppunit-devel systemd-devel + if [[ "$OS_VERSION" == "7" ]]; then + yum -y install cmake3 + CMAKE_BIN_NAME=cmake3 + else + yum -y install cmake + fi + if [ $OS = 'Rocky' ]; then + yum install -y checkpolicy + fi + elif [ $OS = 'openSUSE' ]; then + zypper install -y bison ncurses-devel readline-devel libopenssl-devel cmake libxml2-devel gperf libaio-devel libevent-devel python-devel ruby-devel tree wget pam-devel snappy-devel libicu-devel \ + && zypper install -y libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel \ + && zypper install -y vim wget strace ltrace gdb rsyslog net-tools expect perl-DBI libicu boost-devel jemalloc-devel libcurl-devel liblz4-devel lz4 \ + && zypper install -y gcc gcc-c++ git automake libtool gtest cppunit-devel pcre2-devel systemd-devel libaio-devel snappy-devel cracklib-devel policycoreutils-devel ncurses-devel \ + && zypper install -y make flex libcurl-devel automake libtool rpm-build lsof iproute pam-devel perl-DBI expect createrepo + fi +} -# script -rm -rf $MCS_TMP_DIR/* -rm -rf /var/lib/mysql +stop_service() +{ + message "Stopping MariaDB services" + systemctl stop mariadb + systemctl stop mariadb-columnstore +} -cd $MDB_SOURCE_PATH -if [[ "$DISTRO" = 'bionic' || "$DISTRO" = 'focal' ]]; then - #MDB_CMAKE_FLAGS='-DWITH_SYSTEMD=yes -DPLUGIN_TOKUDB=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_MROONGA=NO -DPLUGIN_GSSAPI=NO -DWITH_MARIABACKUP=NO -DDEB=bionic -DPLUGIN_COLUMNSTORE=YES' - MDB_CMAKE_FLAGS='-DWITH_SYSTEMD=yes -DPLUGIN_COLUMNSTORE=YES -DPLUGIN_MROONGA=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_TOKUDB=NO -DPLUGIN_CONNECT=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_SPHINX=NO -DBUILD_CONFIG=mysql_release -DWITH_WSREP=OFF -DWITH_SSL=system -DDEB=bionic' - # Some development flags - #MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_GTEST=1 -DWITH_ROWGROUP_UT=1 -DWITH_DATACONVERT_UT=1 -DWITH_ARITHMETICOPERATOR_UT=1 -DWITH_ORDERBY_UT=1 -DWITH_CSDECIMAL_UT=1 -DWITH_SORTING_COMPARATORS_UT=1" - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=YES -DWITH_BRM_UT=YES" #-DWITH_GTEST=1 -DWITH_UNITTESTS=YES - cmake . -DCMAKE_BUILD_TYPE=$MDB_BUILD_TYPE ${MDB_CMAKE_FLAGS} && \ +check_service() +{ + if systemctl is-active --quiet $1; then + message "$1 service started$color_green OK $color_normal" + else + error "$1 service failed" + service $1 status + fi +} + +start_service() +{ + message "Starting MariaDB services" + systemctl start mariadb-columnstore + systemctl start mariadb + + check_service mariadb-columnstore + check_service mariadb +} + +clean_old_installation() +{ + message "Cleaning old installation" + rm -rf /var/lib/columnstore/data1/* + rm -rf /var/lib/columnstore/data/ + rm -rf /var/lib/columnstore/local/ + rm -f /var/lib/columnstore/storagemanager/storagemanager-lock + rm -f /var/lib/columnstore/storagemanager/cs-initialized + rm -rf /tmp/* + rm -rf /var/lib/mysql + rm -rf /var/run/mysqld + rm -rf $DATA_DIR + rm -rf /etc/mysql +} + +build() +{ + message "Building sources in $color_yellow$MCS_BUILD_TYPE$color_normal mode" + + local MDB_CMAKE_FLAGS="-DWITH_SYSTEMD=yes + -DPLUGIN_COLUMNSTORE=YES + -DPLUGIN_MROONGA=NO + -DPLUGIN_ROCKSDB=NO + -DPLUGIN_TOKUDB=NO + -DPLUGIN_CONNECT=NO + -DPLUGIN_SPIDER=NO + -DPLUGIN_OQGRAPH=NO + -DPLUGIN_SPHINX=NO + -DWITH_EMBEDDED_SERVER=OFF + -DBUILD_CONFIG=mysql_release + -DWITH_WSREP=OFF + -DWITH_SSL=system + -DWITH_UNITTESTS=YES + -DWITH_BRM_UT=YES + -DCMAKE_INSTALL_PREFIX:PATH=$INSTALL_PREFIX" + + cd $MDB_SOURCE_PATH + + if [[ $SKIP_SUBMODULES = true ]] ; then + warn "Skipping initialization of columnstore submodules" + else + message "Initialization of columnstore submodules" + cd storage/columnstore/columnstore + git submodule update --init + cd - + fi + + if [[ $FORCE_CMAKE_CONFIG = true ]] ; then + warn "Erasing cmake cache" + rm -f "$MDB_SOURCE_PATH/CMakeCache.txt" + rm -rf "$MDB_SOURCE_PATH/CMakeFiles" + fi + + if [[ "$OS" = 'Ubuntu' || "$OS" = 'Debian' ]]; then + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DDEB=bionic" + elif [ $OS = 'CentOS' ]; then + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=CentOS7" + elif [ $OS = 'Rocky' ]; then + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=CentOS7" + elif [ $OS = 'openSUSE' ]; then + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=sles15" + fi + + message "building with flags $MDB_CMAKE_FLAGS" + + local CPUS=$(getconf _NPROCESSORS_ONLN) + ${CMAKE_BIN_NAME} . -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS && \ make -j $CPUS install -elif [ $DISTRO = 'centos' ]; then - MDB_CMAKE_FLAGS='-DWITH_SYSTEMD=yes -DPLUGIN_AUTH_GSSAPI=NO -DPLUGIN_COLUMNSTORE=YES -DPLUGIN_MROONGA=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_TOKUDB=NO -DPLUGIN_CONNECT=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_SPHINX=NO -DBUILD_CONFIG=mysql_release -DWITH_WSREP=OFF -DWITH_SSL=system -DRPM=CentOS7' - cmake . -DCMAKE_BUILD_TYPE=$MDB_BUILD_TYPE $MDB_CMAKE_FLAGS && \ - make -j $CPUS install -fi -if [ $? -ne 0 ]; then - return 1 -fi - -# These two lines are to handle file layout difference b/w RPM- and DEB-based distributions. -# One of the lines always fails. -mv /usr/lib/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_1.so -mv /usr/lib64/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_2.so + if [ $? -ne 0 ]; then + error "!!!! BUILD FAILED" + exit 1 + fi + cd - +} -/usr/bin/mysql_install_db --rpm --user=mysql +check_user_and_group() +{ + if [ -z "$(grep mysql /etc/passwd)" ]; then + message "Adding user mysql into /etc/passwd" + useradd -r -U mysql -d /var/lib/mysql + fi -mv /tmp/ha_columnstore_1.so /usr/lib/mysql/plugin/ha_columnstore.so -mv /tmp/ha_columnstore_2.so /usr/lib64/mysql/plugin/ha_columnstore.so + if [ -z "$(grep mysql /etc/group)" ]; then + GroupID = `awk -F: '{uid[$3]=1}END{for(x=100; x<=999; x++) {if(uid[x] != ""){}else{print x; exit;}}}' /etc/group` + message "Adding group mysql with id $GroupID" + groupadd -g GroupID mysql + fi +} -cp -r /etc/mysql/conf.d /etc/my.cnf.d/ -cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml -cp $MDB_SOURCE_PATH/storage/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml -cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf -cp $MDB_SOURCE_PATH/storage/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf +install() +{ + message "Installing MariaDB" + + check_user_and_group + + mkdir -p /etc/my.cnf.d + + bash -c 'echo "[client-server] +socket=/run/mysqld/mysqld.sock" > /etc/my.cnf.d/socket.cnf' + + mv $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_1.so || mv $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_2.so + message "Running mysql_install_db" + mysql_install_db --rpm --user=mysql + mv /tmp/ha_columnstore_1.so $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so || mv /tmp/ha_columnstore_2.so $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so + + mkdir -p /etc/columnstore + + cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml + cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf + + cp $MDB_SOURCE_PATH/support-files/*.service /lib/systemd/system/ + cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/ + + if [[ "$OS" = 'Ubuntu' || "$OS" = 'Debian' ]]; then + mkdir -p /usr/share/mysql + mkdir -p /etc/mysql/ + cp $MDB_SOURCE_PATH/debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh + cp $MDB_SOURCE_PATH/debian/additions/debian-start /etc/mysql/debian-start + > /etc/mysql/debian.cnf + fi -if [[ "$DISTRO" = 'bionic' || "$DISTRO" = 'focal' ]]; then - cp ./support-files/*.service /lib/systemd/system/ - cp ./storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/ - cp ./storage/columnstore/oam/install_scripts/*.service /lib/systemd/system/ - cp ./debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh - cp ./debian/additions/debian-start /etc/mysql/debian-start systemctl daemon-reload - rm -f /etc/mysql/my.cnf - cp -r /etc/mysql/conf.d/ /etc/my.cnf.d - cp -rp /etc/mysql/mariadb.conf.d/ /etc/my.cnf.d - mkdir /var/lib/columnstore/data1 - mkdir /var/lib/columnstore/data1/systemFiles - mkdir /var/lib/columnstore/data1/systemFiles/dbrm - chown -R mysql:mysql /var/lib/mysql - chown -R mysql:mysql /var/lib/mysql - chown -R mysql.mysql /var/run/mysqld - chown -R mysql:mysql /data/columnstore/* - chmod +x /usr/bin/mariadb* - cp /etc/my.cnf.d/mariadb.conf.d/columnstore.cnf /etc/my.cnf.d/ - ldconfig - columnstore-post-install - chown -R mysql:mysql /data/columnstore/* + if [ -d "/etc/mysql/mariadb.conf.d/" ]; then + message "Copying configs from /etc/mysql/mariadb.conf.d/ to /etc/my.cnf.d" + cp -rp /etc/mysql/mariadb.conf.d/* /etc/my.cnf.d + fi - /usr/sbin/install_mcs_mysql.sh + if [ -d "/etc/mysql/conf.d/" ]; then + message "Copying configs from /etc/mysql/conf.d/ to /etc/my.cnf.d" + cp -rp /etc/mysql/conf.d/* /etc/my.cnf.d + fi - chown -R syslog.syslog /var/log/mariadb/ - chmod 777 /var/log/mariadb/ - chmod 777 /var/log/mariadb/columnstore + mkdir -p /var/lib/columnstore/data1 + mkdir -p /var/lib/columnstore/data1/systemFiles + mkdir -p /var/lib/columnstore/data1/systemFiles/dbrm + mkdir -p /run/mysqld/ -elif [ $DISTRO = 'centos' ]; then - cp ./support-files/*.service /lib/systemd/system/ - cp ./storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/ - cp ./storage/columnstore/oam/install_scripts/*.service /lib/systemd/system/ - systemctl daemon-reload - rm -f /etc/mysql/my.cnf - cp -r /etc/mysql/conf.d/ /etc/my.cnf.d - mkdir /var/lib/columnstore/data1 - mkdir /var/lib/columnstore/data1/systemFiles - mkdir /var/lib/columnstore/data1/systemFiles/dbrm - chown -R mysql:mysql /var/lib/mysql - chown -R mysql:mysql /data/columnstore/* - chown -R mysql.mysql /var/run/mysqld - chmod +x /usr/bin/mariadb* + mkdir -p $DATA_DIR + chown -R mysql:mysql $DATA_DIR + chown -R mysql:mysql /var/lib/columnstore/ + chown -R mysql:mysql /run/mysqld/ - ldconfig - columnstore-post-install - chown -R mysql:mysql /data/columnstore/* + chmod +x $INSTALL_PREFIX/bin/mariadb* - /usr/sbin/install_mcs_mysql.sh + ldconfig - /usr/sbin/install_mcs_mysql.sh - mkdir /var/lib/columnstore/data1 - mkdir /var/lib/columnstore/data1/systemFiles - mkdir /var/lib/columnstore/data1/systemFiles/dbrm + message "Running columnstore-post-install" + mkdir -p /var/lib/columnstore/local + columnstore-post-install --rpmmode=install + message "Running install_mcs_mysql" + install_mcs_mysql.sh - chown -R mysql:mysql /var/log/mariadb/ - chmod 777 /var/log/mariadb/ - chmod 777 /var/log/mariadb/columnstore + chown -R syslog:syslog /var/log/mariadb/ + chmod 777 /var/log/mariadb/ + chmod 777 /var/log/mariadb/columnstore +} + + +select_branch + +if [[ $SKIP_DEPS = false ]] ; then + install_deps fi - -exit 0 +stop_service +clean_old_installation +build +install +start_service +message "$color_green FINISHED $color_normal" diff --git a/build/my.cnf.in b/build/my.cnf.in new file mode 100644 index 000000000..d2d2332ae --- /dev/null +++ b/build/my.cnf.in @@ -0,0 +1,10 @@ +[client-server] + +# +# include *.cnf from the config directory +# +!includedir /etc/my.cnf.d + +[mysqld] +datadir=%DATADIR% +plugin-load-add=ha_columnstore.so \ No newline at end of file diff --git a/build/utils.sh b/build/utils.sh new file mode 100644 index 000000000..133bfb036 --- /dev/null +++ b/build/utils.sh @@ -0,0 +1,377 @@ +color_normal=$(tput sgr0) +color_bold=$(tput bold) +color_red="$color_bold$(tput setaf 1)" +color_green=$(tput setaf 2) +color_fawn=$(tput setaf 3); color_beige="$color_fawn" +color_yellow="$color_bold$color_fawn" +color_darkblue=$(tput setaf 4) +color_blue="$color_bold$color_darkblue" +color_purple=$(tput setaf 5); color_magenta="$color_purple" +color_pink="$color_bold$color_purple" +color_darkcyan=$(tput setaf 6) +color_cyan="$color_bold$color_darkcyan" +color_gray=$(tput setaf 7) +color_darkgray="$color_bold"$(tput setaf 0) +color_white="$color_bold$color_gray" + +message() +{ + echo $color_cyan -- $@$color_normal +} + +warn() +{ + echo $color_yellow -- $@$color_normal +} + +error() +{ + echo $color_red -- $@$color_normal +} + +detect_distro() +{ + if [ -f /etc/os-release ]; then + . /etc/os-release + export OS=$NAME + export OS_VERSION=$VERSION_ID + elif type lsb_release >/dev/null 2>&1; then + # linuxbase.org + export OS=$(lsb_release -si) + export OS_VERSION=$(lsb_release -sr) + elif [ -f /etc/lsb-release ]; then + # For some versions of Debian/Ubuntu without lsb_release command + . /etc/lsb-release + export OS=$DISTRIB_ID + OS_VERSION=$DISTRIB_RELEASE + elif [ -f /etc/debian_version ]; then + # Older Debian/Ubuntu/etc. + OS=Debian + OS_VERSION=$(cat /etc/debian_version) + else + # Fall back to uname, e.g. "Linux ", also works for BSD, etc. + OS=$(uname -s) + OS_VERSION=$(uname -r) + fi + OS=$(echo $OS | cut -f 1 -d " ") + message "Detected $color_yellow$OS $OS_VERSION$color_normal" +} + +menuStr="" + +function hideCursor(){ + printf "\033[?25l" + + # capture CTRL+C so cursor can be reset + trap "showCursor && exit 0" 2 +} + +function showCursor(){ + printf "\033[?25h" +} + +function clearLastMenu(){ + local msgLineCount=$(printf "$menuStr" | wc -l) + # moves the curser up N lines so the output overwrites it + echo -en "\033[${msgLineCount}A" + + # clear to end of screen to ensure there's no text left behind from previous input + [ $1 ] && tput ed +} + +function renderMenu(){ + local start=0 + local selector="" + local instruction="$1" + local selectedIndex=$2 + local listLength=$itemsLength + local longest=0 + local spaces="" + menuStr="\n $instruction\n" + + # Get the longest item from the list so that we know how many spaces to add + # to ensure there's no overlap from longer items when a list is scrolling up or down. + for (( i=0; i<$itemsLength; i++ )); do + if (( ${#menuItems[i]} > longest )); then + longest=${#menuItems[i]} + fi + done + spaces=$(printf ' %.0s' $(eval "echo {1.."$(($longest))"}")) + + if [ $3 -ne 0 ]; then + listLength=$3 + + if [ $selectedIndex -ge $listLength ]; then + start=$(($selectedIndex+1-$listLength)) + listLength=$(($selectedIndex+1)) + fi + fi + + for (( i=$start; i<$listLength; i++ )); do + local currItem="${menuItems[i]}" + currItemLength=${#currItem} + + if [[ $i = $selectedIndex ]]; then + selectedChoice="${currItem}" + selector="${color_green}ᐅ${color_normal}" + currItem="${color_green}${currItem}${color_normal}" + else + selector=" " + fi + + currItem="${spaces:0:0}${currItem}${spaces:currItemLength}" + + menuStr="${menuStr}\n ${selector} ${currItem}" + done + + menuStr="${menuStr}\n" + + # whether or not to overwrite the previous menu output + [ $4 ] && clearLastMenu + + printf "${menuStr}" +} + +function getChoice(){ + local KEY__ARROW_UP=$(echo -e "\033[A") + local KEY__ARROW_DOWN=$(echo -e "\033[B") + local KEY__ENTER=$(echo -e "\n") + local captureInput=true + local displayHelp=false + local maxViewable=0 + local instruction="Select an item from the list:" + local selectedIndex=0 + + remainingArgs=() + while [[ $# -gt 0 ]]; do + key="$1" + + case $key in + -h|--help) + displayHelp=true + shift + ;; + -i|--index) + selectedIndex=$2 + shift 2 + ;; + -m|--max) + maxViewable=$2 + shift 2 + ;; + -o|--options) + menuItems=$2[@] + menuItems=("${!menuItems}") + shift 2 + ;; + -q|--query) + instruction="$2" + shift 2 + ;; + *) + remainingArgs+=("$1") + shift + ;; + esac + done + + # just display help + if $displayHelp; then + echo; + echo "Usage: getChoice [OPTION]..." + echo "Renders a keyboard navigable menu with a visual indicator of what's selected." + echo; + echo " -h, --help Displays this message" + echo " -i, --index The initially selected index for the options" + echo " -m, --max Limit how many options are displayed" + echo " -o, --options An Array of options for a User to choose from" + echo " -q, --query Question or statement presented to the User" + echo; + echo "Example:" + echo " foodOptions=(\"pizza\" \"burgers\" \"chinese\" \"sushi\" \"thai\" \"italian\" \"shit\")" + echo; + echo " getChoice -q \"What do you feel like eating?\" -o foodOptions -i \$((\${#foodOptions[@]}-1)) -m 4" + echo " printf \"\\n First choice is '\${selectedChoice}'\\n\"" + echo; + echo " getChoice -q \"Select another option in case the first isn't available\" -o foodOptions" + echo " printf \"\\n Second choice is '\${selectedChoice}'\\n\"" + echo; + + return 0 + fi + + set -- "${remainingArgs[@]}" + local itemsLength=${#menuItems[@]} + + # no menu items, at least 1 required + if [[ $itemsLength -lt 1 ]]; then + printf "\n [ERROR] No menu items provided\n" + exit 1 + fi + + renderMenu "$instruction" $selectedIndex $maxViewable + hideCursor + + while $captureInput; do + read -rsn3 key # `3` captures the escape (\033'), bracket ([), & type (A) characters. + + case "$key" in + "$KEY__ARROW_UP") + selectedIndex=$((selectedIndex-1)) + (( $selectedIndex < 0 )) && selectedIndex=$((itemsLength-1)) + + renderMenu "$instruction" $selectedIndex $maxViewable true + ;; + + "$KEY__ARROW_DOWN") + selectedIndex=$((selectedIndex+1)) + (( $selectedIndex == $itemsLength )) && selectedIndex=0 + + renderMenu "$instruction" $selectedIndex $maxViewable true + ;; + + "$KEY__ENTER") + clearLastMenu true + showCursor + captureInput=false + ;; + esac + done +} + + +function optparse.throw_error(){ + local message="$1" + error "OPTPARSE: ERROR: $message" + exit 1 +} + +# ----------------------------------------------------------------------------------------------------------------------------- +function optparse.define(){ + if [ $# -lt 3 ]; then + optparse.throw_error "optparse.define [] [] []" + fi + for option_id in $( seq 1 $# ) ; do + local option="$( eval "echo \$$option_id")" + local key="$( echo $option | awk -F "=" '{print $1}' )"; + local value="$( echo $option | awk -F "=" '{print $2}' )"; + + #essentials: shortname, longname, description + if [ "$key" = "short" ]; then + local shortname="$value" + if [ ${#shortname} -ne 1 ]; then + optparse.throw_error "short name expected to be one character long" + fi + local short="-${shortname}" + elif [ "$key" = "long" ]; then + local longname="$value" + if [ ${#longname} -lt 2 ]; then + optparse.throw_error "long name expected to be atleast one character long" + fi + local long="--${longname}" + elif [ "$key" = "desc" ]; then + local desc="$value" + elif [ "$key" = "default" ]; then + local default="$value" + elif [ "$key" = "variable" ]; then + local variable="$value" + elif [ "$key" = "value" ]; then + local val="$value" + fi + done + + if [ "$variable" = "" ]; then + optparse.throw_error "You must give a variable for option: ($short/$long)" + fi + + if [ "$val" = "" ]; then + val="\$OPTARG" + fi + + # build OPTIONS and help + optparse_usage="${optparse_usage}#NL#TB${short} $(printf "%-25s %s" "${long}:" "${desc}")" + if [ "$default" != "" ]; then + optparse_usage="${optparse_usage} [default:$default]" + fi + optparse_contractions="${optparse_contractions}#NL#TB#TB${long})#NL#TB#TB#TBparams=\"\$params ${short}\";;" + if [ "$default" != "" ]; then + optparse_defaults="${optparse_defaults}#NL${variable}=${default}" + fi + optparse_arguments_string="${optparse_arguments_string}${shortname}" + if [ "$val" = "\$OPTARG" ]; then + optparse_arguments_string="${optparse_arguments_string}:" + fi + optparse_process="${optparse_process}#NL#TB#TB${shortname})#NL#TB#TB#TB${variable}=\"$val\";;" +} + +# ----------------------------------------------------------------------------------------------------------------------------- +function optparse.build(){ + local build_file="$(mktemp -t "optparse-XXXXXX.tmp")" + + # Building getopts header here + + # Function usage + cat << EOF > $build_file +function usage(){ +cat << XXX +usage: \$0 [OPTIONS] +OPTIONS: + $optparse_usage + -? --help : usage +XXX +} +# Contract long options into short options +params="" +while [ \$# -ne 0 ]; do + param="\$1" + shift + case "\$param" in + $optparse_contractions + "-?"|--help) + usage + exit 0;; + *) + if [[ "\$param" == --* ]]; then + echo -e "Unrecognized long option: \$param" + usage + exit 1 + fi + params="\$params \"\$param\"";; + esac +done +eval set -- "\$params" +# Set default variable values +$optparse_defaults +# Process using getopts +while getopts "$optparse_arguments_string" option; do + case \$option in + # Substitute actions for different variables + $optparse_process + :) + echo "Option - \$OPTARG requires an argument" + exit 1;; + *) + usage + exit 1;; + esac +done +# Clean up after self +rm $build_file +EOF + + local -A o=( ['#NL']='\n' ['#TB']='\t' ) + + for i in "${!o[@]}"; do + sed -i "s/${i}/${o[$i]}/g" $build_file + done + + # Unset global variables + unset optparse_usage + unset optparse_process + unset optparse_arguments_string + unset optparse_defaults + unset optparse_contractions + + # Return file name to parent + echo "$build_file" +} \ No newline at end of file diff --git a/cmake/FindGTest.cmake b/cmake/FindGTest.cmake deleted file mode 100644 index 5860ca415..000000000 --- a/cmake/FindGTest.cmake +++ /dev/null @@ -1,37 +0,0 @@ -find_path(GTEST_ROOT_DIR - NAMES include/gtest/gtest.h -) - -find_library(GTEST_LIBRARY - NAMES gtest - HINTS ${GTEST_ROOT_DIR}/lib -) - -find_library(GTESTMAIN_LIBRARY - NAMES gtest_main - HINTS ${GTEST_ROOT_DIR}/lib -) - -find_library(PTHREAD_LIBRARY - NAMES pthread - HINTS ${GTEST_ROOT_DIR}/lib -) - -find_path(GTEST_INCLUDE_DIR - NAMES gtest.h - HINTS ${GTEST_ROOT_DIR}/include/gtest -) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(GTest DEFAULT_MSG - GTEST_LIBRARY - GTESTMAIN_LIBRARY - PTHREAD_LIBRARY - GTEST_INCLUDE_DIR -) - -mark_as_advanced( - GTEST_ROOT_DIR - GTEST_LIBRARIES - GTEST_INCLUDE_DIR -) diff --git a/cmake/boost.CMakeLists.txt.in b/cmake/boost.CMakeLists.txt.in index 5df75c405..899c31152 100644 --- a/cmake/boost.CMakeLists.txt.in +++ b/cmake/boost.CMakeLists.txt.in @@ -1,33 +1,32 @@ -cmake_minimum_required(VERSION @CMAKE_VERSION@) - -include(ExternalProject) - -if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") - set(_toolset "gcc") -elseif(CMAKE_CXX_COMPILER_ID MATCHES ".*Clang") - set(_toolset "clang") -elseif(CMAKE_CXX_COMPILER_ID MATCHES "Intel") - set(_toolset "intel-linux") -endif() - -set(_b2args link=shared;threading=multi;variant=release;toolset=${_toolset};--with-system;--with-filesystem;--with-thread;--with-regex;--with-date_time) - -ExternalProject_Add(boost - PREFIX build - URL https://sourceforge.net/projects/boost/files/boost/1.55.0/boost_1_55_0.zip - URL_HASH SHA256=ae85620e810b87a03e1acf8bbf0d4ad87c0cf7040cf6a4e1d8958488ebe42e7e - DOWNLOAD_NO_PROGRESS TRUE - UPDATE_COMMAND "" - CONFIGURE_COMMAND /bootstrap.sh - --with-toolset=${_toolset} - --prefix=${CMAKE_CURRENT_SOURCE_DIR}/../boost - --with-libraries=system,filesystem,thread,regex,date_time - BUILD_COMMAND /b2 -q ${_b2args} - LOG_BUILD TRUE - BUILD_IN_SOURCE TRUE - INSTALL_COMMAND /b2 -q install ${_b2args} - LOG_INSTALL TRUE -) - -unset(_b2args) -unset(_toolset) \ No newline at end of file +cmake_minimum_required(VERSION @CMAKE_VERSION@) +project (BoostExternal) +include(ExternalProject) + +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") + set(_toolset "gcc") +elseif(CMAKE_CXX_COMPILER_ID MATCHES ".*Clang") + set(_toolset "clang") +elseif(CMAKE_CXX_COMPILER_ID MATCHES "Intel") + set(_toolset "intel-linux") +endif() + +set(INSTALL_LOCATION ${CMAKE_BINARY_DIR}/boost-lib) + +set(_cxxargs "-fPIC -DBOOST_NO_AUTO_PTR -fvisibility=default") +set(_b2args cxxflags=${_cxxargs};cflags=-fPIC;threading=multi; toolset=${_toolset} --without-python;--prefix=${INSTALL_LOCATION}) + +ExternalProject_Add(external_boost + PREFIX boost + URL https://boostorg.jfrog.io/artifactory/main/release/1.79.0/source/boost_1_79_0.tar.bz2 + URL_HASH SHA256=475d589d51a7f8b3ba2ba4eda022b170e562ca3b760ee922c146b6c65856ef39 + CONFIGURE_COMMAND ./bootstrap.sh + UPDATE_COMMAND "" + BUILD_COMMAND ./b2 -q ${_b2args} + LOG_BUILD TRUE + BUILD_IN_SOURCE TRUE + INSTALL_COMMAND ./b2 -q install ${_b2args} + LOG_INSTALL TRUE +) + +unset(_b2args) +unset(_toolset) diff --git a/cmake/boost.cmake b/cmake/boost.cmake index ae129287d..25ca5c3fb 100644 --- a/cmake/boost.cmake +++ b/cmake/boost.cmake @@ -3,9 +3,11 @@ configure_file(${CMAKE_CURRENT_LIST_DIR}/boost.CMakeLists.txt.in ${CMAKE_CURRENT_BINARY_DIR}/.boost/CMakeLists.txt @ONLY) +SET(BOOST_ROOT ${CMAKE_CURRENT_BINARY_DIR}/.boost/boost-lib) +message ("-- External: Configuring Boost") execute_process( - COMMAND ${CMAKE_COMMAND} . + COMMAND ${CMAKE_COMMAND} . -G "${CMAKE_GENERATOR}" -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} @@ -17,6 +19,7 @@ if(${_exec_ret}) message(FATAL_ERROR "Error ${_exec_ret} configuring boost dependency.") endif() +message ("-- External: Building Boost") execute_process( COMMAND ${CMAKE_COMMAND} --build . WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/.boost @@ -29,6 +32,13 @@ endif() unset(_exec_ret) -set(BOOST_ROOT ${CMAKE_CURRENT_BINARY_DIR}/boost) -find_package(Boost 1.55.0 REQUIRED COMPONENTS system filesystem thread regex date_time) +message("Boost installed localy at ${BOOST_ROOT}") + + +SET(Boost_USE_STATIC_LIBS ON) +FIND_PACKAGE(Boost 1.78.0 COMPONENTS system filesystem thread regex date_time chrono atomic) +IF (NOT Boost_FOUND) + MESSAGE_ONCE(CS_NO_BOOST "Required Boost libraries not found!") + return() +ENDIF() diff --git a/datatypes/mcs_datatype.cpp b/datatypes/mcs_datatype.cpp index c33269dbb..2ca95ba2c 100644 --- a/datatypes/mcs_datatype.cpp +++ b/datatypes/mcs_datatype.cpp @@ -592,7 +592,7 @@ string TypeHandlerDatetime::format(const SimpleValue& v, const SystemCatalog::Ty string TypeHandlerTimestamp::format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const { - return DataConvert::timestampToString(v.toSInt64(), v.tzname()); + return DataConvert::timestampToString(v.toSInt64(), v.timeZone()); } string TypeHandlerTime::format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const @@ -893,8 +893,8 @@ class SimpleConverter : public boost::any public: SimpleConverter(const SessionParam& sp, const TypeHandler* h, const SystemCatalog::TypeAttributesStd& attr, const char* str) - : boost::any( - h->convertFromString(attr, ConvertFromStringParam(sp.tzname(), true, false), str, initPushWarning())) + : boost::any(h->convertFromString(attr, ConvertFromStringParam(sp.timeZone(), true, false), str, + initPushWarning())) { } round_style_t roundStyle() const @@ -1059,7 +1059,7 @@ SimpleValue TypeHandlerTimestamp::toSimpleValue(const SessionParam& sp, { idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); SimpleConverter anyVal(sp, this, attr, str); - return SimpleValueTimestamp(anyVal.to_uint64(), sp.tzname()); + return SimpleValueTimestamp(anyVal.to_uint64(), sp.timeZone()); } SimpleValue TypeHandlerTime::toSimpleValue(const SessionParam& sp, @@ -1915,4 +1915,3 @@ const uint8_t* TypeHandlerUDecimal128::getEmptyValueForType( } // end of namespace datatypes -// vim:ts=2 sw=2: diff --git a/datatypes/mcs_datatype.h b/datatypes/mcs_datatype.h index 38a113db4..8e2c551cf 100644 --- a/datatypes/mcs_datatype.h +++ b/datatypes/mcs_datatype.h @@ -482,7 +482,11 @@ inline bool isUnsigned(const datatypes::SystemCatalog::ColDataType type) case datatypes::SystemCatalog::USMALLINT: case datatypes::SystemCatalog::UMEDINT: case datatypes::SystemCatalog::UINT: - case datatypes::SystemCatalog::UBIGINT: return true; + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::CHAR: + case datatypes::SystemCatalog::VARCHAR: + case datatypes::SystemCatalog::TEXT: + case datatypes::SystemCatalog::VARBINARY: return true; default: return false; } @@ -561,30 +565,30 @@ enum class round_style_t : uint8_t class SessionParam { - const char* m_tzname; + long m_timeZone; public: - SessionParam(const char* tzname) : m_tzname(tzname) + SessionParam(long timeZone) : m_timeZone(timeZone) { } - const char* tzname() const + long timeZone() const { - return m_tzname; + return m_timeZone; } }; class ConvertFromStringParam { - const std::string& m_timeZone; + const long m_timeZone; const bool m_noRoundup; const bool m_isUpdate; public: - ConvertFromStringParam(const std::string& timeZone, bool noRoundup, bool isUpdate) + ConvertFromStringParam(long timeZone, bool noRoundup, bool isUpdate) : m_timeZone(timeZone), m_noRoundup(noRoundup), m_isUpdate(isUpdate) { } - const std::string& timeZone() const + long timeZone() const { return m_timeZone; } @@ -602,14 +606,14 @@ class SimpleValue { int64_t m_sint64; int128_t m_sint128; - const char* m_tzname; + long m_timeZone; public: - SimpleValue(const int64_t sint64, const int128_t& sint128, const char* tzname) - : m_sint64(sint64), m_sint128(sint128), m_tzname(tzname) + SimpleValue(const int64_t sint64, const int128_t& sint128, long timeZone) + : m_sint64(sint64), m_sint128(sint128), m_timeZone(timeZone) { } - SimpleValue() : m_sint64(0), m_sint128(0), m_tzname(0) + SimpleValue() : m_sint64(0), m_sint128(0), m_timeZone(0) { } int64_t toSInt64() const @@ -624,16 +628,16 @@ class SimpleValue { return m_sint128; } - const char* tzname() const + long timeZone() const { - return m_tzname; + return m_timeZone; } }; class SimpleValueSInt64 : public SimpleValue { public: - SimpleValueSInt64(int64_t value) : SimpleValue(value, 0, NULL) + SimpleValueSInt64(int64_t value) : SimpleValue(value, 0, 0) { } }; @@ -641,7 +645,7 @@ class SimpleValueSInt64 : public SimpleValue class SimpleValueUInt64 : public SimpleValue { public: - SimpleValueUInt64(uint64_t value) : SimpleValue(static_cast(value), 0, NULL) + SimpleValueUInt64(uint64_t value) : SimpleValue(static_cast(value), 0, 0) { } }; @@ -649,7 +653,7 @@ class SimpleValueUInt64 : public SimpleValue class SimpleValueSInt128 : public SimpleValue { public: - SimpleValueSInt128(int128_t value) : SimpleValue(0, value, NULL) + SimpleValueSInt128(int128_t value) : SimpleValue(0, value, 0) { } }; @@ -657,8 +661,7 @@ class SimpleValueSInt128 : public SimpleValue class SimpleValueTimestamp : public SimpleValue { public: - SimpleValueTimestamp(uint64_t value, const char* tzname) - : SimpleValue(static_cast(value), 0, tzname) + SimpleValueTimestamp(uint64_t value, long timeZone) : SimpleValue(static_cast(value), 0, timeZone) { } }; @@ -2521,5 +2524,3 @@ class TypeHandlerTimestamp : public TypeHandlerTemporal }; } // end of namespace datatypes - -// vim:ts=2 sw=2: diff --git a/datatypes/mcs_datatype_basic.h b/datatypes/mcs_datatype_basic.h index 1056c34e9..cd3327997 100644 --- a/datatypes/mcs_datatype_basic.h +++ b/datatypes/mcs_datatype_basic.h @@ -78,4 +78,3 @@ uint64_t xFloatToMCSUInt64Round(SRC value) } // end of namespace datatypes -// vim:ts=2 sw=2: diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h index 621e0aed4..91bdbe55f 100644 --- a/datatypes/mcs_decimal.h +++ b/datatypes/mcs_decimal.h @@ -318,12 +318,12 @@ class TDecimal128 : public TSInt128 static constexpr int128_t minInt128 = TFloat128::minInt128; static constexpr int128_t maxInt128 = TFloat128::maxInt128; - static inline bool isWideDecimalNullValue(const int128_t& val) + static inline bool isWideDecimalNullValue(const int128_t val) { return (val == TSInt128::NullValue); } - static inline bool isWideDecimalEmptyValue(const int128_t& val) + static inline bool isWideDecimalEmptyValue(const int128_t val) { return (val == TSInt128::EmptyValue); } @@ -342,10 +342,10 @@ class TDecimal128 : public TSInt128 TDecimal128() { } - explicit TDecimal128(const int128_t& val) : TSInt128(val) + explicit TDecimal128(const int128_t val) : TSInt128(val) { } - explicit TDecimal128(const TSInt128& val) : TSInt128(val) + explicit TDecimal128(const TSInt128 val) : TSInt128(val) { } explicit TDecimal128(const int128_t* valPtr) : TSInt128(valPtr) diff --git a/datatypes/mcs_double.h b/datatypes/mcs_double.h index b87bf4242..ca05ea757 100644 --- a/datatypes/mcs_double.h +++ b/datatypes/mcs_double.h @@ -54,4 +54,3 @@ class TDouble } // end of namespace datatypes -// vim:ts=2 sw=2: diff --git a/datatypes/mcs_float128.h b/datatypes/mcs_float128.h index 0ea0c521e..2a97dadad 100644 --- a/datatypes/mcs_float128.h +++ b/datatypes/mcs_float128.h @@ -724,4 +724,3 @@ class TFloat128 } // namespace datatypes -// vim:ts=2 sw=2: diff --git a/datatypes/mcs_int128.cpp b/datatypes/mcs_int128.cpp index 13329fc41..58dee474c 100644 --- a/datatypes/mcs_int128.cpp +++ b/datatypes/mcs_int128.cpp @@ -113,4 +113,3 @@ std::ostream& operator<<(std::ostream& os, const TSInt128& x) } } // end of namespace datatypes -// vim:ts=2 sw=2: diff --git a/datatypes/mcs_int128.h b/datatypes/mcs_int128.h index f7949fcda..1dd5d1e35 100644 --- a/datatypes/mcs_int128.h +++ b/datatypes/mcs_int128.h @@ -322,4 +322,3 @@ class TSInt128 } // end of namespace datatypes -// vim:ts=2 sw=2: diff --git a/datatypes/mcs_int64.h b/datatypes/mcs_int64.h index 3737cff95..76ec1e399 100644 --- a/datatypes/mcs_int64.h +++ b/datatypes/mcs_int64.h @@ -179,4 +179,3 @@ class TSInt64Null : public TSInt64, public TNullFlag } // end of namespace datatypes -// vim:ts=2 sw=2: diff --git a/datatypes/mcs_longdouble.h b/datatypes/mcs_longdouble.h index 07995d537..f0b68d835 100644 --- a/datatypes/mcs_longdouble.h +++ b/datatypes/mcs_longdouble.h @@ -54,4 +54,3 @@ class TLongDouble } // end of namespace datatypes -// vim:ts=2 sw=2: diff --git a/datatypes/numericliteral.h b/datatypes/numericliteral.h index 0640f2a91..cdddad95d 100644 --- a/datatypes/numericliteral.h +++ b/datatypes/numericliteral.h @@ -336,6 +336,7 @@ class SignedInteger : public Parser::DD2OM { public: using DD2OM::DD2OM; + bool isNull() const { return UnsignedInteger::isNull(); diff --git a/dbcon/ddlpackage/CMakeLists.txt b/dbcon/ddlpackage/CMakeLists.txt index d2e0b4d74..5557ed3ff 100644 --- a/dbcon/ddlpackage/CMakeLists.txt +++ b/dbcon/ddlpackage/CMakeLists.txt @@ -9,7 +9,7 @@ FIND_PACKAGE(FLEX REQUIRED) FLEX_TARGET(ddl_scan ddl.l ${CMAKE_CURRENT_BINARY_DIR}/ddl-scan.cpp COMPILE_FLAGS "-i -L -Pddl") ADD_FLEX_BISON_DEPENDENCY(ddl_scan ddl_gram) -set_source_files_properties(ddl-scan.cpp PROPERTIES COMPILE_FLAGS "-Wno-sign-compare -DYY_NO_INPUT") +set_source_files_properties(ddl-scan.cpp PROPERTIES COMPILE_FLAGS "-Wno-register -Wno-deprecated-register -Wno-sign-compare -DYY_NO_INPUT") ########### next target ############### diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index b340fbeaa..c8d36b379 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -95,7 +95,7 @@ void fix_column_length(SchemaObject* elem, const CHARSET_INFO* def_cs) { %} %expect 17 -%pure-parser +%define api.pure %lex-param {void * scanner} %parse-param {struct ddlpackage::pass_to_bison * x} diff --git a/dbcon/ddlpackage/ddlpkg.h b/dbcon/ddlpackage/ddlpkg.h index d780c2a6d..7ad8f273c 100644 --- a/dbcon/ddlpackage/ddlpkg.h +++ b/dbcon/ddlpackage/ddlpkg.h @@ -1308,9 +1308,20 @@ struct AlterTableStatement : public SqlStatement return fTableName->fSchema; } + long getTimeZone() const + { + return fTimeZone; + } + void setTimeZone(long timeZone) + { + fTimeZone = timeZone; + } + QualifiedName* fTableName; AlterTableActionList fActions; - std::string fTimeZone; + private: + long fTimeZone; + public: }; /** @brief This is used during parsing when constraint attributes diff --git a/dbcon/ddlpackage/serialize.cpp b/dbcon/ddlpackage/serialize.cpp index c719e8f96..796b57cd0 100644 --- a/dbcon/ddlpackage/serialize.cpp +++ b/dbcon/ddlpackage/serialize.cpp @@ -118,7 +118,9 @@ int AlterTableStatement::unserialize(ByteStream& bytestream) // read table name fTableName->unserialize(bytestream); - bytestream >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + bytestream >> timeZone; + fTimeZone = timeZone; // read alter action list quadbyte action_count; @@ -225,7 +227,8 @@ int AlterTableStatement::serialize(ByteStream& bytestream) // write table name fTableName->serialize(bytestream); - bytestream << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + bytestream << timeZone; write_vec(fActions, bytestream); diff --git a/dbcon/ddlpackageproc/altertableprocessor.cpp b/dbcon/ddlpackageproc/altertableprocessor.cpp index 0e321e063..86457fb57 100644 --- a/dbcon/ddlpackageproc/altertableprocessor.cpp +++ b/dbcon/ddlpackageproc/altertableprocessor.cpp @@ -22,12 +22,12 @@ #include #include +#include #include #include using namespace std; #include -#include #include #include "altertableprocessor.h" @@ -1023,7 +1023,8 @@ void AlterTableProcessor::addColumn(uint32_t sessionID, execplan::CalpontSystemC bs << (ByteStream::byte)column_iterator->colType.colDataType; bs << (uint32_t)column_iterator->colType.colWidth; bs << (ByteStream::byte)column_iterator->colType.compressionType; - bs << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + bs << timeZone; // cout << "sending command fillcolumn " << endl; uint32_t msgRecived = 0; fWEClient->write_to_all(bs); @@ -2082,14 +2083,14 @@ void AlterTableProcessor::tableComment(uint32_t sessionID, execplan::CalpontSyst CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); boost::algorithm::to_upper(ataTableComment.fTableComment); - boost::regex compat("[[:space:]]*AUTOINCREMENT[[:space:]]*=[[:space:]]*", boost::regex_constants::extended); - boost::match_results what; + std::regex compat("[[:space:]]*AUTOINCREMENT[[:space:]]*=[[:space:]]*", std::regex_constants::extended); + std::match_results what; std::string::const_iterator start, end; start = ataTableComment.fTableComment.begin(); end = ataTableComment.fTableComment.end(); - boost::match_flag_type flags = boost::match_default; + std::regex_constants::match_flag_type flags = std::regex_constants::match_default; - if (boost::regex_search(start, end, what, compat, flags) && what[0].matched) + if (std::regex_search(start, end, what, compat, flags) && what[0].matched) { std::string params(&(*(what[0].second))); char* ep = NULL; @@ -2538,7 +2539,6 @@ void AlterTableProcessor::renameColumn(uint32_t sessionID, execplan::CalpontSyst } } // namespace ddlpackageprocessor -// vim:ts=4 sw=4: #ifdef __clang__ #pragma clang diagnostic pop diff --git a/dbcon/ddlpackageproc/altertableprocessor.h b/dbcon/ddlpackageproc/altertableprocessor.h index 9ee0bd50f..7a7f708ad 100644 --- a/dbcon/ddlpackageproc/altertableprocessor.h +++ b/dbcon/ddlpackageproc/altertableprocessor.h @@ -148,7 +148,7 @@ class AlterTableProcessor : public DDLPackageProcessor ddlpackage::AtaTableComment& ataTableComment, ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId); - std::string fTimeZone; + long fTimeZone; protected: void rollBackAlter(const std::string& error, BRM::TxnID txnID, int sessionId, DDLResult& result, diff --git a/dbcon/ddlpackageproc/createtableprocessor.cpp b/dbcon/ddlpackageproc/createtableprocessor.cpp index f51d03958..f0ff32be7 100644 --- a/dbcon/ddlpackageproc/createtableprocessor.cpp +++ b/dbcon/ddlpackageproc/createtableprocessor.cpp @@ -830,4 +830,3 @@ void CreateTableProcessor::rollBackCreateTable(const string& error, BRM::TxnID t } } // namespace ddlpackageprocessor -// vim:ts=4 sw=4: diff --git a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp index 3740035d6..bf27af32f 100644 --- a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp +++ b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp @@ -1491,4 +1491,3 @@ int DDLPackageProcessor::commitTransaction(uint64_t uniqueId, BRM::TxnID txnID) } } // namespace ddlpackageprocessor -// vim:ts=4 sw=4: diff --git a/dbcon/ddlpackageproc/ddlpackageprocessor.h b/dbcon/ddlpackageproc/ddlpackageprocessor.h index 9e25ab1d2..047b99d95 100644 --- a/dbcon/ddlpackageproc/ddlpackageprocessor.h +++ b/dbcon/ddlpackageproc/ddlpackageprocessor.h @@ -890,4 +890,3 @@ bool from_string(T& t, const std::string& s, std::ios_base& (*f)(std::ios_base&) #undef EXPORT -// vim:ts=4 sw=4: diff --git a/dbcon/ddlpackageproc/droptableprocessor.cpp b/dbcon/ddlpackageproc/droptableprocessor.cpp index e1ca785d9..4cfc79467 100644 --- a/dbcon/ddlpackageproc/droptableprocessor.cpp +++ b/dbcon/ddlpackageproc/droptableprocessor.cpp @@ -1367,4 +1367,3 @@ TruncTableProcessor::DDLResult TruncTableProcessor::processPackage( } // namespace ddlpackageprocessor -// vim:ts=4 sw=4: diff --git a/dbcon/dmlpackage/CMakeLists.txt b/dbcon/dmlpackage/CMakeLists.txt index 63825be10..e999cdae1 100644 --- a/dbcon/dmlpackage/CMakeLists.txt +++ b/dbcon/dmlpackage/CMakeLists.txt @@ -10,7 +10,7 @@ FIND_PACKAGE(FLEX REQUIRED) FLEX_TARGET(dml_scan dml.l ${CMAKE_CURRENT_BINARY_DIR}/dml-scan.cpp COMPILE_FLAGS "-i -L -Pdml") ADD_FLEX_BISON_DEPENDENCY(dml_scan dml_gram) -set_source_files_properties(dml-scan.cpp PROPERTIES COMPILE_FLAGS "-Wno-sign-compare -DYY_NO_INPUT") +set_source_files_properties(dml-scan.cpp PROPERTIES COMPILE_FLAGS "-Wno-register -Wno-deprecated-register -Wno-sign-compare -DYY_NO_INPUT") ########### next target ############### diff --git a/dbcon/dmlpackage/calpontdmlpackage.h b/dbcon/dmlpackage/calpontdmlpackage.h index 650dd75a1..651c151a7 100644 --- a/dbcon/dmlpackage/calpontdmlpackage.h +++ b/dbcon/dmlpackage/calpontdmlpackage.h @@ -222,14 +222,14 @@ class CalpontDMLPackage * * @param the timezone to set */ - void set_TimeZone(const std::string& timeZone) + void set_TimeZone(const long timeZone) { fTimeZone = timeZone; } /** @brief get the timezone */ - const std::string get_TimeZone() const + long get_TimeZone() const { return fTimeZone; } @@ -367,7 +367,7 @@ class CalpontDMLPackage void initializeTable(); std::string fSchemaName; - std::string fTimeZone; + long fTimeZone; std::string fTableName; std::string fDMLStatement; std::string fSQLStatement; diff --git a/dbcon/dmlpackage/commanddmlpackage.cpp b/dbcon/dmlpackage/commanddmlpackage.cpp index cf9847396..16c435008 100644 --- a/dbcon/dmlpackage/commanddmlpackage.cpp +++ b/dbcon/dmlpackage/commanddmlpackage.cpp @@ -60,7 +60,8 @@ int CommandDMLPackage::write(messageqcpp::ByteStream& bytestream) bytestream << fSQLStatement; // for cleartablelock, this is table lockID bytestream << (uint8_t)fLogging; bytestream << fSchemaName; - bytestream << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + bytestream << timeZone; bytestream << fTableName; bytestream << fTableOid; bytestream << static_cast(fIsAutocommitOn); @@ -83,7 +84,9 @@ int CommandDMLPackage::read(messageqcpp::ByteStream& bytestream) bytestream >> logging; fLogging = (logging != 0); bytestream >> fSchemaName; - bytestream >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + bytestream >> timeZone; + fTimeZone = timeZone; bytestream >> fTableName; bytestream >> fTableOid; bytestream >> reinterpret_cast(fIsAutocommitOn); diff --git a/dbcon/dmlpackage/deletedmlpackage.cpp b/dbcon/dmlpackage/deletedmlpackage.cpp index 2ab6c6f2a..cdbce4a94 100644 --- a/dbcon/dmlpackage/deletedmlpackage.cpp +++ b/dbcon/dmlpackage/deletedmlpackage.cpp @@ -69,7 +69,8 @@ int DeleteDMLPackage::write(messageqcpp::ByteStream& bytestream) bytestream << fDMLStatement; bytestream << fSQLStatement; bytestream << fSchemaName; - bytestream << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + bytestream << timeZone; if (fTable != 0) { @@ -105,7 +106,9 @@ int DeleteDMLPackage::read(messageqcpp::ByteStream& bytestream) bytestream >> fDMLStatement; bytestream >> fSQLStatement; bytestream >> fSchemaName; - bytestream >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + bytestream >> timeZone; + fTimeZone = timeZone; fTable = new DMLTable(); retval = fTable->read(bytestream); diff --git a/dbcon/dmlpackage/dml.y b/dbcon/dmlpackage/dml.y index 73dcf26ea..3bc62ecb8 100644 --- a/dbcon/dmlpackage/dml.y +++ b/dbcon/dmlpackage/dml.y @@ -90,7 +90,7 @@ char* copy_string(const char *str); } %} -%pure-parser +%define api.pure %lex-param {void * scanner} %parse-param {void * scanner} %debug diff --git a/dbcon/dmlpackage/insertdmlpackage.cpp b/dbcon/dmlpackage/insertdmlpackage.cpp index 6d4693d4d..ef8024590 100644 --- a/dbcon/dmlpackage/insertdmlpackage.cpp +++ b/dbcon/dmlpackage/insertdmlpackage.cpp @@ -64,7 +64,8 @@ int InsertDMLPackage::write(messageqcpp::ByteStream& bytestream) bytestream << fDMLStatement; bytestream << fDMLStatement; bytestream << fSchemaName; - bytestream << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + bytestream << timeZone; bytestream << (uint8_t)fLogging; bytestream << (uint8_t)fLogending; @@ -95,7 +96,9 @@ int InsertDMLPackage::read(messageqcpp::ByteStream& bytestream) bytestream >> fDMLStatement; bytestream >> fSQLStatement; bytestream >> fSchemaName; - bytestream >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + bytestream >> timeZone; + fTimeZone = timeZone; uint8_t logging; bytestream >> logging; fLogging = (logging != 0); @@ -125,7 +128,9 @@ void InsertDMLPackage::readMetaData(messageqcpp::ByteStream& bytestream) bytestream >> fDMLStatement; bytestream >> fSQLStatement; bytestream >> fSchemaName; - bytestream >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + bytestream >> timeZone; + fTimeZone = timeZone; uint8_t logging; bytestream >> logging; fLogging = (logging != 0); diff --git a/dbcon/dmlpackage/updatedmlpackage.cpp b/dbcon/dmlpackage/updatedmlpackage.cpp index 2400e6b5a..d2096b689 100644 --- a/dbcon/dmlpackage/updatedmlpackage.cpp +++ b/dbcon/dmlpackage/updatedmlpackage.cpp @@ -68,7 +68,8 @@ int UpdateDMLPackage::write(messageqcpp::ByteStream& bytestream) bytestream << fDMLStatement; bytestream << fSQLStatement; bytestream << fSchemaName; - bytestream << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + bytestream << timeZone; bytestream << (uint8_t)fIsFromCol; if (fTable != 0) @@ -105,7 +106,9 @@ int UpdateDMLPackage::read(messageqcpp::ByteStream& bytestream) bytestream >> fDMLStatement; bytestream >> fSQLStatement; bytestream >> fSchemaName; - bytestream >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + bytestream >> timeZone; + fTimeZone = timeZone; uint8_t isFromCol; bytestream >> isFromCol; fIsFromCol = (isFromCol != 0); diff --git a/dbcon/dmlpackageproc/autoincrementdata.cpp b/dbcon/dmlpackageproc/autoincrementdata.cpp index 3a71efaf6..875e90d7e 100644 --- a/dbcon/dmlpackageproc/autoincrementdata.cpp +++ b/dbcon/dmlpackageproc/autoincrementdata.cpp @@ -97,4 +97,3 @@ AutoincrementData::OIDNextValue& AutoincrementData::getOidNextValueMap() return fOidNextValueMap; } -// vim:ts=4 sw=4: diff --git a/dbcon/dmlpackageproc/autoincrementdata.h b/dbcon/dmlpackageproc/autoincrementdata.h index 9237a2709..7dcd2a200 100644 --- a/dbcon/dmlpackageproc/autoincrementdata.h +++ b/dbcon/dmlpackageproc/autoincrementdata.h @@ -49,4 +49,3 @@ class AutoincrementData boost::mutex fOIDnextvalLock; }; -// vim:ts=4 sw=4: diff --git a/dbcon/dmlpackageproc/dmlpackageprocessor.cpp b/dbcon/dmlpackageproc/dmlpackageprocessor.cpp index b5217b0e6..78b19b392 100644 --- a/dbcon/dmlpackageproc/dmlpackageprocessor.cpp +++ b/dbcon/dmlpackageproc/dmlpackageprocessor.cpp @@ -913,4 +913,3 @@ int DMLPackageProcessor::endTransaction(uint64_t uniqueId, BRM::TxnID txnID, boo return rc; } } // namespace dmlpackageprocessor -// vim:ts=4 sw=4: diff --git a/dbcon/dmlpackageproc/insertpackageprocessor.cpp b/dbcon/dmlpackageproc/insertpackageprocessor.cpp index fb844b696..3fd6b7ae0 100644 --- a/dbcon/dmlpackageproc/insertpackageprocessor.cpp +++ b/dbcon/dmlpackageproc/insertpackageprocessor.cpp @@ -432,4 +432,3 @@ DMLPackageProcessor::DMLResult InsertPackageProcessor::processPackage(dmlpackage } // namespace dmlpackageprocessor -// vim:ts=4 sw=4: diff --git a/dbcon/dmlpackageproc/tablelockdata.cpp b/dbcon/dmlpackageproc/tablelockdata.cpp index d5f94ec60..1c1699172 100644 --- a/dbcon/dmlpackageproc/tablelockdata.cpp +++ b/dbcon/dmlpackageproc/tablelockdata.cpp @@ -101,4 +101,3 @@ TablelockData::OIDTablelock& TablelockData::getOidTablelockMap() } } // namespace dmlpackageprocessor -// vim:ts=4 sw=4: diff --git a/dbcon/dmlpackageproc/tablelockdata.h b/dbcon/dmlpackageproc/tablelockdata.h index 0cd7e6634..82dd94a55 100644 --- a/dbcon/dmlpackageproc/tablelockdata.h +++ b/dbcon/dmlpackageproc/tablelockdata.h @@ -61,4 +61,3 @@ class TablelockData #undef EXPORT -// vim:ts=4 sw=4: diff --git a/dbcon/execplan/aggregatecolumn.cpp b/dbcon/execplan/aggregatecolumn.cpp index c3919b893..49b01735e 100644 --- a/dbcon/execplan/aggregatecolumn.cpp +++ b/dbcon/execplan/aggregatecolumn.cpp @@ -183,7 +183,8 @@ void AggregateColumn::serialize(messageqcpp::ByteStream& b) const (*rcit)->serialize(b); b << fData; - b << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + b << timeZone; // b << fAlias; b << fTableAlias; b << static_cast(fAsc); @@ -236,7 +237,9 @@ void AggregateColumn::unserialize(messageqcpp::ByteStream& b) } b >> fData; - b >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + b >> timeZone; + fTimeZone = timeZone; // b >> fAlias; b >> fTableAlias; b >> reinterpret_cast(fAsc); @@ -421,6 +424,8 @@ void AggregateColumn::evaluate(Row& row, bool& isNull) else fResult.intVal = atoll((char*)&fResult.origIntVal); + fResult.uintVal = fResult.intVal; + break; case CalpontSystemCatalog::BIGINT: diff --git a/dbcon/execplan/aggregatecolumn.h b/dbcon/execplan/aggregatecolumn.h index 4ab0e1a3c..9b0a75c93 100644 --- a/dbcon/execplan/aggregatecolumn.h +++ b/dbcon/execplan/aggregatecolumn.h @@ -313,12 +313,12 @@ class AggregateColumn : public ReturnedColumn return false; } - inline const std::string timeZone() const + inline long timeZone() const { return fTimeZone; } - inline void timeZone(const std::string& timeZone) + inline void timeZone(const long timeZone) { fTimeZone = timeZone; } @@ -346,7 +346,7 @@ class AggregateColumn : public ReturnedColumn ColumnList fGroupByColList; ColumnList fProjectColList; SRCP fConstCol; - std::string fTimeZone; + long fTimeZone; public: /*********************************************************** diff --git a/dbcon/execplan/arithmeticcolumn.cpp b/dbcon/execplan/arithmeticcolumn.cpp index 70bf7be81..d7fb57027 100644 --- a/dbcon/execplan/arithmeticcolumn.cpp +++ b/dbcon/execplan/arithmeticcolumn.cpp @@ -283,7 +283,7 @@ const string ArithmeticColumn::nextToken(string::size_type& pos, char end) const msg.append(1, end); msg.append(" found in " + fData); throw invalid_argument(msg); - return 0; + return {}; } ostream& operator<<(ostream& output, const ArithmeticColumn& rhs) diff --git a/dbcon/execplan/arithmeticoperator.cpp b/dbcon/execplan/arithmeticoperator.cpp index f3e235f14..56280e891 100644 --- a/dbcon/execplan/arithmeticoperator.cpp +++ b/dbcon/execplan/arithmeticoperator.cpp @@ -72,7 +72,8 @@ ostream& operator<<(ostream& output, const ArithmeticOperator& rhs) void ArithmeticOperator::serialize(messageqcpp::ByteStream& b) const { b << (ObjectReader::id_t)ObjectReader::ARITHMETICOPERATOR; - b << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + b << timeZone; const messageqcpp::ByteStream::byte tmp = fDecimalOverflowCheck; b << tmp; Operator::serialize(b); @@ -81,7 +82,9 @@ void ArithmeticOperator::serialize(messageqcpp::ByteStream& b) const void ArithmeticOperator::unserialize(messageqcpp::ByteStream& b) { ObjectReader::checkType(b, ObjectReader::ARITHMETICOPERATOR); - b >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + b >> timeZone; + fTimeZone = timeZone; messageqcpp::ByteStream::byte tmp; b >> tmp; fDecimalOverflowCheck = tmp; diff --git a/dbcon/execplan/arithmeticoperator.h b/dbcon/execplan/arithmeticoperator.h index aa07fbc9b..8a433361b 100644 --- a/dbcon/execplan/arithmeticoperator.h +++ b/dbcon/execplan/arithmeticoperator.h @@ -59,11 +59,11 @@ class ArithmeticOperator : public Operator return new ArithmeticOperator(*this); } - inline const std::string& timeZone() const + inline long timeZone() const { return fTimeZone; } - inline void timeZone(const std::string& timeZone) + inline void timeZone(const long timeZone) { fTimeZone = timeZone; } @@ -207,7 +207,7 @@ class ArithmeticOperator : public Operator template inline result_t execute(result_t op1, result_t op2, bool& isNull); inline void execute(IDB_Decimal& result, IDB_Decimal op1, IDB_Decimal op2, bool& isNull); - std::string fTimeZone; + long fTimeZone; bool fDecimalOverflowCheck; }; diff --git a/dbcon/execplan/calpontselectexecutionplan.cpp b/dbcon/execplan/calpontselectexecutionplan.cpp index de4291c0e..c53cd86ab 100644 --- a/dbcon/execplan/calpontselectexecutionplan.cpp +++ b/dbcon/execplan/calpontselectexecutionplan.cpp @@ -500,7 +500,8 @@ void CalpontSelectExecutionPlan::serialize(messageqcpp::ByteStream& b) const b << fDJSPartitionSize; b << fUMMemLimit; b << (uint8_t)fIsDML; - b << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + b << timeZone; } void CalpontSelectExecutionPlan::unserialize(messageqcpp::ByteStream& b) @@ -695,7 +696,9 @@ void CalpontSelectExecutionPlan::unserialize(messageqcpp::ByteStream& b) b >> fUMMemLimit; b >> tmp8; fIsDML = tmp8; - b >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + b >> timeZone; + fTimeZone = timeZone; } bool CalpontSelectExecutionPlan::operator==(const CalpontSelectExecutionPlan& t) const diff --git a/dbcon/execplan/calpontselectexecutionplan.h b/dbcon/execplan/calpontselectexecutionplan.h index 91a6d627c..10093f8ce 100644 --- a/dbcon/execplan/calpontselectexecutionplan.h +++ b/dbcon/execplan/calpontselectexecutionplan.h @@ -706,11 +706,11 @@ class CalpontSelectExecutionPlan : public CalpontExecutionPlan return fIsDML; } - void timeZone(const std::string& timezone) + void timeZone(const long timezone) { fTimeZone = timezone; } - const std::string timeZone() const + long timeZone() const { return fTimeZone; } @@ -923,7 +923,7 @@ class CalpontSelectExecutionPlan : public CalpontExecutionPlan int64_t fUMMemLimit; bool fIsDML; - std::string fTimeZone; + long fTimeZone; std::vector fDynamicParseTreeVec; }; @@ -939,4 +939,3 @@ inline std::ostream& operator<<(std::ostream& os, const CalpontSelectExecutionPl } } // namespace execplan -// vim:ts=4 sw=4: diff --git a/dbcon/execplan/calpontsystemcatalog.cpp b/dbcon/execplan/calpontsystemcatalog.cpp index a2f5ea525..500f78119 100644 --- a/dbcon/execplan/calpontsystemcatalog.cpp +++ b/dbcon/execplan/calpontsystemcatalog.cpp @@ -6105,8 +6105,8 @@ const string CalpontSystemCatalog::ColType::toString() const } boost::any CalpontSystemCatalog::ColType::convertColumnData(const std::string& data, bool& pushWarning, - const std::string& timeZone, bool nulFlag, - bool noRoundup, bool isUpdate) const + long timeZone, bool nulFlag, bool noRoundup, + bool isUpdate) const { pushWarning = false; const datatypes::TypeHandler* h = typeHandler(); diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index 352758aaa..055d4846c 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -205,7 +205,6 @@ class CalpontSystemCatalog : public datatypes::SystemCatalog */ struct ColType : public datatypes::SystemCatalog::TypeHolderStd { - ColType(); ConstraintType constraintType; DictOID ddn; std::string defaultValue; @@ -216,11 +215,25 @@ class CalpontSystemCatalog : public datatypes::SystemCatalog uint64_t nextvalue; // next autoincrement value uint32_t charsetNumber; const CHARSET_INFO* cs; + private: + long timeZone; + public: + ColType(); ColType(const ColType& rhs); ColType& operator=(const ColType& rhs); CHARSET_INFO* getCharset(); + + long getTimeZone() const + { + return timeZone; + } + void setTimeZone(long timeZone_) + { + timeZone = timeZone_; + } + // for F&E use. only serialize necessary info for now void serialize(messageqcpp::ByteStream& b) const { @@ -254,7 +267,7 @@ class CalpontSystemCatalog : public datatypes::SystemCatalog * @param nRoundtrip * @param isUpdate */ - boost::any convertColumnData(const std::string& data, bool& bSaturate, const std::string& timeZone, + boost::any convertColumnData(const std::string& data, bool& bSaturate, long timeZone, bool nulFlag = false, bool noRoundup = false, bool isUpdate = false) const; const std::string toString() const; @@ -932,17 +945,18 @@ inline bool isNull(int64_t val, const execplan::CalpontSystemCatalog::ColType& c break; } + case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: { int colWidth = ct.colWidth; if (colWidth <= 8) { - if ((colWidth == 1) && ((int8_t)joblist::CHAR1NULL == val)) + if ((colWidth == 1) && ((uint8_t)joblist::CHAR1NULL == (uint8_t)val)) ret = true; - else if ((colWidth == 2) && ((int16_t)joblist::CHAR2NULL == val)) + else if ((colWidth == 2) && ((uint16_t)joblist::CHAR2NULL == (uint16_t)val)) ret = true; - else if ((colWidth < 5) && ((int32_t)joblist::CHAR4NULL == val)) + else if ((colWidth < 5) && ((uint32_t)joblist::CHAR4NULL == (uint32_t)val)) ret = true; else if ((int64_t)joblist::CHAR8NULL == val) ret = true; @@ -951,7 +965,6 @@ inline bool isNull(int64_t val, const execplan::CalpontSystemCatalog::ColType& c { throw std::logic_error("Not a int column."); } - break; } @@ -1062,27 +1075,6 @@ inline bool isNull(int64_t val, const execplan::CalpontSystemCatalog::ColType& c break; } - case execplan::CalpontSystemCatalog::VARCHAR: - { - int colWidth = ct.colWidth; - - if (colWidth <= 8) - { - if ((colWidth < 3) && ((int16_t)joblist::CHAR2NULL == val)) - ret = true; - else if ((colWidth < 5) && ((int32_t)joblist::CHAR4NULL == val)) - ret = true; - else if ((int64_t)joblist::CHAR8NULL == val) - ret = true; - } - else - { - throw std::logic_error("Not a int column."); - } - - break; - } - case execplan::CalpontSystemCatalog::UTINYINT: { if (joblist::UTINYINTNULL == (uint8_t)val) @@ -1265,5 +1257,3 @@ const std::string colDataTypeToString(CalpontSystemCatalog::ColDataType cdt); bool ctListSort(const CalpontSystemCatalog::ColType& a, const CalpontSystemCatalog::ColType& b); } // namespace execplan - -// vim:ts=4 sw=4: diff --git a/dbcon/execplan/clientrotator.cpp b/dbcon/execplan/clientrotator.cpp index 7616f90ed..395d76482 100644 --- a/dbcon/execplan/clientrotator.cpp +++ b/dbcon/execplan/clientrotator.cpp @@ -399,4 +399,3 @@ void ClientRotator::writeToLog(int line, const string& msg, bool critical) const } } // namespace execplan -// vim:ts=4 sw=4: diff --git a/dbcon/execplan/clientrotator.h b/dbcon/execplan/clientrotator.h index d71109441..273863c53 100644 --- a/dbcon/execplan/clientrotator.h +++ b/dbcon/execplan/clientrotator.h @@ -166,4 +166,3 @@ class ClientRotator }; } // namespace execplan -// vim:ts=4 sw=4: diff --git a/dbcon/execplan/constantcolumn.cpp b/dbcon/execplan/constantcolumn.cpp index 65406b108..acfb883d9 100644 --- a/dbcon/execplan/constantcolumn.cpp +++ b/dbcon/execplan/constantcolumn.cpp @@ -48,18 +48,8 @@ ConstantColumn::ConstantColumn(const string& sql, TYPE type) { fResult.strVal = sql; - if (type == LITERAL && sql.length() < 9) - { - memcpy(tmp, sql.c_str(), sql.length()); - memset(tmp + sql.length(), 0, 8); - fResult.uintVal = uint64ToStr(*((uint64_t*)tmp)); - fResult.intVal = (int64_t)fResult.uintVal; - } - else - { - fResult.intVal = atoll(sql.c_str()); - fResult.uintVal = strtoull(sql.c_str(), NULL, 0); - } + fResult.intVal = atoll(sql.c_str()); + fResult.uintVal = strtoull(sql.c_str(), NULL, 0); fResult.floatVal = atof(sql.c_str()); fResult.doubleVal = atof(sql.c_str()); @@ -254,7 +244,8 @@ void ConstantColumn::serialize(messageqcpp::ByteStream& b) const b << (uint32_t)fType; // b << fAlias; b << fData; - b << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + b << timeZone; b << static_cast(fReturnAll); b << (uint64_t)fResult.intVal; b << fResult.uintVal; @@ -278,7 +269,9 @@ void ConstantColumn::unserialize(messageqcpp::ByteStream& b) b >> fConstval; b >> (uint32_t&)fType; b >> fData; - b >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + b >> timeZone; + fTimeZone = timeZone; b >> reinterpret_cast(fReturnAll); b >> (uint64_t&)fResult.intVal; b >> fResult.uintVal; @@ -344,4 +337,3 @@ bool ConstantColumn::operator!=(const TreeNode* t) const } } // namespace execplan -// vim:ts=4 sw=4: diff --git a/dbcon/execplan/constantcolumn.h b/dbcon/execplan/constantcolumn.h index e246eb390..970ffd2a4 100644 --- a/dbcon/execplan/constantcolumn.h +++ b/dbcon/execplan/constantcolumn.h @@ -113,14 +113,14 @@ class ConstantColumn : public ReturnedColumn /** * accessor */ - inline const std::string& timeZone() const + inline long timeZone() const { return fTimeZone; } /** * mutator */ - inline void timeZone(const std::string& timeZone) + inline void timeZone(const long timeZone) { fTimeZone = timeZone; } @@ -204,7 +204,7 @@ class ConstantColumn : public ReturnedColumn std::string fConstval; int fType; std::string fData; - std::string fTimeZone; + long fTimeZone; /*********************************************************** * F&E framework * diff --git a/dbcon/execplan/constantfilter.cpp b/dbcon/execplan/constantfilter.cpp index 4f837d0ad..18f872e03 100644 --- a/dbcon/execplan/constantfilter.cpp +++ b/dbcon/execplan/constantfilter.cpp @@ -308,4 +308,3 @@ void ConstantFilter::setSimpleColumnList() } } // namespace execplan -// vim:ts=4 sw=4: diff --git a/dbcon/execplan/functioncolumn.cpp b/dbcon/execplan/functioncolumn.cpp index 6aa153528..9740a82aa 100644 --- a/dbcon/execplan/functioncolumn.cpp +++ b/dbcon/execplan/functioncolumn.cpp @@ -272,7 +272,8 @@ void FunctionColumn::serialize(messageqcpp::ByteStream& b) const b << fTableAlias; b << fData; - b << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + b << timeZone; } void FunctionColumn::unserialize(messageqcpp::ByteStream& b) @@ -303,10 +304,11 @@ void FunctionColumn::unserialize(messageqcpp::ByteStream& b) b >> fTableAlias; b >> fData; - b >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + b >> timeZone; + fTimeZone = timeZone; FuncExp* funcExp = FuncExp::instance(); fFunctor = funcExp->getFunctor(fFunctionName); - fFunctor->timeZone(fTimeZone); fFunctor->fix(*this); // @bug 3506. Special treatment for rand() function. reset the seed diff --git a/dbcon/execplan/functioncolumn.h b/dbcon/execplan/functioncolumn.h index 887352021..e0b73dae3 100644 --- a/dbcon/execplan/functioncolumn.h +++ b/dbcon/execplan/functioncolumn.h @@ -121,12 +121,12 @@ class FunctionColumn : public ReturnedColumn fTableAlias = tableAlias; } - inline const std::string timeZone() const + inline long timeZone() const { return fTimeZone; } - inline void timeZone(const std::string& timeZone) + inline void timeZone(const long timeZone) { fTimeZone = timeZone; } @@ -181,7 +181,7 @@ class FunctionColumn : public ReturnedColumn std::string fFunctionName; /// function name std::string fTableAlias; /// table alias which has the column std::string fData; /// SQL representation - std::string fTimeZone; + long fTimeZone; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -217,31 +217,38 @@ class FunctionColumn : public ReturnedColumn public: virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); fResult.strVal = fFunctor->getStrVal(row, fFunctionParms, isNull, fOperationType); return fResult.strVal; } virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); return fFunctor->getIntVal(row, fFunctionParms, isNull, fOperationType); } virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); return fFunctor->getUintVal(row, fFunctionParms, isNull, fOperationType); } virtual float getFloatVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); return fFunctor->getFloatVal(row, fFunctionParms, isNull, fOperationType); } virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); return fFunctor->getDoubleVal(row, fFunctionParms, isNull, fOperationType); } virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); return fFunctor->getLongDoubleVal(row, fFunctionParms, isNull, fOperationType); } virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); IDB_Decimal decimal = fFunctor->getDecimalVal(row, fFunctionParms, isNull, fOperationType); if (UNLIKELY(fResultType.colWidth == utils::MAXLEGACYWIDTH && fResultType.scale == decimal.scale)) @@ -276,10 +283,9 @@ class FunctionColumn : public ReturnedColumn if (fResultType.scale > decimal.scale) decimal.value *= IDB_pow[fResultType.scale - decimal.scale]; else - decimal.value = - (int64_t)(decimal.value > 0 - ? (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] + 0.5 - : (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] - 0.5); + decimal.value = (int64_t)( + decimal.value > 0 ? (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] + 0.5 + : (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] - 0.5); } decimal.scale = fResultType.scale; @@ -288,22 +294,27 @@ class FunctionColumn : public ReturnedColumn } virtual bool getBoolVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); return fFunctor->getBoolVal(row, fFunctionParms, isNull, fOperationType); } virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); return fFunctor->getDateIntVal(row, fFunctionParms, isNull, fOperationType); } virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); return fFunctor->getDatetimeIntVal(row, fFunctionParms, isNull, fOperationType); } virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); return fFunctor->getTimestampIntVal(row, fFunctionParms, isNull, fOperationType); } virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); return fFunctor->getTimeIntVal(row, fFunctionParms, isNull, fOperationType); } diff --git a/dbcon/execplan/mcsanalyzetableexecutionplan.cpp b/dbcon/execplan/mcsanalyzetableexecutionplan.cpp index 982135ed9..d548c8245 100644 --- a/dbcon/execplan/mcsanalyzetableexecutionplan.cpp +++ b/dbcon/execplan/mcsanalyzetableexecutionplan.cpp @@ -97,7 +97,8 @@ void MCSAnalyzeTableExecutionPlan::serialize(messageqcpp::ByteStream& bs) const bs << fSchemaName; bs << fTableName; bs << fLocalQuery; - bs << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + bs << timeZone; bs << fTraceFlags; } @@ -149,7 +150,9 @@ void MCSAnalyzeTableExecutionPlan::unserialize(messageqcpp::ByteStream& bs) bs >> fSchemaName; bs >> fTableName; bs >> fLocalQuery; - bs >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + bs >> timeZone; + fTimeZone = timeZone; bs >> fTraceFlags; } } // namespace execplan diff --git a/dbcon/execplan/mcsanalyzetableexecutionplan.h b/dbcon/execplan/mcsanalyzetableexecutionplan.h index ce95809f4..5dc6d8270 100644 --- a/dbcon/execplan/mcsanalyzetableexecutionplan.h +++ b/dbcon/execplan/mcsanalyzetableexecutionplan.h @@ -169,12 +169,12 @@ class MCSAnalyzeTableExecutionPlan : public CalpontExecutionPlan return fUuid; } - void timeZone(const std::string& timezone) + void timeZone(long timezone) { fTimeZone = timezone; } - const std::string timeZone() const + long timeZone() const { return fTimeZone; } @@ -256,7 +256,7 @@ class MCSAnalyzeTableExecutionPlan : public CalpontExecutionPlan std::string fTableName; uint32_t fTraceFlags; boost::uuids::uuid fUuid; - std::string fTimeZone; + long fTimeZone; uint32_t fStatementID; uint64_t fStringScanThreshold; std::string fData; diff --git a/dbcon/execplan/predicateoperator.cpp b/dbcon/execplan/predicateoperator.cpp index 465f32af2..6ff4d311b 100644 --- a/dbcon/execplan/predicateoperator.cpp +++ b/dbcon/execplan/predicateoperator.cpp @@ -264,20 +264,6 @@ void PredicateOperator::setOpType(Type& l, Type& r) fOperationType.colWidth = 8; } } - // If both sides are unsigned, use UBIGINT as result type, otherwise - // "promote" to BIGINT. - else if (isUnsigned(l.colDataType) && isUnsigned(r.colDataType)) - { - fOperationType.colDataType = execplan::CalpontSystemCatalog::UBIGINT; - fOperationType.colWidth = 8; - } - else if ((isSignedInteger(l.colDataType) && isUnsigned(r.colDataType)) || - (isUnsigned(l.colDataType) && isSignedInteger(r.colDataType)) || - (isSignedInteger(l.colDataType) && isSignedInteger(r.colDataType))) - { - fOperationType.colDataType = execplan::CalpontSystemCatalog::BIGINT; - fOperationType.colWidth = 8; - } else if ((l.colDataType == execplan::CalpontSystemCatalog::CHAR || l.colDataType == execplan::CalpontSystemCatalog::VARCHAR || l.colDataType == execplan::CalpontSystemCatalog::TEXT) && @@ -321,6 +307,20 @@ void PredicateOperator::setOpType(Type& l, Type& r) fOperationType.colWidth = 255; } } + // If both sides are unsigned, use UBIGINT as result type, otherwise + // "promote" to BIGINT. + else if (isUnsigned(l.colDataType) && isInteger(l.colDataType) && isUnsigned(r.colDataType) && isInteger(r.colDataType)) + { + fOperationType.colDataType = execplan::CalpontSystemCatalog::UBIGINT; + fOperationType.colWidth = 8; + } + else if ((isSignedInteger(l.colDataType) && isUnsigned(r.colDataType) && isInteger(r.colDataType)) || + (isUnsigned(l.colDataType) && isInteger(l.colDataType) && isSignedInteger(r.colDataType)) || + (isSignedInteger(l.colDataType) && isSignedInteger(r.colDataType))) + { + fOperationType.colDataType = execplan::CalpontSystemCatalog::BIGINT; + fOperationType.colWidth = 8; + } else if (l.colDataType == execplan::CalpontSystemCatalog::LONGDOUBLE || r.colDataType == execplan::CalpontSystemCatalog::LONGDOUBLE) { @@ -410,7 +410,9 @@ bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, ReturnedCol if (isNull) return false; - return numericCompare(val1, rop->getIntVal(row, isNull)) && !isNull; + int64_t val2 = rop->getIntVal(row, isNull); + + return numericCompare(val1, val2) && !isNull; } case execplan::CalpontSystemCatalog::UBIGINT: diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index da82e0726..b732a2825 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -36,7 +36,6 @@ #endif #include #include -#include #include "expressionparser.h" #include "returnedcolumn.h" diff --git a/dbcon/execplan/sessionmanager.h b/dbcon/execplan/sessionmanager.h index be60a5b6c..2e46b0cd6 100644 --- a/dbcon/execplan/sessionmanager.h +++ b/dbcon/execplan/sessionmanager.h @@ -213,4 +213,3 @@ class SessionManager } // namespace execplan -// vim:ts=4 sw=4: diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index d08144d75..2e3226f5f 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -345,7 +345,8 @@ void SimpleColumn::serialize(messageqcpp::ByteStream& b) const b << fColumnName; b << fIndexName; b << fViewName; - b << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + b << timeZone; b << (uint32_t)fOid; b << fData; b << fTableAlias; @@ -362,7 +363,9 @@ void SimpleColumn::unserialize(messageqcpp::ByteStream& b) b >> fColumnName; b >> fIndexName; b >> fViewName; - b >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + b >> timeZone; + fTimeZone = timeZone; b >> (uint32_t&)fOid; b >> fData; b >> fTableAlias; @@ -562,6 +565,16 @@ void SimpleColumn::evaluate(Row& row, bool& isNull) else fResult.intVal = atoll((char*)&fResult.origIntVal); + // MCOL-4580 - related, probably can be marked with XXX. + // This does not fail in any tests, but it is considered wrong. + // The reasonin behind that is that we changed signedness if characters to unsigned + // and it might be a case with short strings that they were copied as is using + // uint64ToStr encoding into int64_t values. So, potentially, unsuspecting code + // may use getUintVal instead of getIntVal to process short char column, getting + // unitialized value and give floating behavior. + // None of our tests failed, though. + fResult.uintVal = fResult.intVal; + break; } diff --git a/dbcon/execplan/simplecolumn.h b/dbcon/execplan/simplecolumn.h index a7bcaa80a..8f35d3f1f 100644 --- a/dbcon/execplan/simplecolumn.h +++ b/dbcon/execplan/simplecolumn.h @@ -151,11 +151,11 @@ class SimpleColumn : public ReturnedColumn if (lower_case_table_names) boost::algorithm::to_lower(fViewName); } - inline const std::string& timeZone() const + inline long timeZone() const { return fTimeZone; } - inline void timeZone(const std::string& timeZone) + inline void timeZone(const long timeZone) { fTimeZone = timeZone; } @@ -259,7 +259,7 @@ class SimpleColumn : public ReturnedColumn std::string fIndexName; // if belong to view, view name is non-empty std::string fViewName; - std::string fTimeZone; + long fTimeZone; bool fisColumnStore; /** @brief parse SimpleColumn text @@ -349,7 +349,7 @@ class SimpleColumn : public ReturnedColumn inline int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) { evaluate(row, isNull); - return TreeNode::getDatetimeIntVal(); + return TreeNode::getDatetimeIntVal(fTimeZone); } inline int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) diff --git a/dbcon/execplan/simplefilter.cpp b/dbcon/execplan/simplefilter.cpp index ed1459b25..4d48ad212 100644 --- a/dbcon/execplan/simplefilter.cpp +++ b/dbcon/execplan/simplefilter.cpp @@ -55,7 +55,7 @@ SimpleFilter::SimpleFilter(const string& sql) : Filter(sql) parse(sql); } -SimpleFilter::SimpleFilter(const SOP& op, ReturnedColumn* lhs, ReturnedColumn* rhs, const string& timeZone) +SimpleFilter::SimpleFilter(const SOP& op, ReturnedColumn* lhs, ReturnedColumn* rhs, const long timeZone) : fOp(op), fLhs(lhs), fRhs(rhs), fIndexFlag(NOINDEX), fJoinFlag(EQUA), fTimeZone(timeZone) { convertConstant(); @@ -314,7 +314,8 @@ void SimpleFilter::serialize(messageqcpp::ByteStream& b) const b << static_cast(fIndexFlag); b << static_cast(fJoinFlag); - b << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + b << timeZone; } void SimpleFilter::unserialize(messageqcpp::ByteStream& b) @@ -330,7 +331,9 @@ void SimpleFilter::unserialize(messageqcpp::ByteStream& b) fRhs = dynamic_cast(ObjectReader::createTreeNode(b)); b >> reinterpret_cast(fIndexFlag); b >> reinterpret_cast(fJoinFlag); - b >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + b >> timeZone; + fTimeZone = timeZone; fSimpleColumnList.clear(); fAggColumnList.clear(); diff --git a/dbcon/execplan/simplefilter.h b/dbcon/execplan/simplefilter.h index 900afec2d..85d2a06c9 100644 --- a/dbcon/execplan/simplefilter.h +++ b/dbcon/execplan/simplefilter.h @@ -67,7 +67,7 @@ class SimpleFilter : public Filter SimpleFilter(); SimpleFilter(const std::string& sql); - SimpleFilter(const SOP& op, ReturnedColumn* lhs, ReturnedColumn* rhs, const std::string& timeZone = ""); + SimpleFilter(const SOP& op, ReturnedColumn* lhs, ReturnedColumn* rhs, const long timeZone = 0); SimpleFilter(const SimpleFilter& rhs); virtual ~SimpleFilter(); @@ -92,12 +92,12 @@ class SimpleFilter : public Filter return fLhs; } - inline const std::string& timeZone() const + inline long timeZone() const { return fTimeZone; } - inline void timeZone(const std::string& timeZone) + inline void timeZone(const long timeZone) { fTimeZone = timeZone; } @@ -220,7 +220,7 @@ class SimpleFilter : public Filter ReturnedColumn* fRhs; /// right operand int fIndexFlag; /// which side col is index int fJoinFlag; /// hash join type - std::string fTimeZone; + long fTimeZone; void parse(std::string); diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 496a6436f..f8351669b 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -36,6 +36,7 @@ #include "columnwidth.h" #include "mcs_decimal.h" #include "mcs_int64.h" +#include "numericliteral.h" namespace messageqcpp { @@ -62,7 +63,7 @@ typedef datatypes::Decimal IDB_Decimal; #ifdef POSIX_REGEX typedef regex_t IDB_Regex; #else -typedef boost::regex IDB_Regex; +typedef std::regex IDB_Regex; #endif typedef IDB_Regex CNX_Regex; @@ -325,7 +326,7 @@ class TreeNode } inline bool getBoolVal(); - inline const std::string& getStrVal(const std::string& timeZone); + inline const std::string& getStrVal(const long timeZone); inline int64_t getIntVal(); inline uint64_t getUintVal(); inline float getFloatVal(); @@ -333,7 +334,7 @@ class TreeNode inline long double getLongDoubleVal(); inline IDB_Decimal getDecimalVal(); inline int32_t getDateIntVal(); - inline int64_t getDatetimeIntVal(); + inline int64_t getDatetimeIntVal(long timeZone = 0); inline int64_t getTimestampIntVal(); inline int64_t getTimeIntVal(); @@ -457,7 +458,7 @@ inline bool TreeNode::getBoolVal() return fResult.boolVal; } -inline const std::string& TreeNode::getStrVal(const std::string& timeZone) +inline const std::string& TreeNode::getStrVal(const long timeZone) { switch (fResultType.colDataType) { @@ -664,25 +665,19 @@ inline int64_t TreeNode::getIntVal() switch (fResultType.colDataType) { case CalpontSystemCatalog::CHAR: - if (fResultType.colWidth <= 8) - return fResult.intVal; - - return atoll(fResult.strVal.c_str()); - case CalpontSystemCatalog::VARCHAR: - if (fResultType.colWidth <= 7) - return fResult.intVal; - - return atoll(fResult.strVal.c_str()); - - // FIXME: ??? case CalpontSystemCatalog::VARBINARY: case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::TEXT: - if (fResultType.colWidth <= 7) - return fResult.intVal; - - return atoll(fResult.strVal.c_str()); + { + datatypes::DataCondition cnverr; + literal::Converter cnv(fResult.strVal, cnverr); + if (datatypes::DataCondition::Code(cnverr) != 0) + { + cerr << "error in int conversion from '" << fResult.strVal << "'"; + } + return cnv.toSInt(cnverr); + } case CalpontSystemCatalog::BIGINT: case CalpontSystemCatalog::TINYINT: @@ -721,6 +716,20 @@ inline uint64_t TreeNode::getUintVal() { switch (fResultType.colDataType) { + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + { + datatypes::DataCondition cnverr; + literal::Converter cnv(fResult.strVal, cnverr); + if (datatypes::DataCondition::Code(cnverr) != 0) + { + cerr << "error in unsigned int conversion from '" << fResult.strVal << "'"; + } + return cnv.toXIntPositive(cnverr); + } case CalpontSystemCatalog::BIGINT: case CalpontSystemCatalog::TINYINT: case CalpontSystemCatalog::SMALLINT: @@ -1052,7 +1061,7 @@ inline IDB_Decimal TreeNode::getDecimalVal() return fResult.decimalVal; } -inline int64_t TreeNode::getDatetimeIntVal() +inline int64_t TreeNode::getDatetimeIntVal(long timeZone) { if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATE) return (fResult.intVal & 0x00000000FFFFFFC0LL) << 32; @@ -1083,6 +1092,17 @@ inline int64_t TreeNode::getDatetimeIntVal() else if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATETIME) // return (fResult.intVal & 0xFFFFFFFFFFF00000LL); return (fResult.intVal); + else if (fResultType.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) + { + dataconvert::TimeStamp timestamp(fResult.intVal); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone); + dataconvert::DateTime dt(m_time.year, m_time.month, m_time.day, m_time.hour, m_time.minute, m_time.second, + timestamp.msecond); + memcpy(&fResult.intVal, &dt, 8); + return fResult.intVal; + } else return getIntVal(); } diff --git a/dbcon/execplan/windowfunctioncolumn.cpp b/dbcon/execplan/windowfunctioncolumn.cpp index 00b2e9ac5..dc39677d8 100644 --- a/dbcon/execplan/windowfunctioncolumn.cpp +++ b/dbcon/execplan/windowfunctioncolumn.cpp @@ -288,7 +288,8 @@ void WindowFunctionColumn::serialize(messageqcpp::ByteStream& b) const fOrderBy.serialize(b); udafContext.serialize(b); - b << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + b << timeZone; } void WindowFunctionColumn::unserialize(messageqcpp::ByteStream& b) @@ -320,7 +321,9 @@ void WindowFunctionColumn::unserialize(messageqcpp::ByteStream& b) fOrderBy.unserialize(b); udafContext.unserialize(b); - b >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + b >> timeZone; + fTimeZone = timeZone; } void WindowFunctionColumn::addToPartition(vector& groupByList) diff --git a/dbcon/execplan/windowfunctioncolumn.h b/dbcon/execplan/windowfunctioncolumn.h index 5f2b21294..b40c2dd27 100644 --- a/dbcon/execplan/windowfunctioncolumn.h +++ b/dbcon/execplan/windowfunctioncolumn.h @@ -146,12 +146,12 @@ class WindowFunctionColumn : public ReturnedColumn return udafContext; } - inline const std::string timeZone() const + inline long timeZone() const { return fTimeZone; } - inline void timeZone(const std::string& timeZone) + inline void timeZone(const long timeZone) { fTimeZone = timeZone; } @@ -180,7 +180,7 @@ class WindowFunctionColumn : public ReturnedColumn // UDAnF support mcsv1sdk::mcsv1Context udafContext; - std::string fTimeZone; + long fTimeZone; /*********************************************************** * F&E framework * ***********************************************************/ diff --git a/dbcon/joblist/CMakeLists.txt b/dbcon/joblist/CMakeLists.txt index 77fca4a5e..b025005b8 100644 --- a/dbcon/joblist/CMakeLists.txt +++ b/dbcon/joblist/CMakeLists.txt @@ -36,7 +36,6 @@ set(joblist_LIB_SRCS pcolstep.cpp pdictionary.cpp pdictionaryscan.cpp - primitivemsg.cpp pseudocc-jl.cpp resourcedistributor.cpp resourcemanager.cpp @@ -59,7 +58,7 @@ set(joblist_LIB_SRCS ${ENGINE_SRC_DIR}/tools/passwd/secrets.cpp) add_library(joblist SHARED ${joblist_LIB_SRCS}) - +target_include_directories(joblist BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR}) add_dependencies(joblist loggingcpp) install(TARGETS joblist DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) diff --git a/dbcon/joblist/anydatalist.cpp b/dbcon/joblist/anydatalist.cpp index 4e5df4713..da1ae05a2 100644 --- a/dbcon/joblist/anydatalist.cpp +++ b/dbcon/joblist/anydatalist.cpp @@ -170,4 +170,3 @@ std::ostream& omitOidInDL(std::ostream& strm) } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.cpp b/dbcon/joblist/batchprimitiveprocessor-jl.cpp index 702c4c091..4699042bc 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.cpp +++ b/dbcon/joblist/batchprimitiveprocessor-jl.cpp @@ -50,6 +50,8 @@ using namespace messageqcpp; using namespace rowgroup; using namespace joiner; +//#define XXX_BATCHPRIMPROC_TOKENS_RANGES_XXX + namespace joblist { BatchPrimitiveProcessorJL::BatchPrimitiveProcessorJL(const ResourceManager* rm) @@ -152,6 +154,21 @@ void BatchPrimitiveProcessorJL::addFilterStep(const pDictionaryStep& step) cc->setBatchPrimitiveProcessor(this); cc->setQueryUuid(step.queryUuid()); cc->setStepUuid(uuid); + +#if defined(XXX_BATCHPRIMPROC_TOKENS_RANGES_XXX) + if (filterSteps.size() > 0) + { + size_t stepsIndex = filterSteps.size() - 1; + SCommand prevCC = filterSteps[stepsIndex]; + ColumnCommandJL* pcc = dynamic_cast(prevCC.get()); + DictStepJL* ccc = dynamic_cast(cc.get()); + if (pcc && ccc) + { + filterSteps[stepsIndex].reset( + new ColumnCommandJL(*pcc, *ccc)); // column command will use same filters. + } + } +#endif filterSteps.push_back(cc); filterCount++; needStrValues = true; @@ -443,6 +460,7 @@ void BatchPrimitiveProcessorJL::getElementTypes(ByteStream& in, vector> *lbid; + in >> tmp64; *min = (int64_t)tmp64; in >> tmp64; @@ -712,8 +730,9 @@ bool BatchPrimitiveProcessorJL::countThisMsg(messageqcpp::ByteStream& in) const } if (data[offset] != 0) - offset += (data[offset + CP_FLAG_AND_LBID] * 2) + CP_FLAG_AND_LBID + - 1; // skip the CP data with wide min/max values (16/32 bytes each) + offset += (data[offset + CP_FLAG_AND_LBID + 1] * 2) + CP_FLAG_AND_LBID + 1 + + 1; // skip the CP data with wide min/max values (16/32 bytes each). we also skip + // cpFromDictScan flag. else offset += CP_FLAG_AND_LBID; // skip only the "valid CP data" & LBID bytes } @@ -750,9 +769,10 @@ void BatchPrimitiveProcessorJL::deserializeAggregateResult(ByteStream* in, vecto } void BatchPrimitiveProcessorJL::getRowGroupData(ByteStream& in, vector* out, bool* validCPData, - uint64_t* lbid, int128_t* min, int128_t* max, - uint32_t* cachedIO, uint32_t* physIO, uint32_t* touchedBlocks, - bool* countThis, uint32_t threadID, bool* hasWideColumn, + uint64_t* lbid, bool* fromDictScan, int128_t* min, + int128_t* max, uint32_t* cachedIO, uint32_t* physIO, + uint32_t* touchedBlocks, bool* countThis, uint32_t threadID, + bool* hasWideColumn, const execplan::CalpontSystemCatalog::ColType& colType) const { uint64_t tmp64; @@ -789,6 +809,8 @@ void BatchPrimitiveProcessorJL::getRowGroupData(ByteStream& in, vector* { in >> *lbid; in >> tmp8; + *fromDictScan = tmp8 != 0; + in >> tmp8; *hasWideColumn = (tmp8 > utils::MAXLEGACYWIDTH); if (UNLIKELY(*hasWideColumn)) { diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.h b/dbcon/joblist/batchprimitiveprocessor-jl.h index 6dbc76016..a249b3102 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.h +++ b/dbcon/joblist/batchprimitiveprocessor-jl.h @@ -167,9 +167,9 @@ class BatchPrimitiveProcessorJL uint32_t* touchedBlocks) const; void deserializeAggregateResults(messageqcpp::ByteStream* in, std::vector* out) const; void getRowGroupData(messageqcpp::ByteStream& in, std::vector* out, bool* validCPData, - uint64_t* lbid, int128_t* min, int128_t* max, uint32_t* cachedIO, uint32_t* physIO, - uint32_t* touchedBlocks, bool* countThis, uint32_t threadID, bool* hasBinaryColumn, - const execplan::CalpontSystemCatalog::ColType& colType) const; + uint64_t* lbid, bool* fromDictScan, int128_t* min, int128_t* max, uint32_t* cachedIO, + uint32_t* physIO, uint32_t* touchedBlocks, bool* countThis, uint32_t threadID, + bool* hasBinaryColumn, const execplan::CalpontSystemCatalog::ColType& colType) const; void deserializeAggregateResult(messageqcpp::ByteStream* in, std::vector* out) const; bool countThisMsg(messageqcpp::ByteStream& in) const; @@ -365,5 +365,3 @@ class BatchPrimitiveProcessorJL }; } // namespace joblist - -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/bpp-jl.h b/dbcon/joblist/bpp-jl.h index fc64fae2f..4da0d5e1c 100644 --- a/dbcon/joblist/bpp-jl.h +++ b/dbcon/joblist/bpp-jl.h @@ -28,7 +28,6 @@ // // /** @file */ -// #include "primitivemsg.h" #include "bytestream.h" #include "messagequeue.h" #include "serializeable.h" diff --git a/dbcon/joblist/columncommand-jl.cpp b/dbcon/joblist/columncommand-jl.cpp index 8e42a73e6..bc441168b 100644 --- a/dbcon/joblist/columncommand-jl.cpp +++ b/dbcon/joblist/columncommand-jl.cpp @@ -130,6 +130,59 @@ ColumnCommandJL::ColumnCommandJL(const pColStep& step) fFilesPerColumnPartition = cf->uFromText(fpc); } +ColumnCommandJL::ColumnCommandJL(const ColumnCommandJL& prevCmd, const DictStepJL& dictWithFilters) +{ + BRM::DBRM dbrm; + + /* grab necessary vars from scan */ + traceFlags = prevCmd.traceFlags; + // we should call this constructor only when paired with dictionary + // and in that case previous command should not have any filters and + // should be "dict" (tokens) column command. + idbassert(dictWithFilters.getFilterCount() == 0 || prevCmd.filterCount == 0); + idbassert(prevCmd.fIsDict); + + // need to reencode filters. + filterString = dictWithFilters.reencodedFilterString(); + // we have a limitation here. + // consider this: textcol IS NULL AND textcol IN ('a', 'b') + // XXX: should check. + if (filterString.length() > 0 && (BOP = dictWithFilters.getBop() || prevCmd.filterString.length() < 1)) + { + filterCount = dictWithFilters.getFilterCount(); + BOP = dictWithFilters.getBop(); + fContainsRanges = true; + } + else + { + filterCount = prevCmd.filterCount; + filterString = prevCmd.filterString; + BOP = prevCmd.BOP; + } + isScan = prevCmd.isScan; + colType = prevCmd.colType; + extents = prevCmd.extents; + OID = prevCmd.OID; + colName = prevCmd.colName; + rpbShift = prevCmd.rpbShift; + fIsDict = prevCmd.fIsDict; + fLastLbid = prevCmd.fLastLbid; + lbid = prevCmd.lbid; + traceFlags = prevCmd.traceFlags; + dbroot = prevCmd.dbroot; + numDBRoots = prevCmd.numDBRoots; + + /* I think modmask isn't necessary for scans */ + divShift = prevCmd.divShift; + modMask = (1 << divShift) - 1; + + // @Bug 2889. Drop partition enhancement. Read FilesPerColumnPartition and ExtentsPerSegmentFile for use + // in RID calculation. + fFilesPerColumnPartition = prevCmd.fFilesPerColumnPartition; + // MCOL-4685 remove the option to set more than 2 extents per file (ExtentsPreSegmentFile). + fExtentsPerSegmentFile = prevCmd.fExtentsPerSegmentFile; +} + ColumnCommandJL::~ColumnCommandJL() { } @@ -141,9 +194,22 @@ void ColumnCommandJL::createCommand(ByteStream& bs) const colType.serialize(bs); bs << (uint8_t)isScan; bs << traceFlags; - bs << filterString; - bs << BOP; - bs << filterCount; + if (isDict() && fContainsRanges) + { + // XXX: we should discern here between IS (NOT) NULL and other filters. + ByteStream empty; + auto zeroFC = filterCount; + bs << empty; + bs << BOP; + zeroFC = 0; + bs << zeroFC; + } + else + { + bs << filterString; + bs << BOP; + bs << filterCount; + } serializeInlineVector(bs, fLastLbid); CommandJL::createCommand(bs); @@ -250,7 +316,7 @@ string ColumnCommandJL::toString() { ostringstream ret; - ret << "ColumnCommandJL: " << filterCount << " filters colwidth=" << colType.colWidth << " oid=" << OID + ret << "ColumnCommandJL: " << filterCount << " filters, BOP=" << ((int)BOP) << ", colwidth=" << colType.colWidth << " oid=" << OID << " name=" << colName; if (isScan) @@ -286,4 +352,9 @@ void ColumnCommandJL::reloadExtents() sort(extents.begin(), extents.end(), BRM::ExtentSorter()); } +bool ColumnCommandJL::getIsDict() +{ + return fIsDict; +} + }; // namespace joblist diff --git a/dbcon/joblist/columncommand-jl.h b/dbcon/joblist/columncommand-jl.h index 407bef1cc..7f700766e 100644 --- a/dbcon/joblist/columncommand-jl.h +++ b/dbcon/joblist/columncommand-jl.h @@ -33,6 +33,7 @@ #include "primitivestep.h" #include "command-jl.h" +#include "dictstep-jl.h" namespace joblist { @@ -41,15 +42,16 @@ class ColumnCommandJL : public CommandJL public: ColumnCommandJL(const pColScanStep&, std::vector lastLBID); ColumnCommandJL(const pColStep&); + ColumnCommandJL(const ColumnCommandJL&, const DictStepJL&); virtual ~ColumnCommandJL(); - virtual void createCommand(messageqcpp::ByteStream& bs) const; - virtual void runCommand(messageqcpp::ByteStream& bs) const; - void setLBID(uint64_t rid, uint32_t dbroot); - uint8_t getTableColumnType(); - virtual std::string toString(); - uint16_t getWidth(); - CommandType getCommandType() + virtual void createCommand(messageqcpp::ByteStream& bs) const override; + virtual void runCommand(messageqcpp::ByteStream& bs) const override; + void setLBID(uint64_t rid, uint32_t dbroot) override; + uint8_t getTableColumnType() override; + virtual std::string toString() override; + uint16_t getWidth() override; + CommandType getCommandType() override { return COLUMN_COMMAND; } @@ -111,6 +113,7 @@ class ColumnCommandJL : public CommandJL std::vector fLastLbid; bool fIsDict; + bool fContainsRanges = false; // @Bug 2889. Added two members below for drop partition enhancement. // RJD: make sure that we keep enough significant digits around for partition math @@ -125,8 +128,7 @@ class ColumnCommandJL : public CommandJL public: // MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile) static const unsigned DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2; + bool getIsDict() override; }; } // namespace joblist - -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/command-jl.h b/dbcon/joblist/command-jl.h index a45fc5a67..3921b419b 100644 --- a/dbcon/joblist/command-jl.h +++ b/dbcon/joblist/command-jl.h @@ -96,6 +96,11 @@ class CommandJL virtual CommandType getCommandType() = 0; + virtual bool getIsDict() + { + return false; + } + protected: BatchPrimitiveProcessorJL* bpp; uint32_t OID; diff --git a/dbcon/joblist/crossenginestep.cpp b/dbcon/joblist/crossenginestep.cpp index e0475c113..d03f21e0d 100644 --- a/dbcon/joblist/crossenginestep.cpp +++ b/dbcon/joblist/crossenginestep.cpp @@ -828,4 +828,3 @@ void CrossEngineStep::formatMiniStats() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/crossenginestep.h b/dbcon/joblist/crossenginestep.h index d3d1a4dba..ac657f9b5 100644 --- a/dbcon/joblist/crossenginestep.h +++ b/dbcon/joblist/crossenginestep.h @@ -249,4 +249,3 @@ class CrossEngineStep : public BatchPrimitive, public TupleDeliveryStep } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/dictstep-jl.cpp b/dbcon/joblist/dictstep-jl.cpp index 2c3c51737..1e39d2407 100644 --- a/dbcon/joblist/dictstep-jl.cpp +++ b/dbcon/joblist/dictstep-jl.cpp @@ -29,6 +29,7 @@ // #include "bpp-jl.h" +#include "string_prefixes.h" using namespace std; using namespace messageqcpp; @@ -50,7 +51,6 @@ DictStepJL::DictStepJL(const pDictionaryStep& dict) if (hasEqFilter) { - // cout << "saw eqfilter\n"; eqOp = dict.tmpCOP; eqFilter = dict.eqFilter; } @@ -120,4 +120,63 @@ void DictStepJL::setWidth(uint16_t w) colWidth = w; } +messageqcpp::ByteStream DictStepJL::reencodedFilterString() const +{ + messageqcpp::ByteStream bs; + + if (hasEqFilter) + { + idbassert(filterCount == eqFilter.size()); + + for (uint32_t i = 0; i < filterCount; i++) + { + uint8_t roundFlag = 0; + int64_t encodedPrefix = encodeStringPrefix((unsigned char*)eqFilter[i].c_str(), eqFilter[i].size(), charsetNumber); + bs << eqOp; + bs << roundFlag; + bs << encodedPrefix; + } + } + else + { + messageqcpp::ByteStream filterStringCopy( + filterString); // XXX I am not sure about real semantics of messagecpp::ByteStream. So - copy. + // please erfer to pdictionary.cpp in this dicrectory, addFilter function for a proper encoding of string + // filters. + for (uint32_t i = 0; i < filterCount; i++) + { + uint8_t cop, roundFlag = 0; + uint16_t size; + const uint8_t* ptr; + int64_t encodedPrefix; + filterStringCopy >> cop; + // as we are dealing with prefixes, we have to use "... or equal" conditions instead of + // strict ones. + // Consider this: ... WHERE col > 'customer#001' AND col < 'customer#100'. + // "Working with prefixes of 8 bytes" means these conditions reduce to ... WHERE col > 'customer' AND + // col < 'customer' and their AND relation is impossible to satisfy. We do not pass this string to + // primproc and that means we can reencode operation codes here. + switch (cop) + { + case COMPARE_LT: + case COMPARE_NGE: cop = COMPARE_LE; break; + + case COMPARE_GT: + case COMPARE_NLE: cop = COMPARE_GE; break; + + default: break; + } + + bs << cop; + bs << roundFlag; + filterStringCopy >> size; + ptr = filterStringCopy.buf(); + encodedPrefix = encodeStringPrefix(ptr, size, charsetNumber); + bs << encodedPrefix; + filterStringCopy.advance(size); + } + } + return bs; +} + }; // namespace joblist diff --git a/dbcon/joblist/dictstep-jl.h b/dbcon/joblist/dictstep-jl.h index 7bf38fa10..5fe0c618e 100644 --- a/dbcon/joblist/dictstep-jl.h +++ b/dbcon/joblist/dictstep-jl.h @@ -61,6 +61,21 @@ class DictStepJL : public CommandJL void createCommand(messageqcpp::ByteStream&) const; void runCommand(messageqcpp::ByteStream&) const; + messageqcpp::ByteStream getFilterString() const + { + return filterString; + } + uint32_t getFilterCount() const + { + return filterCount; + } + messageqcpp::ByteStream reencodedFilterString() const; + + uint8_t getBop() const + { + return BOP; + } + private: DictStepJL(const DictStepJL&); diff --git a/dbcon/joblist/diskjoinstep.cpp b/dbcon/joblist/diskjoinstep.cpp index 87be21660..2f69f9627 100644 --- a/dbcon/joblist/diskjoinstep.cpp +++ b/dbcon/joblist/diskjoinstep.cpp @@ -163,7 +163,7 @@ void DiskJoinStep::smallReader() RGData rgData; bool more = true; int64_t memUsage = 0, combinedMemUsage = 0; - int rowCount = 0; + [[maybe_unused]] int rowCount = 0; RowGroup l_smallRG = smallRG; try @@ -224,7 +224,7 @@ void DiskJoinStep::largeReader() RGData rgData; bool more = true; int64_t largeSize = 0; - int rowCount = 0; + [[maybe_unused]] int rowCount = 0; RowGroup l_largeRG = largeRG; largeIterationCount++; diff --git a/dbcon/joblist/distributedenginecomm.cpp b/dbcon/joblist/distributedenginecomm.cpp index 026cbf516..32f5ee45e 100644 --- a/dbcon/joblist/distributedenginecomm.cpp +++ b/dbcon/joblist/distributedenginecomm.cpp @@ -199,7 +199,7 @@ void DistributedEngineComm::reset() } DistributedEngineComm::DistributedEngineComm(ResourceManager* rm, bool isExeMgr) - : fRm(rm), fLBIDShift(fRm->getPsLBID_Shift()), pmCount(0), fIsExeMgr(isExeMgr) +: fRm(rm), pmCount(0), fIsExeMgr(isExeMgr) { Setup(); } @@ -250,10 +250,6 @@ void DistributedEngineComm::Setup() if (newPmCount == 0) writeToLog(__FILE__, __LINE__, "Got a config file with 0 PMs", LOG_TYPE_CRITICAL); - // This needs to make sense when compared to the extent size - // fLBIDShift = static_cast(config::Config::uFromText(fConfig->getConfig(section, - // "LBID_Shift"))); - auto* config = fRm->getConfig(); std::vector pmsAddressesAndPorts; for (size_t i = 1; i <= newPmCount; ++i) @@ -292,7 +288,7 @@ void DistributedEngineComm::Setup() catch (std::exception& ex) { if (i < newPmCount) - newPmCount--; + newPmCount = newPmCount > 1 ? newPmCount-1 : 1; // We can't afford to reduce newPmCount to 0 writeToLog(__FILE__, __LINE__, "Could not connect to PMS" + std::to_string(connectionId) + ": " + ex.what(), @@ -306,7 +302,7 @@ void DistributedEngineComm::Setup() catch (...) { if (i < newPmCount) - newPmCount--; + newPmCount = newPmCount > 1 ? newPmCount-1 : 1; // We can't afford to reduce newPmCount to 0 writeToLog(__FILE__, __LINE__, "Could not connect to PMS" + std::to_string(connectionId), LOG_TYPE_ERROR); @@ -1155,4 +1151,3 @@ uint32_t DistributedEngineComm::MQE::getNextConnectionId(const size_t pmIndex, } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/distributedenginecomm.h b/dbcon/joblist/distributedenginecomm.h index 51ae885fe..0ec286289 100644 --- a/dbcon/joblist/distributedenginecomm.h +++ b/dbcon/joblist/distributedenginecomm.h @@ -269,7 +269,6 @@ class DistributedEngineComm boost::mutex fMlock; // sessionMessages mutex std::vector > fWlock; // PrimProc socket write mutexes bool fBusy; - unsigned fLBIDShift; volatile uint32_t pmCount; boost::mutex fOnErrMutex; // to lock function scope to reset pmconnections under error condition boost::mutex fSetupMutex; diff --git a/dbcon/joblist/elementtype.h b/dbcon/joblist/elementtype.h index 742944ffc..a91a1f062 100644 --- a/dbcon/joblist/elementtype.h +++ b/dbcon/joblist/elementtype.h @@ -638,4 +638,3 @@ extern std::ostream& omitOidInDL(std::ostream& strm); #endif -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/errorinfo.h b/dbcon/joblist/errorinfo.h index 88d1d0128..62b404daa 100644 --- a/dbcon/joblist/errorinfo.h +++ b/dbcon/joblist/errorinfo.h @@ -52,4 +52,3 @@ typedef boost::shared_ptr SErrorInfo; } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/expressionstep.cpp b/dbcon/joblist/expressionstep.cpp index 2f5b96afc..b182c7b21 100644 --- a/dbcon/joblist/expressionstep.cpp +++ b/dbcon/joblist/expressionstep.cpp @@ -790,4 +790,3 @@ const string ExpressionStep::toString() const } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/fifo.h b/dbcon/joblist/fifo.h index d73ebc937..46ebf9922 100644 --- a/dbcon/joblist/fifo.h +++ b/dbcon/joblist/fifo.h @@ -529,4 +529,3 @@ void FIFO::totalFileCounts(uint64_t& numFiles, uint64_t& numBytes) co } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index 5eb0e1b10..e44026468 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -373,7 +373,15 @@ void GroupConcatAgUM::applyMapping(const boost::shared_array& mapping, cons } else { - fRow.setIntField(row.getIntField(mapping[i]), i); + if (fRow.getColTypes()[i] == execplan::CalpontSystemCatalog::CHAR || + fRow.getColTypes()[i] == execplan::CalpontSystemCatalog::VARCHAR) + { + fRow.setIntField(row.getUintField(mapping[i]), i); + } + else + { + fRow.setIntField(row.getIntField(mapping[i]), i); + } } } } @@ -1050,4 +1058,3 @@ const string GroupConcatNoOrder::toString() const } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/groupconcat.h b/dbcon/joblist/groupconcat.h index 24f14348d..6db944b57 100644 --- a/dbcon/joblist/groupconcat.h +++ b/dbcon/joblist/groupconcat.h @@ -127,7 +127,7 @@ class GroupConcator int64_t fGroupConcatLen; int64_t fConstantLen; boost::scoped_array fOutputString; - std::string fTimeZone; + long fTimeZone; }; // For GROUP_CONCAT withour distinct or orderby diff --git a/dbcon/joblist/jlf_common.cpp b/dbcon/joblist/jlf_common.cpp index 6ae8724e6..47a1ecc60 100644 --- a/dbcon/joblist/jlf_common.cpp +++ b/dbcon/joblist/jlf_common.cpp @@ -822,4 +822,3 @@ bool compatibleColumnTypes(const CalpontSystemCatalog::ColDataType& dt1, uint32_ } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/jlf_common.h b/dbcon/joblist/jlf_common.h index 19984cbdc..61006e9e8 100644 --- a/dbcon/joblist/jlf_common.h +++ b/dbcon/joblist/jlf_common.h @@ -185,14 +185,8 @@ struct JobInfo , maxElems(rm->getHjMaxElems()) , flushInterval(rm->getJLFlushInterval()) , fifoSize(rm->getJlFifoSize()) - , fifoSizeLargeSideHj(rm->getHjFifoSizeLargeSide()) - , scanLbidReqLimit(rm->getJlScanLbidReqLimit()) - , scanLbidReqThreshold(rm->getJlScanLbidReqThreshold()) - , tempSaveSize(rm->getScTempSaveSize()) , logger(new Logger()) , traceFlags(0) - , tupleDLMaxSize(rm->getTwMaxSize()) - , tupleMaxBuckets(rm->getTwMaxBuckets()) , projectingTableOID(0) , isExeMgr(false) , trace(false) @@ -226,18 +220,8 @@ struct JobInfo JobStepVectorStack stack; uint32_t flushInterval; uint32_t fifoSize; - uint32_t fifoSizeLargeSideHj; - //...joblist does not use scanLbidReqLimit and SdanLbidReqThreshold. - //...They are actually used by pcolscan and pdictionaryscan, but - //...we have joblist get and report the values here since they - //...are global to the job. - uint32_t scanLbidReqLimit; - uint32_t scanLbidReqThreshold; - uint32_t tempSaveSize; SPJL logger; uint32_t traceFlags; - uint64_t tupleDLMaxSize; - uint32_t tupleMaxBuckets; SErrorInfo errorInfo; execplan::CalpontSystemCatalog::OID* projectingTableOID; // DeliveryWSDLs get a reference to this bool isExeMgr; @@ -372,7 +356,7 @@ struct JobInfo int64_t largeSideLimit; uint64_t partitionSize; bool isDML; - std::string timeZone; + long timeZone; // This is for tracking any dynamically allocated ParseTree objects // in simpleScalarFilterToParseTree() for later deletion in diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index a9c0c254b..441ee1ea4 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -132,7 +132,7 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo); /* This looks like an inefficient way to get NULL values. Much easier ways to do it. */ template -void valueNullNum(const CalpontSystemCatalog::ColType& ct, const string& timeZone, T& val) +void valueNullNum(const CalpontSystemCatalog::ColType& ct, const long timeZone, T& val) { T& n = val; bool pushWarning = false; @@ -274,7 +274,7 @@ void valueNullNum(const CalpontSystemCatalog::ColType& ct, const string& timeZon template void convertValueNum(const string& str, const CalpontSystemCatalog::ColType& ct, bool isNull, uint8_t& rf, - const string& timeZone, T& v) + const long timeZone, T& v) { if (str.size() == 0 || isNull) { @@ -3446,7 +3446,6 @@ void JLF_ExecPlanToJobList::addJobSteps(JobStepVector& nsv, JobInfo& jobInfo, bo } } // namespace joblist -// vim:ts=4 sw=4: #ifdef __clang__ #pragma clang diagnostic pop diff --git a/dbcon/joblist/jlf_graphics.cpp b/dbcon/joblist/jlf_graphics.cpp index 7365e1c4e..348fb85d6 100644 --- a/dbcon/joblist/jlf_graphics.cpp +++ b/dbcon/joblist/jlf_graphics.cpp @@ -418,7 +418,6 @@ ostream& writeDotCmds(ostream& dotFile, const JobStepVector& query, const JobSte } // end namespace jlf_graphics -// vim:ts=4 sw=4 syntax=cpp: #ifdef __clang__ #pragma clang diagnostic pop diff --git a/dbcon/joblist/jlf_subquery.cpp b/dbcon/joblist/jlf_subquery.cpp index 5b612d4ae..0ea66ac0f 100644 --- a/dbcon/joblist/jlf_subquery.cpp +++ b/dbcon/joblist/jlf_subquery.cpp @@ -64,7 +64,7 @@ using namespace joblist; namespace { -void getColumnValue(ConstantColumn** cc, uint64_t i, const Row& row, const string& timeZone) +void getColumnValue(ConstantColumn** cc, uint64_t i, const Row& row, const long timeZone) { ostringstream oss; int64_t data = 0; @@ -875,4 +875,3 @@ SJSTEP doUnionSub(CalpontExecutionPlan* ep, JobInfo& jobInfo) } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/jlf_tuplejoblist.cpp b/dbcon/joblist/jlf_tuplejoblist.cpp index 7678d2788..33a1f0104 100644 --- a/dbcon/joblist/jlf_tuplejoblist.cpp +++ b/dbcon/joblist/jlf_tuplejoblist.cpp @@ -1140,9 +1140,10 @@ bool combineJobStepsByTable(TableInfoMap::iterator& mit, JobInfo& jobInfo) for (unsigned i = 0; i < numOfStepsAddToBps; i++) { - bps->setBPP((it + i)->get()); + auto pp = (it + i)->get(); + bps->setBPP(pp); bps->setStepCount(); - bps->setLastTupleId((it + i)->get()->tupleId()); + bps->setLastTupleId(pp->tupleId()); } it += itInc; @@ -4559,7 +4560,6 @@ SJSTEP unionQueries(JobStepVector& queries, uint64_t distinctUnionNum, JobInfo& } } // namespace joblist -// vim:ts=4 sw=4: #ifdef __clang__ #pragma clang diagnostic pop diff --git a/dbcon/joblist/joblist.cpp b/dbcon/joblist/joblist.cpp index fd58bde04..7adc6a980 100644 --- a/dbcon/joblist/joblist.cpp +++ b/dbcon/joblist/joblist.cpp @@ -1233,4 +1233,3 @@ void TupleJobList::abort() #pragma clang diagnostic pop #endif -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/joblist.h b/dbcon/joblist/joblist.h index 1ccaa6dd7..24df23beb 100644 --- a/dbcon/joblist/joblist.h +++ b/dbcon/joblist/joblist.h @@ -260,4 +260,3 @@ typedef boost::shared_ptr STJLP; #undef EXPORT -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/joblistfactory.cpp b/dbcon/joblist/joblistfactory.cpp index 06bdf5920..26809d394 100644 --- a/dbcon/joblist/joblistfactory.cpp +++ b/dbcon/joblist/joblistfactory.cpp @@ -2097,9 +2097,7 @@ SJLP makeJobList_(CalpontExecutionPlan* cplan, ResourceManager* rm, bool isExeMg oss << endl; oss << endl << "job parms: " << endl; oss << "maxBuckets = " << jobInfo.maxBuckets << ", maxElems = " << jobInfo.maxElems - << ", flushInterval = " << jobInfo.flushInterval << ", fifoSize = " << jobInfo.fifoSize - << ", ScanLimit/Threshold = " << jobInfo.scanLbidReqLimit << "/" << jobInfo.scanLbidReqThreshold - << endl; + << ", flushInterval = " << jobInfo.flushInterval << ", fifoSize = " << jobInfo.fifoSize << endl; oss << "UUID: " << jobInfo.uuid << endl; oss << endl << "job filter steps: " << endl; ostream_iterator oIter(oss, "\n"); @@ -2316,7 +2314,6 @@ SJLP JobListFactory::makeJobList(CalpontExecutionPlan* cplan, ResourceManager* r } } // namespace joblist -// vim:ts=4 sw=4: #ifdef __clang__ #pragma clang diagnostic pop diff --git a/dbcon/joblist/jobstep.cpp b/dbcon/joblist/jobstep.cpp index 36d13f35e..8f42dff10 100644 --- a/dbcon/joblist/jobstep.cpp +++ b/dbcon/joblist/jobstep.cpp @@ -244,4 +244,3 @@ void JobStep::handleException(std::exception_ptr e, const int errorCode, const u } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/jobstep.h b/dbcon/joblist/jobstep.h index d9e95992e..28bd4f994 100644 --- a/dbcon/joblist/jobstep.h +++ b/dbcon/joblist/jobstep.h @@ -417,11 +417,11 @@ class JobStep fOnClauseFilter = b; } - void timeZone(const std::string& timezone) + void timeZone(const long timezone) { fTimeZone = timezone; } - const std::string timeZone() const + long timeZone() const { return fTimeZone; } @@ -479,7 +479,7 @@ class JobStep bool fDelivery; bool fOnClauseFilter; volatile bool fDie; - volatile uint32_t fWaitToRunStepCnt; + uint32_t fWaitToRunStepCnt; std::string fExtendedInfo; std::string fMiniInfo; @@ -496,7 +496,7 @@ class JobStep uint64_t fProgress; int64_t fStartTime; int64_t fLastStepTeleTime; - std::string fTimeZone; + long fTimeZone; private: static boost::mutex fLogMutex; @@ -566,4 +566,3 @@ typedef boost::shared_ptr SJSTEP; } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index 26080f0c2..34a5f23d5 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -338,8 +338,8 @@ int LBIDList::getMinMaxFromEntries(T& min, T& max, int32_t& seq, int64_t lbid, } template -void LBIDList::UpdateMinMax(T min, T max, int64_t lbid, const CalpontSystemCatalog::ColType& type, - bool validData) +void LBIDList::UpdateMinMax(T min, T max, int64_t lbid, bool dictScan, + const CalpontSystemCatalog::ColType& type, bool validData) { MinMaxPartition* mmp = NULL; #ifdef DEBUG @@ -372,18 +372,20 @@ void LBIDList::UpdateMinMax(T min, T max, int64_t lbid, const CalpontSystemCatal if (mmp->isValid == BRM::CP_INVALID) { - if (datatypes::isCharType(type.colDataType)) + if (!dictScan && datatypes::isCharType(type.colDataType)) { datatypes::Charset cs(const_cast(type).getCharset()); if (datatypes::TCharShort::strnncollsp(cs, min, mmp->min, type.colWidth) < 0 || - mmp->min == numeric_limits::max()) + // WIP + static_cast(mmp->min) == numeric_limits::max()) mmp->min = min; if (datatypes::TCharShort::strnncollsp(cs, max, mmp->max, type.colWidth) > 0 || - mmp->max == numeric_limits::min()) + // WIP + static_cast(mmp->max) == numeric_limits::min()) mmp->max = max; } - else if (datatypes::isUnsigned(type.colDataType)) + else if (dictScan || datatypes::isUnsigned(type.colDataType)) { if (static_cast(min) < static_cast(mmp->min)) mmp->min = min; @@ -526,7 +528,7 @@ bool LBIDList::CasualPartitionDataType(const CalpontSystemCatalog::ColDataType t case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: return size < 8; + case CalpontSystemCatalog::TEXT: return size <= 8; case CalpontSystemCatalog::TINYINT: case CalpontSystemCatalog::SMALLINT: @@ -695,15 +697,19 @@ bool LBIDList::checkRangeOverlap(T min, T max, T tmin, T tmax, bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, const messageqcpp::ByteStream* bs, const uint16_t NOPS, - const execplan::CalpontSystemCatalog::ColType& ct, const uint8_t BOP) + const execplan::CalpontSystemCatalog::ColType& ct, const uint8_t BOP, + bool isDict) { int length = bs->length(), pos = 0; const char* MsgDataPtr = (const char*)bs->buf(); bool scan = true; int64_t value = 0; int128_t bigValue = 0; - bool bIsUnsigned = datatypes::isUnsigned(ct.colDataType); - bool bIsChar = datatypes::isCharType(ct.colDataType); + // MCOL-4580 - related. + // We definitely can compute isDict flag themselves here, as we have column type and width. + // But, we may also use already computed isDict flags in the steps, available with getIsDict() method.. + bool bIsUnsigned = isDict || datatypes::isUnsigned(ct.colDataType); + bool bIsChar = !isDict && datatypes::isCharType(ct.colDataType); for (int i = 0; i < NOPS; i++) { @@ -800,9 +806,12 @@ bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, { continue; } - else if (execplan::isNull(value, ct)) // This will work even if the data column is unsigned. + else { - continue; + if (execplan::isNull(value, ct)) // This will work even if the data column is unsigned. + { + continue; + } } if (bIsChar) @@ -898,11 +907,11 @@ template bool LBIDList::GetMinMax(int64_t* min, int64_t* max, int64_t* const tr1::unordered_map& entries, execplan::CalpontSystemCatalog::ColDataType colDataType); -template void LBIDList::UpdateMinMax(int128_t min, int128_t max, int64_t lbid, +template void LBIDList::UpdateMinMax(int128_t min, int128_t max, int64_t lbid, bool dictScan, const execplan::CalpontSystemCatalog::ColType& type, bool validData = true); -template void LBIDList::UpdateMinMax(int64_t min, int64_t max, int64_t lbid, +template void LBIDList::UpdateMinMax(int64_t min, int64_t max, int64_t lbid, bool dictScan, const execplan::CalpontSystemCatalog::ColType& type, bool validData = true); @@ -920,4 +929,3 @@ template bool LBIDList::checkRangeOverlap(int64_t min, int64_t max, int } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/lbidlist.h b/dbcon/joblist/lbidlist.h index 1e49db07f..bf7c6310b 100644 --- a/dbcon/joblist/lbidlist.h +++ b/dbcon/joblist/lbidlist.h @@ -98,8 +98,8 @@ class LBIDList execplan::CalpontSystemCatalog::ColDataType type); template - void UpdateMinMax(T min, T max, int64_t lbid, const execplan::CalpontSystemCatalog::ColType& type, - bool validData = true); + void UpdateMinMax(T min, T max, int64_t lbid, bool dictScan, + const execplan::CalpontSystemCatalog::ColType& type, bool validData = true); void UpdateAllPartitionInfo(const execplan::CalpontSystemCatalog::ColType& colType); @@ -107,7 +107,8 @@ class LBIDList bool CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, const messageqcpp::ByteStream* MsgDataPtr, const uint16_t NOPS, - const execplan::CalpontSystemCatalog::ColType& ct, const uint8_t BOP); + const execplan::CalpontSystemCatalog::ColType& ct, const uint8_t BOP, + bool isDict); template bool checkSingleValue(T min, T max, T value, const execplan::CalpontSystemCatalog::ColType& type); diff --git a/dbcon/joblist/limitedorderby.cpp b/dbcon/joblist/limitedorderby.cpp index 3aee6fc02..f4e573bd7 100644 --- a/dbcon/joblist/limitedorderby.cpp +++ b/dbcon/joblist/limitedorderby.cpp @@ -303,4 +303,3 @@ const string LimitedOrderBy::toString() const } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/passthrucommand-jl.cpp b/dbcon/joblist/passthrucommand-jl.cpp index b512e37b3..84e63f9c5 100644 --- a/dbcon/joblist/passthrucommand-jl.cpp +++ b/dbcon/joblist/passthrucommand-jl.cpp @@ -110,4 +110,3 @@ uint16_t PassThruCommandJL::getWidth() } }; // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/pcolscan.cpp b/dbcon/joblist/pcolscan.cpp index 53dfbcfb0..2a3e64fba 100644 --- a/dbcon/joblist/pcolscan.cpp +++ b/dbcon/joblist/pcolscan.cpp @@ -53,64 +53,6 @@ using namespace execplan; //#define DEBUG 1 //#define DEBUG2 1 -namespace -{ -//// const uint32_t defaultScanLbidReqLimit = 10000; -//// const uint32_t defaultScanLbidReqThreshold = 5000; -// -// struct pColScanStepPrimitive -//{ -// pColScanStepPrimitive(pColScanStep* pColScanStep) : fPColScanStep(pColScanStep) -// {} -// pColScanStep *fPColScanStep; -// void operator()() -// { -// try -// { -// fPColScanStep->sendPrimitiveMessages(); -// } -// catch(std::exception& re) -// { -// string msg = re.what(); -// cerr << "pColScanStep: send thread threw an exception: " << msg << endl; -// -// //Whoa! is this really what we want to do? It's not clear that any good can be had by -// //sticking around, but this seems drastic... -// if (msg.find("there are no primitive processors") != string::npos) -// { -// SPJL logger = fPColScanStep->logger(); -// logger->logMessage(LOG_TYPE_CRITICAL, LogNoPrimProcs, Message::Args(), -//LoggingID(5)); exit(1); -// } -// } -// } -//}; -// -// struct pColScanStepAggregater -//{ -// pColScanStepAggregater(pColScanStep* pColScanStep, uint64_t index) : -// fPColScanStepCol(pColScanStep), fThreadId(index) -// {} -// pColScanStep *fPColScanStepCol; -// uint64_t fThreadId; -// -// void operator()() -// { -// try -// { -// fPColScanStepCol->receivePrimitiveMessages(fThreadId); -// } -// catch(std::exception& re) -// { -// cerr << fPColScanStepCol->toString() << ": receive thread threw an exception: " << -//re.what() << endl; -// } -// } -//}; -// - -} // namespace - namespace joblist { pColScanStep::pColScanStep(CalpontSystemCatalog::OID o, CalpontSystemCatalog::OID t, @@ -118,20 +60,11 @@ pColScanStep::pColScanStep(CalpontSystemCatalog::OID o, CalpontSystemCatalog::OI : JobStep(jobInfo) , fRm(jobInfo.rm) , fMsgHeader() - , fNumThreads(fRm->getJlNumScanReceiveThreads()) , fFilterCount(0) , fOid(o) , fTableOid(t) , fColType(ct) , fBOP(BOP_OR) - , sentCount(0) - , recvCount(0) - , fScanLbidReqLimit(fRm->getJlScanLbidReqLimit()) - , fScanLbidReqThreshold(fRm->getJlScanLbidReqThreshold()) - , fStopSending(false) - , fSingleThread(false) - , fPhysicalIO(0) - , fCacheIO(0) , fNumBlksSkipped(0) , fMsgBytesIn(0) , fMsgBytesOut(0) @@ -234,332 +167,6 @@ pColScanStep::pColScanStep(CalpontSystemCatalog::OID o, CalpontSystemCatalog::OI throw runtime_error("pColScan: Block size and column width must be a power of 2"); } -pColScanStep::~pColScanStep() -{ - // pthread_mutex_destroy(&mutex); - // pthread_mutex_destroy(&dlMutex); - // pthread_mutex_destroy(&cpMutex); - // pthread_cond_destroy(&condvar); - // pthread_cond_destroy(&condvarWakeupProducer); - // delete lbidList; - // delete [] fProducerThread; - // if (fDec) - // fDec->removeQueue(uniqueID); // in case it gets aborted -} - -//------------------------------------------------------------------------------ -// Initialize configurable parameters -//------------------------------------------------------------------------------ -void pColScanStep::initializeConfigParms() -{ - // const string section ( "JobList" ); - // const string sendLimitName ( "ScanLbidReqLimit" ); - // const string sendThresholdName ( "ScanLbidReqThreshold" ); - // const string numReadThreadsName ( "NumScanReceiveThreads" ); - // Config* cf = Config::makeConfig(); - - // string strVal; - - //...Get the tuning parameters that throttle msgs sent to primproc - //...fScanLbidReqLimit puts a cap on how many LBID's we will request from - //... primproc, before pausing to let the consumer thread catch up. - //... Without this limit, there is a chance that PrimProc could flood - //... ExeMgr with thousands of messages that will consume massive - //... amounts of memory for a 100 gigabyte database. - //...fScanLbidReqThreshold is the level at which the number of outstanding - //... LBID reqs must fall below, before the producer can send more LBIDs. - // strVal = cf->getConfig(section, sendLimitName); - // if (strVal.size() > 0) - // fScanLbidReqLimit = static_cast(Config::uFromText(strVal)); - // - // strVal = cf->getConfig(section, sendThresholdName); - // if (strVal.size() > 0) - // fScanLbidReqThreshold = static_cast(Config::uFromText(strVal)); - // - // fNumThreads = 8; - // strVal = cf->getConfig(section, numReadThreadsName); - // if (strVal.size() > 0) - // fNumThreads = static_cast(Config::uFromText(strVal)); - - // fProducerThread = new SPTHD[fNumThreads]; -} - -void pColScanStep::startPrimitiveThread() -{ - // fConsumerThread.reset(new boost::thread(pColScanStepPrimitive(this))); -} - -void pColScanStep::startAggregationThread() -{ - // for (uint32_t i = 0; i < fNumThreads; i++) - // fProducerThread[i].reset(new boost::thread(pColScanStepAggregater(this, i))); -} - -void pColScanStep::run() -{ - // if (traceOn()) - // { - // syslogStartStep(16, // exemgr subsystem - // std::string("pColScanStep")); // step name - // } - // - // //"consume" input datalist. In this case, there is no IDL, we just send one primitive - // startPrimitiveThread(); - // //produce output datalist - // //Don't start this yet...see below - // //startAggregationThread(); -} - -void pColScanStep::join() -{ - // fConsumerThread->join(); - // if ( !fSingleThread ) { - // for (uint32_t i = 0; i < fNumThreads; i++) - // fProducerThread[i]->join(); - // } -} - -void pColScanStep::sendPrimitiveMessages() -{ - // //The presence of an input DL means we (probably) have a pDictionaryScan step feeding this scan step - // // a list of tokens to get the rids for. Convert the input tokens to a filter string. - // if (fInputJobStepAssociation.outSize() > 0) - // { - // addFilters(); - // if (fTableOid >= 3000) - // cout << toString() << endl; - // //If we got no input rids (as opposed to no input DL at all) then there were no matching rows from - // // the previous step, so this step should not return any rows either. This would be the case, for - // // instance, if P_NAME LIKE '%xxxx%' produced no signature matches. - // if (fFilterCount == 0) { - // rDoNothing=true; - // startAggregationThread(); - // return; - // } - // } - // - // startAggregationThread(); - // - // /* for all the blocks that need to be sent - // * build out a message for each block with the primitive message header filled - // * out and all the NOPS and BOP structures as well. - // * Then serialize it to a BytStream and then send it on its way - // */ - // - // LBIDRange_v::iterator it; - // uint64_t fbo; - // - // ISMPacketHeader ism; - // ism.Flags = planFlagsToPrimFlags(fTraceFlags); - // ism.Command=COL_BY_SCAN_RANGE; - // ism.Size = sizeof(ISMPacketHeader) + sizeof(ColByScanRangeRequestHeader) + fFilterString.length(); - // ism.Type=2; - // //bool firstWrite = true; - // - // //...Counter used to track the number of LBIDs we are requesting from - // //...primproc in the current set of msgs, till we reach fScanLbidReqLimit - // uint32_t runningLbidCount = 0; - // bool exitLoop = false; - // const bool ignoreCP = ((fTraceFlags & CalpontSelectExecutionPlan::IGNORE_CP) != 0); - // - // for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) - // { - // BRM::LBID_t lbid = (*it).start; - // - // fbo = getFBO(lbid); - // if (hwm < fbo) - // continue; - // - // if (fOid >= 3000 && lbidList->CasualPartitionDataType(fColType.colDataType, fColType.colWidth) ) - // { - // int64_t Min=0; - // int64_t Max=0; - // int64_t SeqNum=0; - // bool MinMaxValid=true; - // bool cpPredicate=true; - // - // // can we consolidate these crit sections? - // cpMutex.lock(); //pthread_mutex_lock(&cpMutex); - // MinMaxValid = lbidList->GetMinMax(Min, Max, SeqNum, lbid, 0); - // - // if (MinMaxValid) - // { - // cpPredicate=lbidList->CasualPartitionPredicate(Min, - // Max, - // &fFilterString, - // fFilterCount, - // fColType, - // fBOP) || - //ignoreCP; - // } - // cpMutex.unlock(); //pthread_mutex_unlock(&cpMutex); - // - // if (cpPredicate==false){ //don't scan this extent - //#ifdef DEBUG - // cout << "Scan Skip " << lbid << endl; - //#endif - // //...Track the number of LBIDs we skip due to Casual Partioning. - // //...We use the same equation we use to initialize remainingLbids - // //...in the code that follows down below this. - // fNumBlksSkipped += ( (hwm > (fbo + it->size - 1)) ? - // (it->size) : (hwm - fbo + 1) ); - // continue; - // } - //#ifdef DEBUG - // else - // cout << "Scan " << lbid << endl; - //#endif - // - // } - // - // LBID_t msgLbidStart = it->start; - // uint32_t remainingLbids = - // ( (hwm > (fbo + it->size - 1)) ? (it->size) : (hwm - fbo + 1) ); - // uint32_t msgLbidCount = 0; - // - // while ( remainingLbids > 0 ) - // { - // //...Break up this range of LBIDs when we reach the msg size - // //...limit for one request (fScanLbidReqLimit) - // if ( (runningLbidCount + remainingLbids) >= fScanLbidReqLimit ) - // { - // msgLbidCount = fScanLbidReqLimit - runningLbidCount; - // sendAPrimitiveMessage(ism, msgLbidStart, msgLbidCount ); - // //...Wait for the consuming thread to catch up if our - // //...backlog of work is >= the allowable threshold limit - // - // mutex.lock(); //pthread_mutex_lock(&mutex); - // sentCount += msgLbidCount; - // if (recvWaiting) - // condvar.notify_all(); //pthread_cond_broadcast(&condvar); //signal consumer to - //resume - // - // while ( ((sentCount - recvCount) >= fScanLbidReqThreshold) - // && !fStopSending ) - // { - // sendWaiting = true; - //#ifdef DEBUG2 - // if (fOid >= 3000) - // cout << "pColScanStep producer WAITING: " << - // "st:" << fStepId << - // "; sentCount-" << sentCount << - // "; recvCount-" << recvCount << - // "; threshold-" << fScanLbidReqThreshold << endl; - //#endif - // condvarWakeupProducer.wait(mutex); //pthread_cond_wait ( &condvarWakeupProducer, &mutex - //); #ifdef DEBUG2 if (fOid >= 3000) cout << "pColScanStep producer RESUMING: " << "st:" << fStepId << endl; - //#endif - // sendWaiting = false; - // } - // - // //...Set flag to quit if consumer thread tells us to - // if (fStopSending) - // exitLoop = true; - // - // mutex.unlock(); //pthread_mutex_unlock(&mutex); - // - // runningLbidCount = 0; - // } - // else - // { - // msgLbidCount = remainingLbids; - // - // sendAPrimitiveMessage(ism, msgLbidStart, msgLbidCount ); - // mutex.lock(); //pthread_mutex_lock(&mutex); - // sentCount += msgLbidCount; - // if (recvWaiting) - // condvar.notify_all(); //pthread_cond_broadcast(&condvar); //signal consumer to - //resume - // - // //...Set flag to quit if consumer thread tells us to - // if (fStopSending) - // exitLoop = true; - // - // mutex.unlock(); //pthread_mutex_unlock(&mutex); - // - // runningLbidCount += msgLbidCount; - // } - // - // //...If consuming thread has quit, then we should do the same. - // //...This can happen if consuming thread receives empty ByteStream - // if (exitLoop) - // break; - // - // remainingLbids -= msgLbidCount; - // msgLbidStart += msgLbidCount; - // } - // - // if (exitLoop) - // break; - // - // } // end of loop through LBID ranges to be requested from primproc - // - // mutex.lock(); //pthread_mutex_lock(&mutex); - // finishedSending = true; - // if (recvWaiting) - // condvar.notify_all(); //pthread_cond_broadcast(&condvar); - // mutex.unlock(); //pthread_mutex_unlock(&mutex); - //// cerr << "send side exiting" << endl; - // - //#ifdef DEBUG2 - // if (fOid >= 3000) - // { - // time_t t = time(0); - // char timeString[50]; - // ctime_r(&t, timeString); - // timeString[strlen(timeString)-1 ] = '\0'; - // cout << "pColScanStep Finished sending primitives for: " << - // fOid << " at " << timeString << endl; - // } - //#endif - // -} - -//------------------------------------------------------------------------------ -// Construct and send a single primitive message to primproc -//------------------------------------------------------------------------------ -void pColScanStep::sendAPrimitiveMessage(ISMPacketHeader& ism, BRM::LBID_t msgLbidStart, - uint32_t msgLbidCount) -{ - // ByteStream bs; - // - // bs.load(reinterpret_cast(&ism), sizeof(ism)); - // - // fMsgHeader.LBID = msgLbidStart; - // fMsgHeader.DataSize = fColType.colWidth; - // fMsgHeader.DataType = fColType.colDataType; - // fMsgHeader.CompType = fColType.compressionType; - // if (fFilterCount > 0) - // fMsgHeader.OutputType = 3; // pairs - // else - // fMsgHeader.OutputType = OT_DATAVALUE; - // fMsgHeader.BOP = fBOP; - // fMsgHeader.NOPS = fFilterCount; - // fMsgHeader.NVALS = 0; - // fMsgHeader.Count = msgLbidCount; - // fMsgHeader.Hdr.SessionID = fSessionId; - // //fMsgHeader.Hdr.StatementID = 0; - // fMsgHeader.Hdr.TransactionID = fTxnId; - // fMsgHeader.Hdr.VerID = fVerId; - // fMsgHeader.Hdr.StepID = fStepId; - // fMsgHeader.Hdr.UniqueID = uniqueID; - // - // bs.append(reinterpret_cast(&fMsgHeader), - // sizeof(fMsgHeader)); - // bs += fFilterString; - // - //#ifdef DEBUG2 - // if (fOid >= 3000) - // cout << "pColScanStep producer st: " << fStepId << - // ": sending req for lbid start " << msgLbidStart << - // "; lbid count " << msgLbidCount << endl; - //#endif - // - // fMsgBytesOut += bs.lengthWithHdrOverhead(); - // fDec->write(bs); - // fMsgsToPm++; -} - struct CPInfo { CPInfo(int64_t MIN, int64_t MAX, uint64_t l) : min(MIN), max(MAX), LBID(l){}; @@ -568,327 +175,6 @@ struct CPInfo uint64_t LBID; }; -void pColScanStep::receivePrimitiveMessages(uint64_t tid) -{ - // AnyDataListSPtr dl = fOutputJobStepAssociation.outAt(0); - // DataList_t* dlp = dl->dataList(); - // FifoDataList *fifo = dl->fifoDL(); - // BucketDL *bucket = dynamic_cast *>(dlp); - // ZDL *zdl = dynamic_cast *>(dlp); - // int64_t l_ridsReturned = 0; - // uint64_t l_physicalIO = 0, l_cachedIO = 0; - // uint64_t fbo; - // uint64_t ridBase; - // vector v; - // UintRowGroup rw; - // vector > bsv; - // uint32_t i, k, size, bsLength; - // bool lastThread = false; - // vector cpv; - // - // if (bucket || zdl) - // dlp->setMultipleProducers(true); - // - // mutex.lock(); //pthread_mutex_lock(&mutex); - // - // // count the LBIDs - // for (; !rDoNothing; ) { - // - // // sync with the send side - // while (!finishedSending && sentCount == recvCount) { - // recvWaiting++; - // condvar.wait(mutex); //pthread_cond_wait(&condvar, &mutex); - // recvWaiting--; - // } - // if (sentCount == recvCount && finishedSending) { - //// cout << "done recving" << endl; - // break; - // } - // - // fDec->read_some(uniqueID, fNumThreads, bsv); - // for (unsigned int jj=0; jjlengthWithHdrOverhead(); - // } - // - // - // size = bsv.size(); - // - // if (size == 0) { - // /* XXXPAT: Need to give other threads a chance to update recvCount. - // As of 2/25/08, each BS contains multiple responses. With the current - // protocol, the exact # isn't known until they're processed. Threads - // waiting for more input will have to busy wait on either recvCount - // being updated or input. It's only an issue at the tail end of the - // responses, and probably doesn't matter then either. */ - // - // mutex.unlock(); //pthread_mutex_unlock(&mutex); - // usleep(1000); // 1ms Good? - // mutex.lock(); //pthread_mutex_lock(&mutex); - // continue; - // } - // - // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) - // dlTimes.setFirstReadTime(); - // if (fOid>=3000) dlTimes.setLastReadTime(); - // - //// cerr << "got a response of " << size << " msgs\n"; - // - // mutex.unlock(); //pthread_mutex_unlock(&mutex); - // - // uint32_t msgCount = 0; - // for (i = 0; i < size; i++) { - // const ByteStream::byte* bsp = bsv[i]->buf(); - // - // bsLength = bsv[i]->length(); - // k = 0; - // while (k < bsLength) { - // ++msgCount; - //// cout << "got msg " << msgCount << " k = " << k << endl; - // k += sizeof(ISMPacketHeader); - // const ColResultHeader* crh = reinterpret_cast(&bsp[k]); - // // get the ColumnResultHeader out of the bytestream - // k += sizeof(ColResultHeader); - // - // l_cachedIO += crh->CacheIO; - // l_physicalIO += crh->PhysicalIO; - // fbo = getFBO(crh->LBID); - // ridBase = fbo << rpbShift; - // - // for(int j = 0; j < crh->NVALS; j++) - // { - // uint64_t dv; - // uint64_t rid; - // - // if (crh->OutputType == OT_DATAVALUE) { - // if (isEmptyVal(&bsp[k])) { - // k += fColType.colWidth; - // continue; - // } - // rid = j + ridBase; - // } - // else { - // rid = *((const uint16_t *) &bsp[k]) + ridBase; - // k += sizeof(uint16_t); - // } - // - // switch (fColType.colWidth) { - // case 8: dv = *((const uint64_t *) &bsp[k]); k += 8; break; - // case 4: dv = *((const uint32_t *) &bsp[k]); k += 4; break; - // case 2: dv = *((const uint16_t *) &bsp[k]); k += 2; break; - // case 1: dv = *((const uint8_t *) &bsp[k]); ++k; break; - // default: - // throw runtime_error("pColStep: invalid column - //width!"); - // } - // - // v.push_back(ElementType(rid, dv)); - // ++l_ridsReturned; - //#ifdef DEBUG - //// if (fOid >=3000) - //// cout << " -- inserting <" << rid << ", " << dv << ">" << - ///endl; - //#endif - // // per row operations... - // } // for - // - // // per block operations... - // - //#ifdef DEBUG - // cout << "recvPrimMsgs Oid " << fOid - // << " valid " << crh->ValidMinMax - // << " LBID " << crh->LBID - // << " Mn/Mx " << crh->Min << "/" << crh->Max - // << " nvals " << crh->NVALS - // << "/" << l_ridsReturned << endl; - //#endif - // if (fOid >= 3000 && crh->ValidMinMax) - // cpv.push_back(CPInfo(crh->Min, crh->Max, crh->LBID)); - // } - // // per ByteStream operations... - // } - // // per read operations.... - // - // if (bucket) { - // if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) - // dlTimes.setFirstInsertTime(); - // - // bucket->insert(v); - // } - // else if (zdl) { - // zdl->insert(v); - // } - // else { - // size = v.size(); - // if (size>0) - // if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) - // dlTimes.setFirstInsertTime(); - // - // dlMutex.lock(); //pthread_mutex_lock(&dlMutex); - // for (i = 0; i < size; ++i) { - // rw.et[rw.count++] = v[i]; - // if (rw.count == rw.ElementsPerGroup) - // { - // fifo->insert(rw); - // rw.count = 0; - // } - // } - // - // if (rw.count > 0) - // { - // fifo->insert(rw); - // rw.count = 0; - // } - // - // dlMutex.unlock(); //pthread_mutex_unlock(&dlMutex); - // } - // v.clear(); - // - // size = cpv.size(); - // if (size > 0) { - // cpMutex.lock(); //pthread_mutex_lock(&cpMutex); - // for (i = 0; i < size; i++) { - // CPInfo *cpi = &(cpv[i]); - // lbidList->UpdateMinMax(cpi->min, cpi->max, cpi->LBID, fColType.colDataType); - // } - // cpMutex.unlock(); //pthread_mutex_unlock(&cpMutex); - // cpv.clear(); - // } - // - // mutex.lock(); //pthread_mutex_lock(&mutex); - // recvCount += msgCount; - // //...If producer is waiting, and we have gone below our threshold value, - // //...then we signal the producer to request more data from primproc - // if ( (sendWaiting) && ( (sentCount - recvCount) < fScanLbidReqThreshold ) ) - // { - //#ifdef DEBUG2 - // if (fOid >= 3000) - // cout << "pColScanStep consumer signaling producer for more data: "<< - // "st:" << fStepId << - // "; sentCount-" << sentCount << - // "; recvCount-" << recvCount << - // "; threshold-" << fScanLbidReqThreshold << endl; - //#endif - // condvarWakeupProducer.notify_one(); //pthread_cond_signal(&condvarWakeupProducer); - // } - // } // end of loop to read LBID responses from primproc - // - // fPhysicalIO += l_physicalIO; - // fCacheIO += l_cachedIO; - // ridsReturned += l_ridsReturned; - //// cerr << "out of the main loop " << recvExited << endl; - // if (++recvExited == fNumThreads) { - // //...Casual partitioning could cause us to do no processing. In that - // //...case these time stamps did not get set. So we set them here. - // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { - // dlTimes.setFirstReadTime(); - // dlTimes.setLastReadTime(); - // dlTimes.setFirstInsertTime(); - // } - // if (fOid>=3000) dlTimes.setEndOfInputTime(); - // - // //@bug 699: Reset StepMsgQueue - // fDec->removeQueue(uniqueID); - // - // if (fifo) - // fifo->endOfInput(); - // else - // dlp->endOfInput(); - // lastThread = true; - // } - // - // mutex.unlock(); //pthread_mutex_unlock(&mutex); - // - // if (fTableOid >= 3000 && lastThread) - // { - // //...Construct timestamp using ctime_r() instead of ctime() not - // //...necessarily due to re-entrancy, but because we want to strip - // //...the newline ('\n') off the end of the formatted string. - // time_t t = time(0); - // char timeString[50]; - // ctime_r(&t, timeString); - // timeString[strlen(timeString)-1 ] = '\0'; - // - // FifoDataList* pFifo = 0; - // uint64_t totalBlockedReadCount = 0; - // uint64_t totalBlockedWriteCount = 0; - // - // //...Sum up the blocked FIFO reads for all input associations - // size_t inDlCnt = fInputJobStepAssociation.outSize(); - // for (size_t iDataList=0; iDataListfifoDL(); - // if (pFifo) - // { - // totalBlockedReadCount += pFifo->blockedReadCount(); - // } - // } - // - // //...Sum up the blocked FIFO writes for all output associations - // size_t outDlCnt = fOutputJobStepAssociation.outSize(); - // for (size_t iDataList=0; iDataListfifoDL(); - // if (pFifo) - // { - // totalBlockedWriteCount += pFifo->blockedWriteCount(); - // } - // } - // - // //...Roundoff inbound msg byte count to nearest KB for display; - // //...no need to do so for outbound, because it should be small. - // uint64_t msgBytesInKB = fMsgBytesIn >> 10; - // if (fMsgBytesIn & 512) - // msgBytesInKB++; - // // @bug 807 - // if (fifo) - // fifo->totalSize(ridsReturned); - // - // if (traceOn()) - // { - // //...Print job step completion information - // ostringstream logStr; - // logStr << "ses:" << fSessionId << - // " st: " << fStepId << " finished at " << - // timeString << "; PhyI/O-" << fPhysicalIO << "; CacheI/O-" << - // fCacheIO << "; MsgsSent-" << fMsgsToPm << "; MsgsRcvd-" << recvCount << - // "; BlockedFifoIn/Out-" << totalBlockedReadCount << - // "/" << totalBlockedWriteCount << - // "; output size-" << ridsReturned << endl << - // "\tPartitionBlocksEliminated-" << fNumBlksSkipped << - // "; MsgBytesIn-" << msgBytesInKB << "KB" << - // "; MsgBytesOut-" << fMsgBytesOut << "B" << endl << - // "\t1st read " << dlTimes.FirstReadTimeString() << - // "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" << - // JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(),dlTimes.FirstReadTime()) << - // "s" << endl; - // - // logEnd(logStr.str().c_str()); - // - // syslogReadBlockCounts(16, // exemgr subsystem - // fPhysicalIO, // # blocks read from disk - // fCacheIO, // # blocks read from cache - // fNumBlksSkipped); // # casual partition block hits - // syslogProcessingTimes(16, // exemgr subsystem - // dlTimes.FirstReadTime(), // first datalist read - // dlTimes.LastReadTime(), // last datalist read - // dlTimes.FirstInsertTime(), // first datalist write - // dlTimes.EndOfInputTime()); // last (endOfInput) datalist write - // syslogEndStep(16, // exemgr subsystem - // totalBlockedReadCount, // blocked datalist input - // totalBlockedWriteCount, // blocked datalist output - // fMsgBytesIn, // incoming msg byte count - // fMsgBytesOut); // outgoing msg byte count - // } - // - // } - // - // if (fOid >=3000 && lastThread) - // lbidList->UpdateAllPartitionInfo(); - // - //// cerr << "recv thread exiting" << endl; -} - void pColScanStep::addFilter(int8_t COP, float value) { fFilterString << (uint8_t)COP; @@ -937,22 +223,6 @@ void pColScanStep::addFilter(int8_t COP, int64_t value, uint8_t roundFlag) fFilterCount++; } -void pColScanStep::setBOP(int8_t B) -{ - fBOP = B; -} - -void pColScanStep::setSingleThread(bool b) -{ - fSingleThread = b; - fNumThreads = 1; -} - -void pColScanStep::setOutputType(int8_t OutputType) -{ - fOutputType = OutputType; -} - const string pColScanStep::toString() const { ostringstream oss; @@ -983,10 +253,7 @@ uint64_t pColScanStep::getFBO(uint64_t lbid) { lastLBID = extents[i].range.start + (extents[i].range.size << 10) - 1; - // lastLBID = extents[i].range.start + (extents[i].range.size * 1024) - 1; - // cerr << "start: " << extents[i].range.start << " end:" << lastLBID <= (uint64_t)extents[i].range.start && lbid <= lastLBID) - // return (lbid - extents[i].range.start) + (extentSize * i); return (lbid - extents[i].range.start) + (i << divShift); } @@ -996,7 +263,6 @@ uint64_t pColScanStep::getFBO(uint64_t lbid) pColScanStep::pColScanStep(const pColStep& rhs) : JobStep(rhs), fRm(rhs.resourceManager()), fMsgHeader() { - fNumThreads = fRm->getJlNumScanReceiveThreads(); fFilterCount = rhs.filterCount(); fFilterString = rhs.filterString(); isFilterFeeder = rhs.getFeederFlag(); @@ -1005,14 +271,6 @@ pColScanStep::pColScanStep(const pColStep& rhs) : JobStep(rhs), fRm(rhs.resource fColType = rhs.colType(); fBOP = rhs.BOP(); fIsDict = rhs.isDictCol(); - sentCount = 0; - recvCount = 0; - fScanLbidReqLimit = fRm->getJlScanLbidReqLimit(); - fScanLbidReqThreshold = fRm->getJlScanLbidReqThreshold(); - fStopSending = false; - fSingleThread = false; - fPhysicalIO = 0; - fCacheIO = 0; fNumBlksSkipped = 0; fMsgBytesIn = 0; fMsgBytesOut = 0; @@ -1040,11 +298,6 @@ pColScanStep::pColScanStep(const pColStep& rhs) : JobStep(rhs), fRm(rhs.resource numExtents = extents.size(); extentSize = (fRm->getExtentRows() * fColType.colWidth) / BLOCK_SIZE; lbidList = rhs.lbidList; - // pthread_mutex_init(&mutex, NULL); - // pthread_mutex_init(&dlMutex, NULL); - // pthread_mutex_init(&cpMutex, NULL); - // pthread_cond_init(&condvar, NULL); - // pthread_cond_init(&condvarWakeupProducer, NULL); finishedSending = sendWaiting = rDoNothing = false; recvWaiting = 0; recvExited = 0; @@ -1054,12 +307,7 @@ pColScanStep::pColScanStep(const pColStep& rhs) : JobStep(rhs), fRm(rhs.resource rpbShift = rhs.rpbShift; divShift = rhs.divShift; - // initializeConfigParms ( ); fTraceFlags = rhs.fTraceFlags; - // uniqueID = UniqueNumberGenerator::instance()->getUnique32(); - // if (fDec) - // fDec->addQueue(uniqueID); - // fProducerThread = new SPTHD[fNumThreads]; } void pColScanStep::addFilters() @@ -1224,4 +472,3 @@ void pColScanStep::appendFilter(const std::vector& fs) } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/pcolstep.cpp b/dbcon/joblist/pcolstep.cpp index eefaf02e1..3a314a7d6 100644 --- a/dbcon/joblist/pcolstep.cpp +++ b/dbcon/joblist/pcolstep.cpp @@ -58,47 +58,6 @@ using namespace BRM; namespace joblist { -#if 0 -//const uint32_t defaultProjectBlockReqLimit = 32768; -//const uint32_t defaultProjectBlockReqThreshold = 16384; -struct pColStepPrimitive -{ - pColStepPrimitive(pColStep* pColStep) : fPColStep(pColStep) - {} - pColStep* fPColStep; - void operator()() - { - try - { - fPColStep->sendPrimitiveMessages(); - } - catch (exception& re) - { - cerr << "pColStep: send thread threw an exception: " << re.what() << - "\t" << this << endl; - } - } -}; - -struct pColStepAggregator -{ - pColStepAggregator(pColStep* pColStep) : fPColStepCol(pColStep) - {} - pColStep* fPColStepCol; - void operator()() - { - try - { - fPColStepCol->receivePrimitiveMessages(); - } - catch (exception& re) - { - cerr << fPColStepCol->toString() << ": recv thread threw an exception: " << re.what() << endl; - } - } -}; -#endif - pColStep::pColStep(CalpontSystemCatalog::OID o, CalpontSystemCatalog::OID t, const CalpontSystemCatalog::ColType& ct, const JobInfo& jobInfo) : JobStep(jobInfo) @@ -117,14 +76,8 @@ pColStep::pColStep(CalpontSystemCatalog::OID o, CalpontSystemCatalog::OID t, , fIsDict(false) , isEM(jobInfo.isExeMgr) , ridCount(0) - , fFlushInterval(jobInfo.flushInterval) , fSwallowRows(false) - , fProjectBlockReqLimit(fRm->getJlProjectBlockReqLimit()) - , fProjectBlockReqThreshold(fRm->getJlProjectBlockReqThreshold()) - , fStopSending(false) , isFilterFeeder(false) - , fPhysicalIO(0) - , fCacheIO(0) , fNumBlksSkipped(0) , fMsgBytesIn(0) , fMsgBytesOut(0) @@ -135,11 +88,6 @@ pColStep::pColStep(CalpontSystemCatalog::OID o, CalpontSystemCatalog::OID t, int err, i; uint32_t mask; - if (fFlushInterval == 0 || !isEM) - fOutputType = OT_BOTH; - else - fOutputType = OT_TOKEN; - if (fOid < 1000) throw runtime_error("pColStep: invalid column"); @@ -279,15 +227,7 @@ pColStep::pColStep(const pColScanStep& rhs) , recvWaiting(false) , fIsDict(rhs.isDictCol()) , ridCount(0) - , - // Per Cindy, it's save to put fFlushInterval to be 0 - fFlushInterval(0) , fSwallowRows(false) - , fProjectBlockReqLimit(fRm->getJlProjectBlockReqLimit()) - , fProjectBlockReqThreshold(fRm->getJlProjectBlockReqThreshold()) - , fStopSending(false) - , fPhysicalIO(0) - , fCacheIO(0) , fNumBlksSkipped(0) , fMsgBytesIn(0) , fMsgBytesOut(0) @@ -390,15 +330,7 @@ pColStep::pColStep(const PassThruStep& rhs) , recvWaiting(false) , fIsDict(rhs.isDictCol()) , ridCount(0) - , - // Per Cindy, it's save to put fFlushInterval to be 0 - fFlushInterval(0) , fSwallowRows(false) - , fProjectBlockReqLimit(fRm->getJlProjectBlockReqLimit()) - , fProjectBlockReqThreshold(fRm->getJlProjectBlockReqThreshold()) - , fStopSending(false) - , fPhysicalIO(0) - , fCacheIO(0) , fNumBlksSkipped(0) , fMsgBytesIn(0) , fMsgBytesOut(0) @@ -474,107 +406,6 @@ pColStep::pColStep(const PassThruStep& rhs) sort(extents.begin(), extents.end(), ExtentSorter()); numExtents = extents.size(); - // uniqueID = UniqueNumberGenerator::instance()->getUnique32(); - // if (fDec) - // fDec->addQueue(uniqueID); - // initializeConfigParms ( ); -} - -pColStep::~pColStep() -{ - // join? - // delete lbidList; - // if (fDec) - // fDec->removeQueue(uniqueID); -} - -//------------------------------------------------------------------------------ -// Initialize configurable parameters -//------------------------------------------------------------------------------ -void pColStep::initializeConfigParms() -{ - // const string section ( "JobList" ); - // const string sendLimitName ( "ProjectBlockReqLimit" ); - // const string sendThresholdName ( "ProjectBlockReqThreshold" ); - // Config* cf = Config::makeConfig(); - // - // string strVal; - // uint64_t numVal; - - //...Get the tuning parameters that throttle msgs sent to primproc - //...fFilterRowReqLimit puts a cap on how many rids we will request from - //... primproc, before pausing to let the consumer thread catch up. - //... Without this limit, there is a chance that PrimProc could flood - //... ExeMgr with thousands of messages that will consume massive - //... amounts of memory for a 100 gigabyte database. - //...fFilterRowReqThreshhold is the level at which the number of outstanding - //... rids must fall below, before the producer can send more rids. - - // strVal = cf->getConfig(section, sendLimitName); - // if (strVal.size() > 0) - // { - // errno = 0; - // numVal = Config::uFromText(strVal); - // if ( errno == 0 ) - // fProjectBlockReqLimit = (uint32_t)numVal; - // } - // - // strVal = cf->getConfig(section, sendThresholdName); - // if (strVal.size() > 0) - // { - // errno = 0; - // numVal = Config::uFromText(strVal); - // if ( errno == 0 ) - // fProjectBlockReqThreshold = (uint32_t)numVal; - // } -} - -void pColStep::startPrimitiveThread() -{ - // pThread.reset(new boost::thread(pColStepPrimitive(this))); -} - -void pColStep::startAggregationThread() -{ - // cThread.reset(new boost::thread(pColStepAggregator(this))); -} - -void pColStep::run() -{ - // if (traceOn()) - // { - // syslogStartStep(16, // exemgr subsystem - // std::string("pColStep")); // step name - // } - // - // size_t sz = fInputJobStepAssociation.outSize(); - // idbassert(sz > 0); - // const AnyDataListSPtr& dl = fInputJobStepAssociation.outAt(0); - // DataList_t* dlp = dl->dataList(); - // DataList* strDlp = dl->stringDataList(); - // if ( dlp ) - // setRidList(dlp); - // else - // { - // setStrRidList( strDlp ); - // } - // //Sort can be set through the jobstep or the input JSA if fFlushinterval is 0 - // fToSort = (fFlushInterval) ? 0 : (!fToSort) ? fInputJobStepAssociation.toSort() : fToSort; - // fToSort = 0; - // //pthread_mutex_init(&mutex, NULL); - // //pthread_cond_init(&condvar, NULL); - // //pthread_cond_init(&flushed, NULL); - // startPrimitiveThread(); - // startAggregationThread(); -} - -void pColStep::join() -{ - // pThread->join(); - // cThread->join(); - // //pthread_mutex_destroy(&mutex); - // //pthread_cond_destroy(&condvar); - // //pthread_cond_destroy(&flushed); } void pColStep::addFilter(int8_t COP, float value) @@ -636,732 +467,6 @@ void pColStep::addFilter(int8_t COP, const int128_t& value, uint8_t roundFlag) fFilterCount++; } -void pColStep::setRidList(DataList* dl) -{ - ridList = dl; -} - -void pColStep::setStrRidList(DataList* strDl) -{ - strRidList = strDl; -} - -void pColStep::setBOP(int8_t b) -{ - fBOP = b; -} - -void pColStep::setOutputType(int8_t OutputType) -{ - fOutputType = OutputType; -} - -void pColStep::setSwallowRows(const bool swallowRows) -{ - fSwallowRows = swallowRows; -} - -void pColStep::sendPrimitiveMessages() -{ - // int it = -1; - // int msgRidCount = 0; - // int ridListIdx = 0; - // bool more = false; - // uint64_t absoluteRID = 0; - // int64_t msgLBID = -1; - // int64_t nextLBID = -1; - // int64_t msgLargeBlock = -1; - // int64_t nextLargeBlock = -1; - // uint16_t blockRelativeRID; - // uint32_t msgCount = 0; - // uint32_t sentBlockCount = 0; - // int msgsSkip=0; - // bool scan=false; - // bool scanThisBlock=false; - // ElementType e; - // UintRowGroup rw; - // StringElementType strE; - // StringRowGroup strRw; - // - // ByteStream msgRidList; - // ByteStream primMsg(MAX_BUFFER_SIZE); //the MAX_BUFFER_SIZE as of 8/20 - // - // NewColRequestHeader hdr; - // - // AnyDataListSPtr dl; - // FifoDataList *fifo = NULL; - // StringFifoDataList* strFifo = NULL; - // - // const bool ignoreCP = ((fTraceFlags & CalpontSelectExecutionPlan::IGNORE_CP) != 0); - // - // //The presence of more than 1 input DL means we (probably) have a pDictionaryScan step feeding this - //step - // // a list of tokens to get the rids for. Convert the input tokens to a filter string. We also have a - //rid - // // list as the second input dl - // if (fInputJobStepAssociation.outSize() > 1) - // { - // addFilters(); - // if (fTableOid >= 3000) - // cout << toString() << endl; - // //If we got no input rids (as opposed to no input DL at all) then there were no matching rows - //from - // // the previous step, so this step should not return any rows either. This would be the case, - //for - // // instance, if P_NAME LIKE '%xxxx%' produced no signature matches. - // if (fFilterCount == 0) - // { - // goto done; - // } - // } - // - // // determine which ranges/extents to eliminate from this step - // - //#ifdef DEBUG - // if (fOid>=3000) - // cout << "oid " << fOid << endl; - //#endif - // - // scanFlags.resize(numExtents); - // - // for (uint32_t idx=0; idx CasualPartitionPredicate( - // extents[idx].partition.cprange.loVal, - // extents[idx].partition.cprange.hiVal, - // &fFilterString, - // fFilterCount, - // fColType, - // fBOP) || ignoreCP; - // scanFlags[idx]=flag; - //#ifdef DEBUG - // if (fOid >= 3000 && flushInterval == 0) - // cout << (flag ? " will scan " : " will not scan ") - // << "extent with range " << extents[idx].partition.cprange.loVal - // << "-" << extents[idx].partition.cprange.hiVal << endl; - //#endif - // - // } - // - //// if (fOid>=3000) - //// cout << " " << scanFlags[idx]; - // } - //// if (scanFlags.size()>0) - //// cout << endl; - // - // // If there was more than 1 input DL, the first is a list of filters and the second is a list of rids, - // // otherwise the first is the list of rids. - // if (fInputJobStepAssociation.outSize() > 1) - // ridListIdx = 1; - // else - // ridListIdx = 0; - // - // dl = fInputJobStepAssociation.outAt(ridListIdx); - // ridList = dl->dataList(); - // if ( ridList ) - // { - // fifo = dl->fifoDL(); - // - // if (fifo) - // it = fifo->getIterator(); - // else - // it = ridList->getIterator(); - // } - // else - // { - // strRidList = dl->stringDataList(); - // strFifo = dl->stringDL(); - // - // if (strFifo) - // it = strFifo->getIterator(); - // else - // it = strRidList->getIterator(); - // } - // - // if (ridList) - // { - // if (fifo) - // { - // more = fifo->next(it, &rw); - // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { - // dlTimes.setFirstReadTime(); - // } - // absoluteRID = rw.et[0].first; - // } - // else - // { - // more = ridList->next(it, &e); - // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { - // dlTimes.setFirstReadTime(); - // } - // absoluteRID = e.first; - // rw.count = 1; - // } - // } - // else - // { - // if (strFifo) - // { - // more = strFifo->next(it, &strRw); - // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { - // dlTimes.setFirstReadTime(); - // } - // absoluteRID = strRw.et[0].first; - // } - // else - // { - // more = strRidList->next(it, &strE); - // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { - // dlTimes.setFirstReadTime(); - // } - // absoluteRID = strE.first; - // strRw.count = 1; - // } - // } - // - // if (more) - // msgLBID = getLBID(absoluteRID, scan); - // scanThisBlock = scan; - // msgLargeBlock = absoluteRID >> blockSizeShift; - // - // while (more || msgRidCount > 0) { - // uint64_t rwCount; - // if ( ridList) - // rwCount = rw.count; - // else - // rwCount = strRw.count; - // - // for (uint64_t i = 0; ((i < rwCount) || (!more && msgRidCount > 0)); ) - // { - // if ( ridList) - // { - // if (fifo) - // absoluteRID = rw.et[i].first; - // else - // absoluteRID = e.first; - // } - // else - // { - // if (strFifo) - // absoluteRID = strRw.et[i].first; - // else - // absoluteRID = strE.first; - // } - // - // if (more) { - // nextLBID = getLBID(absoluteRID, scan); - // nextLargeBlock = absoluteRID >> blockSizeShift; - // } - // - // //XXXPAT: need to prove N & S here - // if (nextLBID == msgLBID && more) { - //// blockRelativeRID = absoluteRID % ridsPerBlock; - // blockRelativeRID = absoluteRID & rpbMask; - // msgRidList << blockRelativeRID; - // msgRidCount++; - // ++i; - // } - // else { - // //Bug 831: move building msg after the check of scanThisBlock - // if (scanThisBlock==true) - // { - // hdr.ism.Interleave=0; - // hdr.ism.Flags=planFlagsToPrimFlags(fTraceFlags); - // hdr.ism.Command=COL_BY_SCAN; - // hdr.ism.Size=sizeof(NewColRequestHeader) + fFilterString.length() + - // msgRidList.length(); - // hdr.ism.Type=2; - // - // hdr.hdr.SessionID = fSessionId; - // //hdr.hdr.StatementID = 0; - // hdr.hdr.TransactionID = fTxnId; - // hdr.hdr.VerID = fVerId; - // hdr.hdr.StepID = fStepId; - // hdr.hdr.UniqueID = uniqueID; - // - // hdr.LBID = msgLBID; - //// idbassert(hdr.LBID >= 0); - // hdr.DataSize = fColType.colWidth; - // hdr.DataType = fColType.colDataType; - // hdr.CompType = fColType.compressionType; - // hdr.OutputType = fOutputType; - // hdr.BOP = fBOP; - // hdr.NOPS = fFilterCount; - // hdr.NVALS = msgRidCount; - // hdr.sort = fToSort; - // - // primMsg.append((const uint8_t *) &hdr, sizeof(NewColRequestHeader)); - // primMsg += fFilterString; - // primMsg += msgRidList; - // ridCount += msgRidCount; - // ++sentBlockCount; - // - //#ifdef DEBUG - // if (flushInterval == 0 && fOid >= 3000) - // cout << "sending a prim msg for LBID " << msgLBID << endl; - //#endif - // ++msgCount; - //// cout << "made a primitive\n"; - // if (msgLargeBlock != nextLargeBlock || !more) { - //// cout << "writing " << msgCount << " primitives\n"; - // fMsgBytesOut += primMsg.lengthWithHdrOverhead(); - // fDec->write(primMsg); - // msgsSent += msgCount; - // msgCount = 0; - // primMsg.restart(); - // msgLargeBlock = nextLargeBlock; - // - // // @bug 769 - Added "&& !fSwallowRows" condition below to fix problem - //with - // // caltraceon(16) not working for tpch01 and some other queries. If a - //query - // // ever held off requesting more blocks, it would lock and never finish. - // //Bug 815 - // if (( sentBlockCount >= fProjectBlockReqLimit) && !fSwallowRows - //&& - // (( msgsSent - msgsRecvd) > fProjectBlockReqThreshold)) - // { - // mutex.lock(); //pthread_mutex_lock(&mutex); - // fStopSending = true; - // - // // @bug 836. Wake up the receiver if he's sleeping. - // if (recvWaiting) - // condvar.notify_one(); - ////pthread_cond_signal(&condvar); flushed.wait(mutex); //pthread_cond_wait(&flushed, &mutex); fStopSending - //= false; mutex.unlock(); //pthread_mutex_unlock(&mutex); sentBlockCount = 0; - // } - // } - // } - // else - // { - // msgsSkip++; - // } - // msgLBID = nextLBID; - // msgRidList.restart(); - // msgRidCount = 0; - // - // mutex.lock(); //pthread_mutex_lock(&mutex); - // - // if (scanThisBlock) { - // if (recvWaiting) - // condvar.notify_one(); //pthread_cond_signal(&condvar); - // #ifdef DEBUG - //// cout << "msgsSent++ = " << msgsSent << endl; - // #endif - // } - // scanThisBlock = scan; - // mutex.unlock(); //pthread_mutex_unlock(&mutex); - // - // // break the for loop - // if (!more) - // break; - // } - // } // for rw.count - // - // if (more) - // { - // if ( ridList ) - // { - // if (fifo) - // { - // rw.count = 0; - // more = fifo->next(it, &rw); - // } - // else - // { - // rw.count = 1; - // more = ridList->next(it, &e); - // } - // } - // else - // { - // if (strFifo) - // { - // strRw.count = 0; - // more = strFifo->next(it, &strRw); - // } - // else - // { - // strRw.count = 1; - // more = strRidList->next(it, &strE); - // } - // } - // } - // } - // - // if (fOid>=3000) dlTimes.setLastReadTime(); - // - // done: - // mutex.lock(); //pthread_mutex_lock(&mutex); - // finishedSending = true; - // if (recvWaiting) - // condvar.notify_one(); //pthread_cond_signal(&condvar); - // mutex.unlock(); //pthread_mutex_unlock(&mutex); - // - //#ifdef DEBUG - // if (fOid >=3000) - // cout << "pColStep msgSent " - // << msgsSent << "/" << msgsSkip - // << " rids " << ridCount - // << " oid " << fOid << " " << msgLBID << endl; - //#endif - // //...Track the number of LBIDs we skip due to Casual Partioning. - // fNumBlksSkipped += msgsSkip; -} - -void pColStep::receivePrimitiveMessages() -{ - // int64_t ridResults = 0; - // AnyDataListSPtr dl = fOutputJobStepAssociation.outAt(0); - // DataList_t* dlp = dl->dataList(); - // uint64_t fbo; - // FifoDataList *fifo = dl->fifoDL(); - // UintRowGroup rw; - // uint64_t ridBase; - // boost::shared_ptr bs; - // uint32_t i = 0, length; - // - // while (1) { - // // sync with the send side - // mutex.lock(); //pthread_mutex_lock(&mutex); - // while (!finishedSending && msgsSent == msgsRecvd) { - // recvWaiting = true; - // #ifdef DEBUG - // cout << "c sleeping" << endl; - // #endif - // // @bug 836. Wake up the sender if he's sleeping. - // if (fStopSending) - // flushed.notify_one(); //pthread_cond_signal(&flushed); - // condvar.wait(mutex); //pthread_cond_wait(&condvar, &mutex); - // #ifdef DEBUG - // cout << "c waking" << endl; - // #endif - // recvWaiting = false; - // } - // if (msgsSent == msgsRecvd) { - // mutex.unlock(); //pthread_mutex_unlock(&mutex); - // break; - // } - // mutex.unlock(); //pthread_mutex_unlock(&mutex); - // - // // do the recv - // fDec->read(uniqueID, bs); - // fMsgBytesIn += bs->lengthWithHdrOverhead(); - // - // // no more messages, and buffered messages should be already processed by now. - // if (bs->length() == 0) break; - // - // #ifdef DEBUG - // cout << "msgsRecvd++ = " << msgsRecvd << ". RidResults = " << ridResults << endl; - // cout << "Got a ColResultHeader!: " << bs.length() << " bytes" << endl; - // #endif - // - // const ByteStream::byte* bsp = bs->buf(); - // - // // get the ISMPacketHeader out of the bytestream - // //const ISMPacketHeader* ism = reinterpret_cast(bsp); - // - // // get the ColumnResultHeader out of the bytestream - // const ColResultHeader* crh = reinterpret_cast - // (&bsp[sizeof(ISMPacketHeader)]); - // - // bool firstRead = true; - // length = bs->length(); - // - // i = 0; - // uint32_t msgCount = 0; - // while (i < length) { - // ++msgCount; - // - // i += sizeof(ISMPacketHeader); - // crh = reinterpret_cast(&bsp[i]); - // // double check the sequence number is increased by one each time - // i += sizeof(ColResultHeader); - // - // fCacheIO += crh->CacheIO; - // fPhysicalIO += crh->PhysicalIO; - // - // // From this point on the rest of the bytestream is the data that comes back from the - //primitive server - // // This needs to be fed to a datalist that is retrieved from the outputassociation - //object. - // - // fbo = getFBO(crh->LBID); - // ridBase = fbo << rpbShift; - // - // #ifdef DEBUG - //// cout << " NVALS = " << crh->NVALS << " fbo = " << fbo << " lbid = " << crh->LBID << - ///endl; - // #endif - // - // //Check output type - // if ( fOutputType == OT_RID ) - // { - // ridResults += crh->NVALS; - // } - // - // /* XXXPAT: This clause is executed when ExeMgr calls the new nextBand(BS) fcn. - // - // TODO: both classes have to agree - // on which nextBand() variant will be called. pColStep - // currently has to infer that from flushInterval and the - // Table OID. It would be better to have a more explicit form - // of agreement. - // - // The goal of the nextBand(BS) fcn is to avoid iterating over - // every row except at unserialization. This clause copies - // the raw results from the PrimProc response directly into - // the memory used for the ElementType array. DeliveryStep - // will also treat the ElementType array as raw memory and - // serialize that. TableColumn now parses the packed data - // instead of whole ElementTypes. - // */ - // else if (fOutputType == OT_TOKEN && fFlushInterval > 0 && !fIsDict) { - // - // if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) - // dlTimes.setFirstInsertTime(); - // ridResults += crh->NVALS; - // - // /* memcpy the bytestream into the output set */ - // uint32_t toCopy, bsPos = 0; - // uint8_t *pos; - // while (bsPos < crh->NVALS) { - // toCopy = (crh->NVALS - bsPos > rw.ElementsPerGroup - rw.count ? - // rw.ElementsPerGroup - rw.count : crh->NVALS - bsPos); - // pos = ((uint8_t *) &rw.et[0]) + (rw.count * fColType.colWidth); - // memcpy(pos, &bsp[i], toCopy * fColType.colWidth); - // bsPos += toCopy; - // i += toCopy * fColType.colWidth; - // rw.count += toCopy; - // if (rw.count == rw.ElementsPerGroup) { - // if (!fSwallowRows) - // fifo->insert(rw); - // rw.count = 0; - // } - // } - // } - // else if ( fOutputType == OT_TOKEN) - // { - // uint64_t dv; - // uint64_t rid; - // - // if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) - // dlTimes.setFirstInsertTime(); - // ridResults += crh->NVALS; - // for(int j = 0; j < crh->NVALS; ++j) - // { - // // XXXPAT: Only use this when the RID doesn't matter or when - // // the response contains every row. - // - // rid = j + ridBase; - // switch (fColType.colWidth) { - // case 8: dv = *((const uint64_t *) &bsp[i]); i += 8; break; - // case 4: dv = *((const uint32_t *) &bsp[i]); i += 4; break; - // case 2: dv = *((const uint16_t *) &bsp[i]); i += 2; break; - // case 1: dv = *((const uint8_t *) &bsp[i]); ++i; break; - // default: - // throw runtime_error("pColStep: invalid column - //width!"); - // } - // - // // @bug 663 - Don't output any rows if fSwallowRows (caltraceon(16)) is - //on. - // // This options swallows rows in the project steps. - // if (!fSwallowRows) - // { - // if (fifo) - // { - // rw.et[rw.count].first = rid; - // rw.et[rw.count++].second = dv; - // if (rw.count == rw.ElementsPerGroup) - // { - // fifo->insert(rw); - // rw.count = 0; - // } - // } - // else - // { - // dlp->insert(ElementType(rid, dv)); - // } - // #ifdef DEBUG - // //cout << " -- inserting <" << rid << ", " << dv << "> " << *prid << - //endl; #endif - // } - // } - // } - // else if ( fOutputType == OT_BOTH ) - // { - // ridResults += crh->NVALS; - // for(int j = 0; j < crh->NVALS; ++j) - // { - // uint64_t dv; - // uint64_t rid; - // - // rid = *((const uint16_t *) &bsp[i]) + ridBase; - // i += sizeof(uint16_t); - // switch (fColType.colWidth) { - // case 8: dv = *((const uint64_t *) &bsp[i]); i += 8; - //break; case 4: dv = *((const uint32_t *) &bsp[i]); i += 4; break; case 2: dv = *((const uint16_t *) - //&bsp[i]); i += 2; break; case 1: dv = *((const uint8_t *) &bsp[i]); ++i; break; default: throw - //runtime_error("pColStep: invalid column width!"); - // } - // - // // @bug 663 - Don't output any rows if fSwallowRows (caltraceon(16)) is - //on. - // // This options swallows rows in the project steps. - // if (!fSwallowRows) { - // if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) - // dlTimes.setFirstInsertTime(); - // if(fifo) - // { - //// rw.et[rw.count++] = ElementType(rid, dv); - // rw.et[rw.count].first = rid; - // rw.et[rw.count++].second = dv; - // if (rw.count == rw.ElementsPerGroup) - // { - // fifo->insert(rw); - // rw.count = 0; - // } - // } - // else - // { - // dlp->insert(ElementType(rid, dv)); - // } - // #ifdef DEBUG - // //cout << " -- inserting <" << rid << ", " << dv << "> " << *prid << - //endl; #endif - // } - // } - // } - // } // unpacking the BS - // - // //Bug 815: Check whether we have enough to process - // //++lockCount; - // mutex.lock(); //pthread_mutex_lock(&mutex); - // if ( fStopSending && ((msgsSent - msgsRecvd ) <= fProjectBlockReqThreshold) ) - // { - // flushed.notify_one(); //pthread_cond_signal(&flushed); - // } - // mutex.unlock(); //pthread_mutex_unlock(&mutex); - // - // firstRead = false; - // msgsRecvd += msgCount; - // } // read loop - // // done reading - // - // if (fifo && rw.count > 0) - // fifo->insert(rw); - // - // //...Casual partitioning could cause us to do no processing. In that - // //...case these time stamps did not get set. So we set them here. - // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { - // dlTimes.setFirstReadTime(); - // dlTimes.setLastReadTime(); - // dlTimes.setFirstInsertTime(); - // } - // if (fOid>=3000) dlTimes.setEndOfInputTime(); - // - // //@bug 699: Reset StepMsgQueue - // fDec->removeQueue(uniqueID); - // - // if (fifo) - // fifo->endOfInput(); - // else - // dlp->endOfInput(); - // - // if (fTableOid >= 3000) - // { - // //...Construct timestamp using ctime_r() instead of ctime() not - // //...necessarily due to re-entrancy, but because we want to strip - // //...the newline ('\n') off the end of the formatted string. - // time_t t = time(0); - // char timeString[50]; - // ctime_r(&t, timeString); - // timeString[strlen(timeString)-1 ] = '\0'; - // - // FifoDataList* pFifo = 0; - // uint64_t totalBlockedReadCount = 0; - // uint64_t totalBlockedWriteCount = 0; - // - // //...Sum up the blocked FIFO reads for all input associations - // size_t inDlCnt = fInputJobStepAssociation.outSize(); - // for (size_t iDataList=0; iDataListfifoDL(); - // if (pFifo) - // { - // totalBlockedReadCount += pFifo->blockedReadCount(); - // } - // } - // - // //...Sum up the blocked FIFO writes for all output associations - // size_t outDlCnt = fOutputJobStepAssociation.outSize(); - // for (size_t iDataList=0; iDataListfifoDL(); - // if (pFifo) - // { - // totalBlockedWriteCount += pFifo->blockedWriteCount(); - // } - // } - // - // //...Roundoff msg byte counts to nearest KB for display - // uint64_t msgBytesInKB = fMsgBytesIn >> 10; - // uint64_t msgBytesOutKB = fMsgBytesOut >> 10; - // if (fMsgBytesIn & 512) - // msgBytesInKB++; - // if (fMsgBytesOut & 512) - // msgBytesOutKB++; - // - // // @bug 828 - // if (fifo) - // fifo->totalSize(ridResults); - // - // if (traceOn()) - // { - // //...Print job step completion information - // ostringstream logStr; - // logStr << "ses:" << fSessionId << - // " st: " << fStepId << " finished at " << - // timeString << "; PhyI/O-" << fPhysicalIO << "; CacheI/O-" << - // fCacheIO << "; MsgsRvcd-" << msgsRecvd << - // "; BlockedFifoIn/Out-" << totalBlockedReadCount << - // "/" << totalBlockedWriteCount << - // "; output size-" << ridResults << endl << - // "\tPartitionBlocksEliminated-" << fNumBlksSkipped << - // "; MsgBytesIn-" << msgBytesInKB << "KB" << - // "; MsgBytesOut-" << msgBytesOutKB << "KB" << endl << - // "\t1st read " << dlTimes.FirstReadTimeString() << - // "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" << - // JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(),dlTimes.FirstReadTime()) << - // "s" << endl; - // - // logEnd(logStr.str().c_str()); - // - // syslogReadBlockCounts(16, // exemgr sybsystem - // fPhysicalIO, // # blocks read from disk - // fCacheIO, // # blocks read from cache - // fNumBlksSkipped); // # casual partition block hits - // syslogProcessingTimes(16, // exemgr subsystem - // dlTimes.FirstReadTime(), // first datalist read - // dlTimes.LastReadTime(), // last datalist read - // dlTimes.FirstInsertTime(), // first datalist write - // dlTimes.EndOfInputTime()); // last (endOfInput) datalist write - // syslogEndStep(16, // exemgr subsystem - // totalBlockedReadCount, // blocked datalist input - // totalBlockedWriteCount, // blocked datalist output - // fMsgBytesIn, // incoming msg byte count - // fMsgBytesOut); // outgoing msg byte count - // } - // } -} - const string pColStep::toString() const { ostringstream oss; @@ -1511,4 +616,3 @@ void pColStep::appendFilter(const std::vector& fs) } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/pdictionary.cpp b/dbcon/joblist/pdictionary.cpp index 71a941713..7372a15ed 100644 --- a/dbcon/joblist/pdictionary.cpp +++ b/dbcon/joblist/pdictionary.cpp @@ -105,64 +105,11 @@ pDictionaryStep::pDictionaryStep(CalpontSystemCatalog::OID o, CalpontSystemCatal , fFilterCount(0) , requestList(0) , fInterval(jobInfo.flushInterval) - , fPhysicalIO(0) - , fCacheIO(0) , fMsgBytesIn(0) , fMsgBytesOut(0) , fRm(jobInfo.rm) , hasEqualityFilter(false) { - // uniqueID = UniqueNumberGenerator::instance()->getUnique32(); - - // fColType.compressionType = fColType.ddn.compressionType = ct; -} - -pDictionaryStep::~pDictionaryStep() -{ - // if (fDec) - // fDec->removeQueue(uniqueID); -} - -void pDictionaryStep::startPrimitiveThread() -{ - // pThread.reset(new boost::thread(pDictionaryStepPrimitive(this))); -} - -void pDictionaryStep::startAggregationThread() -{ - // cThread.reset(new boost::thread(pDictStepAggregator(this))); -} - -void pDictionaryStep::run() -{ - // if (traceOn()) - // { - // syslogStartStep(16, // exemgr subsystem - // std::string("pDictionaryStep")); // step name - // } - // - // const AnyDataListSPtr& dl = fInputJobStepAssociation.outAt(0); - // DataList_t* dlp = dl->dataList(); - // setInputList(dlp); - // - // startPrimitiveThread(); - // startAggregationThread(); -} - -void pDictionaryStep::join() -{ - // pThread->join(); - // cThread->join(); -} - -void pDictionaryStep::setInputList(DataList_t* dl) -{ - requestList = dl; -} - -void pDictionaryStep::setBOP(int8_t b) -{ - fBOP = b; } void pDictionaryStep::addFilter(int8_t COP, const string& value) @@ -190,322 +137,6 @@ void pDictionaryStep::addFilter(int8_t COP, const string& value) } } -void pDictionaryStep::sendPrimitiveMessages() -{ - // int it = -1; - // int msgRidCount = 0; - // bool more; - // int64_t sigToken, msgLBID, nextLBID = -1; - // uint16_t sigOrd; - // ByteStream msgRidList, primMsg(65536); //the MAX_BUFFER_SIZE as of 8/20 - // DictSignatureRequestHeader hdr; - // ISMPacketHeader ism; - // OldGetSigParams pt; - // FifoDataList* fifo = fInputJobStepAssociation.outAt(0)->fifoDL(); - // UintRowGroup rw; - // - ///* XXXPAT: Does this primitive need to care about the HWM as a sanity check, given - // that a ridlist is supplied? */ - // - // if (fifo == 0) - // throw logic_error("Use p_colscanrange instead here"); - // - // try{ - // it = fifo->getIterator(); - // }catch(exception& ex) { - // cerr << "pDictionaryStep::sendPrimitiveMessages: caught exception: " << ex.what() << endl; - // }catch(...) { - // cerr << "pDictionaryStep::sendPrimitiveMessages: caught exception" << endl; - // } - // - // more = fifo->next(it, &rw); - // - // sigToken = rw.et[0].second; - // msgLBID = sigToken >> 10; - // while (more || msgRidCount > 0) { - // for (uint64_t i = 0; ((i < rw.count) || (!more && msgRidCount > 0)); ) - // { - // if (more) - // { - // ridCount++; - // sigToken = rw.et[i].second; - // nextLBID = sigToken >> 10; - //#ifdef DEBUG - // cout << "sigToken = " << sigToken << " lbid = " << nextLBID << endl; - //#endif - // } - // - // // @bug 472 - // if (nextLBID == msgLBID && more && msgRidCount < 8000) { //XXXPAT: need to prove N & S - //here sigOrd = sigToken & 0x3ff; pt.rid = (nextLBID >= 0 ? rw.et[i].first : 0x8000000000000000LL | - //rw.et[i].first); pt.offsetIndex = sigOrd; msgRidList.append(reinterpret_cast(&pt), - //sizeof(pt)); msgRidCount++; - // ++i; - //#ifdef DEBUG - // cout << "added signature ordinal " << sigOrd << endl; - //#endif - // } - // else { - //#ifdef DEBUG - // cout << "sending a prim msg" << endl; - //#endif - // - // // send the primitive, start constructing the next msg - // ism.Interleave=0; - // ism.Flags=planFlagsToPrimFlags(fTraceFlags); - // ism.Command=DICT_SIGNATURE; - // ism.Size=sizeof(DictSignatureRequestHeader) + msgRidList.length(); - // ism.Type=2; - // - // hdr.Hdr.SessionID = fSessionId; - // //hdr.Hdr.StatementID = 0; - // hdr.Hdr.TransactionID = fTxnId; - // hdr.Hdr.VerID = fVerId; - // hdr.Hdr.StepID = fStepId; - // hdr.Hdr.UniqueID = uniqueID; - // - // hdr.LBID = msgLBID; - // idbassert(msgRidCount <= 8000); - // hdr.NVALS = msgRidCount; - // hdr.CompType = fColType.ddn.compressionType; - // - // primMsg.load((const uint8_t *) &ism, sizeof(ism)); - // primMsg.append((const uint8_t *) &hdr, sizeof(DictSignatureRequestHeader)); - // primMsg += msgRidList; - // fMsgBytesOut += primMsg.lengthWithHdrOverhead(); - // fDec->write(primMsg); - // - // msgLBID = nextLBID; - // primMsg.restart(); - // msgRidList.restart(); - // msgRidCount = 0; - // - // mutex.lock(); - // msgsSent++; - // if (recvWaiting) - // condvar.notify_one(); - //#ifdef DEBUG - // cout << "msgsSent++ = " << msgsSent << endl; - //#endif - // mutex.unlock(); - // - // if (!more) - // break; - // } - // } // rw.count - // - // if (more) - // { - // rw.count = 0; - // more = fifo->next(it, &rw); - // } - // } - // - // mutex.lock(); - // finishedSending = true; - // if (recvWaiting) - // condvar.notify_one(); - // mutex.unlock(); -} - -void pDictionaryStep::receivePrimitiveMessages() -{ - // int64_t ridResults = 0; - // AnyDataListSPtr dl = fOutputJobStepAssociation.outAt(0); - // StrDataList* dlp = dl->stringDataList(); - // StringFifoDataList *fifo = fOutputJobStepAssociation.outAt(0)->stringDL(); - // StringRowGroup rw; - // - // while (1) { - // - // // sync with the send side - // mutex.lock(); - // - // while (!finishedSending && msgsSent==msgsRecvd) { - // recvWaiting = true; - // condvar.wait(mutex); - // if (msgsSent == msgsRecvd) { - // mutex.unlock(); - // break; - // } - // recvWaiting = false; - // } - // - // if (finishedSending != 0 && msgsRecvd >= msgsSent) { - // goto junk; - // } - // mutex.unlock(); - // - // // do the recv - // - // ByteStream bs = fDec->read(uniqueID); - // fMsgBytesIn += bs.lengthWithHdrOverhead(); - // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) - // dlTimes.setFirstReadTime(); - // if (fOid>=3000) dlTimes.setLastReadTime(); - // - // msgsRecvd++; - // if (bs.length() == 0) - // break; - // - // const ByteStream::byte* bsp = bs.buf(); - // - // // get the ResultHeader out of the bytestream - // const DictOutput* drh = reinterpret_cast(bsp); - // - // bsp += sizeof(DictOutput); - // - // fCacheIO += drh->CacheIO; - // fPhysicalIO += drh->PhysicalIO; - // - // // From this point on the rest of the bytestream is the data that comes back from the primitive - //server - // // This needs to be fed to a datalist that is retrieved from the outputassociation object. - // - // char d[8192]; - //// memset(d, 0, 8192); - // if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) - // dlTimes.setFirstInsertTime(); - // for(int j = 0; j < drh->NVALS; j++) - // { - // const uint64_t* ridp = (const uint64_t*)bsp; - // bsp += sizeof(*ridp); - // uint64_t rid = *ridp; - // const uint16_t* lenp = (const uint16_t*)bsp; - // bsp += sizeof(*lenp); - // uint16_t len = *lenp; - // memcpy(d, bsp, len); - // bsp += len; - // d[len] = 0; - // if (rid == 0xFFFFFFFFFFFFFFFFULL) - // { - // strcpy(d, CPNULLSTRMARK.c_str()); - // } - //#ifdef FIFO_SINK - // if (fOid < 3000) - //#endif - // if (fifo) - // { - // rw.et[rw.count++] = StringElementType(rid, d); - // if (rw.count == rw.ElementsPerGroup) - // { - // fifo->insert(rw); - // rw.count = 0; - // } - // } - // else - // { - // dlp->insert(StringElementType(rid, d)); - // } - // - //#ifdef DEBUG - // cout << " -- inserting <" << rid << ", " << d << ">" << endl; - //#endif - // ridResults++; - // - // } - // } - // - // junk: - // - // if (fifo && rw.count > 0) - // fifo->insert(rw); - // - // //@bug 699: Reset StepMsgQueue - // fDec->removeQueue(uniqueID); - // - // if (fOid>=3000) dlTimes.setEndOfInputTime(); - // dlp->endOfInput(); - // - // if (fTableOid >= 3000) - // { - // //...Construct timestamp using ctime_r() instead of ctime() not - // //...necessarily due to re-entrancy, but because we want to strip - // //...the newline ('\n') off the end of the formatted string. - // time_t t = time(0); - // char timeString[50]; - // ctime_r(&t, timeString); - // timeString[strlen(timeString)-1 ] = '\0'; - // - // FifoDataList* pFifo = 0; - // uint64_t totalBlockedReadCount = 0; - // uint64_t totalBlockedWriteCount = 0; - // - // //...Sum up the blocked FIFO reads for all input associations - // size_t inDlCnt = fInputJobStepAssociation.outSize(); - // for (size_t iDataList=0; iDataListfifoDL(); - // if (pFifo) - // { - // totalBlockedReadCount += pFifo->blockedReadCount(); - // } - // } - // - // //...Sum up the blocked FIFO writes for all output associations - // size_t outDlCnt = fOutputJobStepAssociation.outSize(); - // for (size_t iDataList=0; iDataListfifoDL(); - // if (pFifo) - // { - // totalBlockedWriteCount += pFifo->blockedWriteCount(); - // } - // } - // - // - // - // //...Roundoff msg byte counts to nearest KB for display - // uint64_t msgBytesInKB = fMsgBytesIn >> 10; - // uint64_t msgBytesOutKB = fMsgBytesOut >> 10; - // if (fMsgBytesIn & 512) - // msgBytesInKB++; - // if (fMsgBytesOut & 512) - // msgBytesOutKB++; - // - // // @bug 807 - // if (fifo) - // fifo->totalSize(ridResults); - // - // if (traceOn()) - // { - // //...Print job step completion information - // ostringstream logStr; - // logStr << "ses:" << fSessionId << " st: " << fStepId << - // " finished at " << - // timeString << "; PhyI/O-" << fPhysicalIO << "; CacheI/O-" << - // fCacheIO << "; MsgsRcvd-" << msgsRecvd << - // "; BlockedFifoIn/Out-" << totalBlockedReadCount << - // "/" << totalBlockedWriteCount << - // "; output size-" << ridResults << endl << - // "\tMsgBytesIn-" << msgBytesInKB << "KB" << - // "; MsgBytesOut-" << msgBytesOutKB << "KB" << endl << - // "\t1st read " << dlTimes.FirstReadTimeString() << - // "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" << - // JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(),dlTimes.FirstReadTime()) << - // "s" << endl; - // - // logEnd(logStr.str().c_str()); - // - // syslogReadBlockCounts(16, // exemgr subsystem - // fPhysicalIO, // # blocks read from disk - // fCacheIO, // # blocks read from cache - // 0); // # casual partition block hits - // syslogProcessingTimes(16, // exemgr subsystem - // dlTimes.FirstReadTime(), // first datalist read - // dlTimes.LastReadTime(), // last datalist read - // dlTimes.FirstInsertTime(), // first datalist write - // dlTimes.EndOfInputTime()); // last (endOfInput) datalist write - // syslogEndStep(16, // exemgr subsystem - // totalBlockedReadCount, // blocked datalist input - // totalBlockedWriteCount, // blocked datalist output - // fMsgBytesIn, // incoming msg byte count - // fMsgBytesOut); // outgoing msg byte count - // } - // } - // -} - const string pDictionaryStep::toString() const { ostringstream oss; @@ -546,9 +177,6 @@ void pDictionaryStep::appendFilter(const messageqcpp::ByteStream& filter, unsign addFilter(COP, value); bs.advance(size); } - - // fFilterString += filter; - // fFilterCount += count; } void pDictionaryStep::addFilter(const Filter* f) diff --git a/dbcon/joblist/pdictionaryscan.cpp b/dbcon/joblist/pdictionaryscan.cpp index df795b862..164ac157c 100644 --- a/dbcon/joblist/pdictionaryscan.cpp +++ b/dbcon/joblist/pdictionaryscan.cpp @@ -138,10 +138,8 @@ pDictionaryScan::pDictionaryScan(CalpontSystemCatalog::OID o, CalpontSystemCatal , fColType(ct) , pThread(0) , cThread(0) - , fScanLbidReqLimit(jobInfo.rm->getJlScanLbidReqLimit()) , fScanLbidReqThreshold(jobInfo.rm->getJlScanLbidReqThreshold()) , fStopSending(false) - , fSingleThread(false) , fPhysicalIO(0) , fCacheIO(0) , fMsgBytesIn(0) @@ -915,5 +913,19 @@ void pDictionaryScan::abort() fDec->shutdownQueue(uniqueID); } +// Unfortuneately we have 32 bits in the execplan flags, but only 16 that can be sent to +// PrimProc, so we have to convert them (throwing some away). +uint16_t pDictionaryScan::planFlagsToPrimFlags(uint32_t planFlags) +{ + uint16_t flags = 0; + + if (planFlags & CalpontSelectExecutionPlan::TRACE_LBIDS) + flags |= PF_LBID_TRACE; + + if (planFlags & CalpontSelectExecutionPlan::PM_PROFILE) + flags |= PF_PM_PROF; + + return flags; +} + } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/primitivemsg.cpp b/dbcon/joblist/primitivemsg.cpp deleted file mode 100644 index 47e7b8e2f..000000000 --- a/dbcon/joblist/primitivemsg.cpp +++ /dev/null @@ -1,74 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/* - * $Id: primitivemsg.cpp 9210 2013-01-21 14:10:42Z rdempsey $ - */ - -#include -using namespace std; - -#include "calpontselectexecutionplan.h" -using namespace execplan; - -#include "primitivemsg.h" -#include "primitivestep.h" -using namespace joblist; - -namespace joblist -{ -void PrimitiveMsg::send() -{ - throw logic_error("somehow ended up in PrimitiveMsg::send()!"); -} - -void PrimitiveMsg::buildPrimitiveMessage(ISMPACKETCOMMAND, void*, void*) -{ - throw logic_error("somehow ended up in PrimitiveMsg::buildPrimitiveMessage()!"); -} - -void PrimitiveMsg::receive() -{ - throw logic_error("somehow ended up in PrimitiveMsg::receive()!"); -} - -void PrimitiveMsg::sendPrimitiveMessages() -{ - throw logic_error("somehow ended up in PrimitiveMsg::sendPrimitiveMessages()!"); -} - -void PrimitiveMsg::receivePrimitiveMessages() -{ - throw logic_error("somehow ended up in PrimitiveMsg::receivePrimitiveMessages()!"); -} - -// Unfortuneately we have 32 bits in the execplan flags, but only 16 that can be sent to -// PrimProc, so we have to convert them (throwing some away). -uint16_t PrimitiveMsg::planFlagsToPrimFlags(uint32_t planFlags) -{ - uint16_t flags = 0; - - if (planFlags & CalpontSelectExecutionPlan::TRACE_LBIDS) - flags |= PF_LBID_TRACE; - - if (planFlags & CalpontSelectExecutionPlan::PM_PROFILE) - flags |= PF_PM_PROF; - - return flags; -} - -} // namespace joblist diff --git a/dbcon/joblist/primitivemsg.h b/dbcon/joblist/primitivemsg.h index 0d495b891..fe0c52ec2 100644 --- a/dbcon/joblist/primitivemsg.h +++ b/dbcon/joblist/primitivemsg.h @@ -884,4 +884,3 @@ struct LbidAtVer #pragma pack(pop) -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/primitivestep.h b/dbcon/joblist/primitivestep.h index 97372ff9c..87187e360 100644 --- a/dbcon/joblist/primitivestep.h +++ b/dbcon/joblist/primitivestep.h @@ -93,39 +93,9 @@ enum PrimitiveStepType AGGRFILTERSTEP }; -/** @brief class PrimitiveMsg - * - */ -class PrimitiveMsg -{ - public: - /** @brief virtual void Send method - */ - virtual void send(); - /** @brief virtual void Receive method - */ - virtual void receive(); - /** @brief virtual void BuildPrimitiveMessage method - */ - virtual void buildPrimitiveMessage(ISMPACKETCOMMAND cmd, void* filterValues, void* ridArray); - virtual void sendPrimitiveMessages(); - virtual void receivePrimitiveMessages(); - - PrimitiveMsg() - { - } - - virtual ~PrimitiveMsg() - { - } - - uint16_t planFlagsToPrimFlags(uint32_t planFlags); - - private: -}; class pColScanStep; -class pColStep : public JobStep, public PrimitiveMsg +class pColStep : public JobStep { typedef std::pair element_t; @@ -141,48 +111,30 @@ class pColStep : public JobStep, public PrimitiveMsg pColStep(const PassThruStep& rhs); - virtual ~pColStep(); + virtual ~pColStep(){}; /** @brief Starts processing. Set at least the RID list before calling. * * Starts processing. Set at least the RID list before calling this. */ - virtual void run(); + virtual void run(){}; /** @brief Sync's the caller with the end of execution. * * Does nothing. Returns when this instance is finished. */ - virtual void join(); + virtual void join(){}; virtual const std::string toString() const; virtual bool isDictCol() const { return fIsDict; - }; + } bool isExeMgr() const { return isEM; } - /** @brief Set config parameters for this JobStep. - * - * Set the config parameters this JobStep. - */ - void initializeConfigParms(); - - /** @brief The main loop for the send-side thread - * - * The main loop for the primitive-issuing thread. Don't call it directly. - */ - void sendPrimitiveMessages(); - - /** @brief The main loop for the recv-side thread - * - * The main loop for the receive-side thread. Don't call it directly. - */ - void receivePrimitiveMessages(); - /** @brief Add a filter. Use this interface when the column stores anything but 4-byte floats. * * Add a filter. Use this interface when the column stores anything but 4-byte floats. @@ -197,35 +149,38 @@ class pColStep : public JobStep, public PrimitiveMsg * this class from pColScan. Use pColScan if the every RID should be considered; it's * faster at that. */ - void setRidList(DataList* rids); - + void setRidList(DataList* rids) + { + ridList = rids; + } /** @brief Sets the String DataList to get RID values from. * * Sets the string DataList to get RID values from. Filtering by RID distinguishes * this class from pColScan. Use pColScan if the every RID should be considered; it's * faster at that. */ - void setStrRidList(DataList* strDl); - + void setStrRidList(DataList* strDl) + { + strRidList = strDl; + } /** @brief Set the binary operator for the filter predicate (BOP_AND or BOP_OR). * * Set the binary operator for the filter predicate (BOP_AND or BOP_OR). */ - void setBOP(int8_t BOP); - - /** @brief Set the output type. - * - * Set the output type (1 = RID, 2 = Token, 3 = Both). - */ - void setOutputType(int8_t OutputType); + void setBOP(int8_t BOP) + { + fBOP = BOP; + } /** @brief Set the swallowRows flag. * * * If true, no rows will be inserted to the output datalists. */ - void setSwallowRows(const bool swallowRows); - + void setSwallowRows(const bool swallowRows) + { + fSwallowRows = swallowRows; + } /** @brief Get the swallowRows flag. * * @@ -263,10 +218,6 @@ class pColStep : public JobStep, public PrimitiveMsg return fColType; } void appendFilter(const messageqcpp::ByteStream& filter, unsigned count); - uint32_t flushInterval() const - { - return fFlushInterval; - } bool getFeederFlag() const { return isFilterFeeder; @@ -276,14 +227,6 @@ class pColStep : public JobStep, public PrimitiveMsg { isFilterFeeder = filterFeeder; } - virtual uint64_t phyIOCount() const - { - return fPhysicalIO; - } - virtual uint64_t cacheIOCount() const - { - return fCacheIO; - } virtual uint64_t msgsRcvdCount() const { return msgsRecvd; @@ -329,14 +272,6 @@ class pColStep : public JobStep, public PrimitiveMsg */ explicit pColStep(); - /** @brief StartPrimitiveThread - * Utility function to start worker thread that sends primitive messages - */ - void startPrimitiveThread(); - /** @brief StartAggregationThread - * Utility function to start worker thread that receives result aggregation from primitive servers - */ - void startAggregationThread(); uint64_t getLBID(uint64_t rid, bool& scan); uint64_t getFBO(uint64_t lbid); @@ -347,7 +282,6 @@ class pColStep : public JobStep, public PrimitiveMsg execplan::CalpontSystemCatalog::ColType fColType; uint32_t fFilterCount; int8_t fBOP; - int8_t fOutputType; uint16_t realWidth; DataList_t* ridList; StrDataList* strRidList; @@ -359,29 +293,18 @@ class pColStep : public JobStep, public PrimitiveMsg bool finishedSending, recvWaiting, fIsDict; bool isEM; int64_t ridCount; - uint32_t fFlushInterval; // @bug 663 - Added fSwallowRows for calpont.caltrace(16) which is TRACE_FLAGS::TRACE_NO_ROWS4. // Running with this one will swallow rows at projection. bool fSwallowRows; - uint32_t fProjectBlockReqLimit; // max number of rids to send in a scan - // request to primproc - uint32_t fProjectBlockReqThreshold; // min level of rids backlog before - // consumer will tell producer to send - // more rids scan requests to primproc - volatile bool fStopSending; bool isFilterFeeder; - uint64_t fPhysicalIO; // total physical I/O count - uint64_t fCacheIO; // total cache I/O count uint64_t fNumBlksSkipped; // total number of block scans skipped due to CP uint64_t fMsgBytesIn; // total byte count for incoming messages uint64_t fMsgBytesOut; // total byte count for outcoming messages BRM::DBRM dbrm; - // boost::shared_ptr cThread; //consumer thread - // boost::shared_ptr pThread; //producer thread boost::mutex mutex; boost::condition condvar; boost::condition flushed; @@ -413,7 +336,7 @@ class pColStep : public JobStep, public PrimitiveMsg * c) send messages to the primitive server as quickly as possible */ -class pColScanStep : public JobStep, public PrimitiveMsg +class pColScanStep : public JobStep { public: /** @brief pColScanStep constructor @@ -422,38 +345,25 @@ class pColScanStep : public JobStep, public PrimitiveMsg const execplan::CalpontSystemCatalog::ColType& ct, const JobInfo& jobInfo); pColScanStep(const pColStep& rhs); - ~pColScanStep(); + ~pColScanStep(){} /** @brief Starts processing. * * Starts processing. */ - virtual void run(); + virtual void run(){} /** @brief Sync's the caller with the end of execution. * * Does nothing. Returns when this instance is finished. */ - virtual void join(); + virtual void join(){} virtual bool isDictCol() const { return fIsDict; }; - /** @brief The main loop for the send-side thread - * - * The main loop for the primitive-issuing thread. Don't call it directly. - */ - void sendPrimitiveMessages(); - - /** @brief The main loop for the recv-side thread - * - * The main loop for the receive-side thread. Don't call it directly. - */ - using PrimitiveMsg::receivePrimitiveMessages; - void receivePrimitiveMessages(uint64_t i = 0); - /** @brief Add a filter when the column is a 4-byte float type * * Add a filter when the column is a 4-byte float type @@ -472,7 +382,11 @@ class pColScanStep : public JobStep, public PrimitiveMsg * Set the binary operator for the filter predicates (BOP_AND or BOP_OR). * It is initialized to OR. */ - void setBOP(int8_t BOP); // AND or OR + void setBOP(int8_t BOP) // AND or OR + { + fBOP = BOP; + } + int8_t BOP() const { return fBOP; @@ -496,17 +410,6 @@ class pColScanStep : public JobStep, public PrimitiveMsg return fFilterString; } - void setSingleThread(bool b); - bool getSingleThread() - { - return fSingleThread; - } - - /** @brief Set the output type. - * - * Set the output type (1 = RID, 2 = Token, 3 = Both).pColScan - */ - void setOutputType(int8_t OutputType); uint32_t filterCount() const { return fFilterCount; @@ -532,18 +435,6 @@ class pColScanStep : public JobStep, public PrimitiveMsg return fRm; } - virtual uint64_t phyIOCount() const - { - return fPhysicalIO; - } - virtual uint64_t cacheIOCount() const - { - return fCacheIO; - } - virtual uint64_t msgsRcvdCount() const - { - return recvCount; - } virtual uint64_t msgBytesIn() const { return fMsgBytesIn; @@ -599,18 +490,12 @@ class pColScanStep : public JobStep, public PrimitiveMsg // pColScanStep& operator=(const pColScanStep& rhs); typedef boost::shared_ptr SPTHD; - void startPrimitiveThread(); - void startAggregationThread(); - void initializeConfigParms(); - void sendAPrimitiveMessage(ISMPacketHeader& ism, BRM::LBID_t msgLbidStart, uint32_t msgLbidCount); uint64_t getFBO(uint64_t lbid); bool isEmptyVal(const uint8_t* val8) const; ResourceManager* fRm; ColByScanRangeRequestHeader fMsgHeader; SPTHD fConsumerThread; - /// number of threads on the receive side - uint32_t fNumThreads; SPTHD* fProducerThread; messageqcpp::ByteStream fFilterString; @@ -619,16 +504,10 @@ class pColScanStep : public JobStep, public PrimitiveMsg execplan::CalpontSystemCatalog::OID fTableOid; execplan::CalpontSystemCatalog::ColType fColType; int8_t fBOP; - int8_t fOutputType; - uint32_t sentCount; - uint32_t recvCount; BRM::LBIDRange_v lbidRanges; BRM::DBRM dbrm; SP_LBIDList lbidList; - boost::mutex mutex; - boost::mutex dlMutex; - boost::mutex cpMutex; boost::condition condvar; boost::condition condvarWakeupProducer; bool finishedSending, sendWaiting, rDoNothing, fIsDict; @@ -638,17 +517,7 @@ class pColScanStep : public JobStep, public PrimitiveMsg uint32_t extentSize, divShift, ridsPerBlock, rpbShift, numExtents; // config::Config *fConfig; - uint32_t fScanLbidReqLimit; // max number of LBIDs to send in a scan - // request to primproc - uint32_t fScanLbidReqThreshold; // min level of scan LBID backlog before - // consumer will tell producer to send - // more LBID scan requests to primproc - - bool fStopSending; - bool fSingleThread; bool isFilterFeeder; - uint64_t fPhysicalIO; // total physical I/O count - uint64_t fCacheIO; // total cache I/O count uint64_t fNumBlksSkipped; // total number of block scans skipped due to CP uint64_t fMsgBytesIn; // total byte count for incoming messages uint64_t fMsgBytesOut; // total byte count for outcoming messages @@ -667,35 +536,10 @@ class pColScanStep : public JobStep, public PrimitiveMsg friend class TupleBPS; }; -#if 0 -class pIdxStep : public JobStep -{ -public: - /** @brief pIdxStep constructor - * @param in the inputAssociation pointer - * @param out the outputAssociation pointer - * @param ec the DistributedEngineComm pointer - */ - pIdxStep(JobStepAssociation* in, JobStepAssociation* out, DistributedEngineComm* ec); - /** @brief virtual void Run method - */ - virtual void run(); -private: - pIdxStep(); - void startPrimitveThread(); - void startAggregationThread(); - -protected: - DistributedEngineComm* fDec; - JobStepAssociation* fInputJobStepAssociation; - JobStepAssociation* fOutputJobStepAssociation; -}; -#endif - /** @brief class pDictionaryStep * */ -class pDictionaryStep : public JobStep, public PrimitiveMsg +class pDictionaryStep : public JobStep { public: /** @brief pDictionaryStep constructor @@ -704,17 +548,22 @@ class pDictionaryStep : public JobStep, public PrimitiveMsg pDictionaryStep(execplan::CalpontSystemCatalog::OID oid, execplan::CalpontSystemCatalog::OID tabelOid, const execplan::CalpontSystemCatalog::ColType& ct, const JobInfo& jobInfo); - virtual ~pDictionaryStep(); + virtual ~pDictionaryStep(){} /** @brief virtual void Run method */ - virtual void run(); - virtual void join(); + virtual void run(){} + virtual void join(){} // void setOutList(StringDataList* rids); - void setInputList(DataList_t* rids); - void setBOP(int8_t b); - void sendPrimitiveMessages(); - void receivePrimitiveMessages(); + void setInputList(DataList_t* rids) + { + requestList = rids; + } + + void setBOP(int8_t b) + { + fBOP = b; + } virtual const std::string toString() const; @@ -735,14 +584,6 @@ class pDictionaryStep : public JobStep, public PrimitiveMsg { return fTableOid; } - virtual uint64_t phyIOCount() const - { - return fPhysicalIO; - } - virtual uint64_t cacheIOCount() const - { - return fCacheIO; - } virtual uint64_t msgsRcvdCount() const { return msgsRecvd; @@ -780,8 +621,6 @@ class pDictionaryStep : public JobStep, public PrimitiveMsg private: pDictionaryStep(); - void startPrimitiveThread(); - void startAggregationThread(); boost::shared_ptr sysCat; execplan::CalpontSystemCatalog::OID fOid; @@ -804,8 +643,6 @@ class pDictionaryStep : public JobStep, public PrimitiveMsg boost::mutex mutex; boost::condition condvar; uint32_t fInterval; - uint64_t fPhysicalIO; // total physical I/O count - uint64_t fCacheIO; // total cache I/O count uint64_t fMsgBytesIn; // total byte count for incoming messages uint64_t fMsgBytesOut; // total byte count for outcoming messages uint32_t uniqueID; @@ -828,7 +665,7 @@ class pDictionaryStep : public JobStep, public PrimitiveMsg /** @brief class pDictionaryScan * */ -class pDictionaryScan : public JobStep, public PrimitiveMsg +class pDictionaryScan : public JobStep { public: /** @brief pDictionaryScan constructor @@ -953,6 +790,7 @@ class pDictionaryScan : public JobStep, public PrimitiveMsg private: pDictionaryScan(); + uint16_t planFlagsToPrimFlags(uint32_t planFlags); void startPrimitiveThread(); void startAggregationThread(); void initializeConfigParms(); @@ -990,12 +828,10 @@ class pDictionaryScan : public JobStep, public PrimitiveMsg uint64_t extentSize; uint64_t divShift; uint64_t numExtents; - uint32_t fScanLbidReqLimit; // max number of LBIDs to send in a scan // request to primproc uint32_t fScanLbidReqThreshold; // min level of scan LBID backlog before // consumer will tell producer to send bool fStopSending; - bool fSingleThread; uint64_t fPhysicalIO; // total physical I/O count uint64_t fCacheIO; // total cache I/O count uint64_t fMsgBytesIn; // total byte count for incoming messages @@ -1018,7 +854,7 @@ class pDictionaryScan : public JobStep, public PrimitiveMsg void destroyEqualityFilter(); }; -class BatchPrimitive : public JobStep, public PrimitiveMsg, public DECEventListener +class BatchPrimitive : public JobStep, public DECEventListener { public: BatchPrimitive(const JobInfo& jobInfo) : JobStep(jobInfo) @@ -1054,9 +890,10 @@ class BatchPrimitive : public JobStep, public PrimitiveMsg, public DECEventListe struct _CPInfo { - _CPInfo(int64_t MIN, int64_t MAX, uint64_t l, bool val) : min(MIN), max(MAX), LBID(l), valid(val){}; + _CPInfo(int64_t MIN, int64_t MAX, uint64_t l, bool dictScan, bool val) + : min(MIN), max(MAX), LBID(l), valid(val), dictScan(dictScan) {}; _CPInfo(int128_t BIGMIN, int128_t BIGMAX, uint64_t l, bool val) - : bigMin(BIGMIN), bigMax(BIGMAX), LBID(l), valid(val){}; + : bigMin(BIGMIN), bigMax(BIGMAX), LBID(l), valid(val), dictScan(false) {}; union { int128_t bigMin; @@ -1069,6 +906,7 @@ struct _CPInfo }; uint64_t LBID; bool valid; + bool dictScan; }; /** @brief class TupleBPS @@ -1404,8 +1242,8 @@ class TupleBPS : public BatchPrimitive, public TupleDeliveryStep SP_LBIDList lbidList; uint64_t ridsRequested; uint64_t totalMsgs; - volatile uint64_t msgsSent; - volatile uint64_t msgsRecvd; + uint64_t msgsSent; + uint64_t msgsRecvd; volatile bool finishedSending; bool firstRead; bool sendWaiting; @@ -1700,7 +1538,7 @@ class FilterStep : public JobStep /** @brief class PassThruStep * */ -class PassThruStep : public JobStep, public PrimitiveMsg +class PassThruStep : public JobStep { typedef std::pair element_t; @@ -1839,5 +1677,3 @@ class PseudoColStep : public pColStep }; } // namespace joblist - -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/resourcemanager.cpp b/dbcon/joblist/resourcemanager.cpp index 94d5193e0..8edaf9c35 100644 --- a/dbcon/joblist/resourcemanager.cpp +++ b/dbcon/joblist/resourcemanager.cpp @@ -1,4 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2022 Mariadb Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -29,9 +30,6 @@ #include using namespace std; -#include -using namespace boost; - #include "resourcemanager.h" #include "jl_logger.h" @@ -43,44 +41,29 @@ using namespace config; namespace joblist { -// const string ResourceManager::fExeMgrStr("ExeMgr1"); -const string ResourceManager::fHashJoinStr("HashJoin"); -const string ResourceManager::fHashBucketReuseStr("HashBucketReuse"); -const string ResourceManager::fJobListStr("JobList"); -const string ResourceManager::fPrimitiveServersStr("PrimitiveServers"); -// const string ResourceManager::fSystemConfigStr("SystemConfig"); -const string ResourceManager::fTupleWSDLStr("TupleWSDL"); -const string ResourceManager::fZDLStr("ZDL"); -const string ResourceManager::fExtentMapStr("ExtentMap"); -// const string ResourceManager::fDMLProcStr("DMLProc"); -// const string ResourceManager::fBatchInsertStr("BatchInsert"); -const string ResourceManager::fOrderByLimitStr("OrderByLimit"); -const string ResourceManager::fRowAggregationStr("RowAggregation"); - ResourceManager* ResourceManager::fInstance = NULL; boost::mutex mx; -ResourceManager* ResourceManager::instance(bool runningInExeMgr) +ResourceManager* ResourceManager::instance(bool runningInExeMgr, config::Config* aConfig) { boost::mutex::scoped_lock lk(mx); if (!fInstance) - fInstance = new ResourceManager(runningInExeMgr); + fInstance = new ResourceManager(runningInExeMgr, aConfig); return fInstance; } -ResourceManager::ResourceManager(bool runningInExeMgr) +ResourceManager::ResourceManager(bool runningInExeMgr, config::Config* aConfig) : fExeMgrStr("ExeMgr1") , fSystemConfigStr("SystemConfig") , fDMLProcStr("DMLProc") , fBatchInsertStr("BatchInsert") - , fConfig(Config::makeConfig()) + , fConfig(aConfig == nullptr ? Config::makeConfig() : aConfig) , fNumCores(8) , fHjNumThreads(defaultNumThreads) , fJlProcessorThreadsPerScan(defaultProcessorThreadsPerScan) , fJlNumScanReceiveThreads(defaultScanReceiveThreads) - , fTwNumThreads(defaultNumThreads) , fJlMaxOutstandingRequests(defaultMaxOutstandingRequests) , fHJUmMaxMemorySmallSideDistributor( fHashJoinStr, "UmMaxMemorySmallSide", @@ -117,7 +100,6 @@ ResourceManager::ResourceManager(bool runningInExeMgr) { fHjNumThreads = fNumCores; fJlNumScanReceiveThreads = fNumCores; - fTwNumThreads = fNumCores; } // possibly override any calculated values @@ -155,11 +137,6 @@ ResourceManager::ResourceManager(bool runningInExeMgr) fDECConnectionsPerQuery = (fDECConnectionsPerQuery) ? fDECConnectionsPerQuery : getPsConnectionsPerPrimProc(); - temp = getIntVal(fTupleWSDLStr, "NumThreads", -1); - - if (temp > 0) - fTwNumThreads = temp; - pmJoinMemLimit = getUintVal(fHashJoinStr, "PmMaxMemorySmallSide", defaultHJPmMaxMemorySmallSide); // Need to use different limits if this instance isn't running on the UM, @@ -336,106 +313,6 @@ void ResourceManager::logResourceChangeMessage(logging::LOG_TYPE logType, uint32 log.logMessage(logType, mid, args, logging::LoggingID(5, sessionID)); } -void ResourceManager::emServerThreads() -{ -} -void ResourceManager::emServerQueueSize() -{ -} -void ResourceManager::emSecondsBetweenMemChecks() -{ -} -void ResourceManager::emMaxPct() -{ -} -void ResourceManager::emPriority() -{ -} -void ResourceManager::emExecQueueSize() -{ -} - -void ResourceManager::hjNumThreads() -{ -} -void ResourceManager::hjMaxBuckets() -{ -} -void ResourceManager::hjMaxElems() -{ -} -void ResourceManager::hjFifoSizeLargeSide() -{ -} -void ResourceManager::hjPmMaxMemorySmallSide() -{ -} - -void ResourceManager::jlFlushInterval() -{ -} -void ResourceManager::jlFifoSize() -{ -} -void ResourceManager::jlScanLbidReqLimit() -{ -} -void ResourceManager::jlScanLbidReqThreshold() -{ -} -void ResourceManager::jlProjectBlockReqLimit() -{ -} -void ResourceManager::jlProjectBlockReqThreshold() -{ -} -void ResourceManager::jlNumScanReceiveThreads() -{ -} - -void ResourceManager::psCount() -{ -} -void ResourceManager::psConnectionsPerPrimProc() -{ -} -void ResourceManager::psLBID_Shift() -{ -} - -void ResourceManager::scTempDiskPath() -{ -} -void ResourceManager::scTempSaveSize() -{ -} -void ResourceManager::scWorkingDir() -{ -} - -void ResourceManager::twMaxSize() -{ -} -void ResourceManager::twInitialCapacity() -{ -} -void ResourceManager::twMaxBuckets() -{ -} -void ResourceManager::twNumThreads() -{ -} -void ResourceManager::zdl_MaxElementsInMem() -{ -} -void ResourceManager::zdl_MaxElementsPerBucket() -{ -} - -void ResourceManager::hbrPredicate() -{ -} - bool ResourceManager::getMysqldInfo(std::string& h, std::string& u, std::string& w, unsigned int& p) const { static const std::string hostUserUnassignedValue("unassigned"); diff --git a/dbcon/joblist/resourcemanager.h b/dbcon/joblist/resourcemanager.h index ff712419b..72f32dfb7 100644 --- a/dbcon/joblist/resourcemanager.h +++ b/dbcon/joblist/resourcemanager.h @@ -1,4 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2022 Mariadb Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -51,31 +52,18 @@ const uint32_t defaultNumThreads = 8; // joblistfactory const uint32_t defaultFlushInterval = 8 * 1024; const uint32_t defaultFifoSize = 10; -const uint32_t defaultHJFifoSizeLargeSide = 128; const uint64_t defaultHJMaxElems = 512 * 1024; // hashjoin uses 8192 const int defaultHJMaxBuckets = 32; // hashjoin uses 4 const uint64_t defaultHJPmMaxMemorySmallSide = 1 * 1024 * 1024 * 1024ULL; const uint64_t defaultHJUmMaxMemorySmallSide = 4 * 1024 * 1024 * 1024ULL; -const uint32_t defaultTempSaveSize = defaultHJMaxElems; const uint64_t defaultTotalUmMemory = 8 * 1024 * 1024 * 1024ULL; -const uint64_t defaultHUATotalMem = 8 * 1024 * 1024 * 1024ULL; - -const uint32_t defaultTupleDLMaxSize = 64 * 1024; - const uint32_t defaultJLThreadPoolSize = 100; // pcolscan.cpp -const uint32_t defaultScanLbidReqLimit = 10000; const uint32_t defaultScanLbidReqThreshold = 5000; const uint32_t defaultLogicalBlocksPerScan = 1024; // added for bug 1264. -const uint32_t defaultScanBlockThreshhold = 10000; // in jobstep.h - const uint32_t defaultScanReceiveThreads = 8; -// pcolstep.cpp -const uint32_t defaultProjectBlockReqLimit = 32 * 1024; -const uint32_t defaultProjectBlockReqThreshold = 16 * 1024; // 256 in jobstep.h - // BatchPrimitiveStep const uint32_t defaultRequestSize = 1; const uint32_t defaultMaxOutstandingRequests = 20; @@ -95,15 +83,6 @@ const uint64_t defaultMaxBPPSendQueue = 250000000; // ~250MB // bucketreuse const std::string defaultTempDiskPath = "/tmp"; -const std::string defaultWorkingDir = "."; //"/tmp"; - -// largedatalist -const uint32_t defaultLDLMaxElements = 32 * 1024 * 1024; - -// zdl -const uint64_t defaultMaxElementsInMem = 32 * 1024 * 1024; -const uint64_t defaultNumBuckets = 128; -const uint64_t defaultMaxElementsPerBuckert = 16 * 1024 * 1024; const int defaultEMServerThreads = 50; const int defaultEMSecondsBetweenMemChecks = 1; @@ -111,11 +90,8 @@ const int defaultEMMaxPct = 95; const int defaultEMPriority = 21; // @Bug 3385 const int defaultEMExecQueueSize = 20; -const uint64_t defaultInitialCapacity = 1024 * 1024; -const int defaultTWMaxBuckets = 256; const int defaultPSCount = 0; const int defaultConnectionsPerPrimProc = 1; -const uint32_t defaultLBID_Shift = 13; const uint64_t defaultExtentRows = 8 * 1024 * 1024; // DMLProc @@ -130,13 +106,8 @@ const uint64_t defaultRowsPerBatch = 10000; /* HJ CP feedback, see bug #1465 */ const uint32_t defaultHjCPUniqueLimit = 100; -// Order By and Limit -const uint64_t defaultOrderByLimitMaxMemory = 1 * 1024 * 1024 * 1024ULL; - const uint64_t defaultDECThrottleThreshold = 200000000; // ~200 MB -const uint8_t defaultUseCpimport = 1; - const bool defaultAllowDiskAggregation = false; /** @brief ResourceManager @@ -149,11 +120,8 @@ class ResourceManager /** @brief ctor * */ - EXPORT ResourceManager(bool runningInExeMgr = false); - static ResourceManager* instance(bool runningInExeMgr = false); - // ResourceManager(const config::Config *cf); - // ResourceManager(const std::string& config); - // passed by ExeMgr and DistributedEngineComm to MessageQueueServer or -Client + EXPORT ResourceManager(bool runningInExeMgr = false, config::Config *aConfig = nullptr); + static ResourceManager* instance(bool runningInExeMgr = false, config::Config *aConfig = nullptr); config::Config* getConfig() { return fConfig; @@ -185,7 +153,7 @@ class ResourceManager { return getUintVal(fExeMgrStr, "MaxPct", defaultEMMaxPct); } - EXPORT int getEmPriority() const; + EXPORT int getEmPriority() const; // FOr Windows only int getEmExecQueueSize() const { return getIntVal(fExeMgrStr, "ExecQueueSize", defaultEMExecQueueSize); @@ -213,10 +181,6 @@ class ResourceManager { return getUintVal(fHashJoinStr, "MaxElems", defaultHJMaxElems); } - uint32_t getHjFifoSizeLargeSide() const - { - return getUintVal(fHashJoinStr, "FifoSizeLargeSide", defaultHJFifoSizeLargeSide); - } uint32_t getHjCPUniqueLimit() const { return getUintVal(fHashJoinStr, "CPUniqueLimit", defaultHjCPUniqueLimit); @@ -234,10 +198,6 @@ class ResourceManager { return getUintVal(fJobListStr, "FifoSize", defaultFifoSize); } - uint32_t getJlScanLbidReqLimit() const - { - return getUintVal(fJobListStr, "ScanLbidReqLimit", defaultScanLbidReqLimit); - } uint32_t getJlScanLbidReqThreshold() const { return getUintVal(fJobListStr, "ScanLbidReqThreshold", defaultScanLbidReqThreshold); @@ -263,14 +223,6 @@ class ResourceManager { return getUintVal(fJobListStr, "LogicalBlocksPerScan", defaultLogicalBlocksPerScan); } - uint32_t getJlProjectBlockReqLimit() const - { - return getUintVal(fJobListStr, "ProjectBlockReqLimit", defaultProjectBlockReqLimit); - } - uint32_t getJlProjectBlockReqThreshold() const - { - return getUintVal(fJobListStr, "ProjectBlockReqThreshold", defaultProjectBlockReqThreshold); - } uint32_t getJlNumScanReceiveThreads() const { return fJlNumScanReceiveThreads; @@ -303,49 +255,15 @@ class ResourceManager { return getUintVal(fPrimitiveServersStr, "ConnectionsPerPrimProc", defaultConnectionsPerPrimProc); } - uint32_t getPsLBID_Shift() const - { - return getUintVal(fPrimitiveServersStr, "LBID_Shift", defaultLBID_Shift); - } - std::string getScTempDiskPath() const { return startup::StartUp::tmpDir(); } - uint64_t getScTempSaveSize() const - { - return getUintVal(fSystemConfigStr, "TempSaveSize", defaultTempSaveSize); - } std::string getScWorkingDir() const { return startup::StartUp::tmpDir(); } - uint32_t getTwMaxSize() const - { - return getUintVal(fTupleWSDLStr, "MaxSize", defaultTupleDLMaxSize); - } - uint64_t getTwInitialCapacity() const - { - return getUintVal(fTupleWSDLStr, "InitialCapacity", defaultInitialCapacity); - } - int getTwMaxBuckets() const - { - return getUintVal(fTupleWSDLStr, "MaxBuckets", defaultTWMaxBuckets); - } - uint8_t getTwNumThreads() const - { - return fTwNumThreads; - } // getUintVal(fTupleWSDLStr, "NumThreads", defaultNumThreads ); } - uint64_t getZdl_MaxElementsInMem() const - { - return getUintVal(fZDLStr, "ZDL_MaxElementsInMem", defaultMaxElementsInMem); - } - uint64_t getZdl_MaxElementsPerBucket() const - { - return getUintVal(fZDLStr, "ZDL_MaxElementsPerBucket", defaultMaxElementsPerBuckert); - } - uint64_t getExtentRows() const { return getUintVal(fExtentMapStr, "ExtentRows", defaultExtentRows); @@ -360,13 +278,6 @@ class ResourceManager return getUintVal(fPrimitiveServersStr, "Count", 1); } - std::vector getHbrPredicate() const - { - std::vector columns; - fConfig->getConfig(fHashBucketReuseStr, "Predicate", columns); - return columns; - } - uint64_t getDMLMaxDeleteRows() const { return getUintVal(fDMLProcStr, "MaxDeleteRows", defaultDMLMaxDeleteRows); @@ -377,17 +288,6 @@ class ResourceManager return getUintVal(fBatchInsertStr, "RowsPerBatch", defaultRowsPerBatch); } - uint8_t getUseCpimport() const - { - int val = getIntVal(fBatchInsertStr, "UseCpimport", defaultUseCpimport); - return val; - } - - uint64_t getOrderByLimitMaxMemory() const - { - return getUintVal(fOrderByLimitStr, "MaxMemory", defaultOrderByLimitMaxMemory); - } - uint64_t getDECThrottleThreshold() const { return getUintVal(fJobListStr, "DECThrottleThreshold", defaultDECThrottleThreshold); @@ -478,32 +378,6 @@ class ResourceManager fHJUmMaxMemorySmallSideDistributor.returnResource(mem); } - EXPORT void jlFlushInterval(); - EXPORT void jlFifoSize(); - EXPORT void jlScanLbidReqLimit(); - EXPORT void jlScanLbidReqThreshold(); - EXPORT void jlProjectBlockReqLimit(); - EXPORT void jlProjectBlockReqThreshold(); - EXPORT void jlNumScanReceiveThreads(); - - EXPORT void psCount(); - EXPORT void psConnectionsPerPrimProc(); - EXPORT void psLBID_Shift(); - - EXPORT void scTempDiskPath(); - EXPORT void scTempSaveSize(); - EXPORT void scWorkingDir(); - - EXPORT void twMaxSize(); - EXPORT void twInitialCapacity(); - EXPORT void twMaxBuckets(); - EXPORT void twNumThreads(); - - EXPORT void zdl_MaxElementsInMem(); - EXPORT void zdl_MaxElementsPerBucket(); - - EXPORT void hbrPredicate(); - void setTraceFlags(uint32_t flags) { fTraceFlags = flags; @@ -598,19 +472,16 @@ class ResourceManager void logMessage(logging::LOG_TYPE logLevel, logging::Message::MessageID mid, uint64_t value = 0, uint32_t sessionId = 0); - /*static const*/ std::string fExeMgrStr; - static const std::string fHashJoinStr; - static const std::string fHashBucketReuseStr; - static const std::string fJobListStr; - static const std::string fPrimitiveServersStr; + std::string fExeMgrStr; + inline static const std::string fHashJoinStr = "HashJoin"; + inline static const std::string fJobListStr = "JobList"; + inline static const std::string fPrimitiveServersStr = "PrimitiveServers"; /*static const*/ std::string fSystemConfigStr; - static const std::string fTupleWSDLStr; - static const std::string fZDLStr; - static const std::string fExtentMapStr; + inline static const std::string fExtentMapStr = "ExtentMap"; /*static const*/ std::string fDMLProcStr; /*static const*/ std::string fBatchInsertStr; - static const std::string fOrderByLimitStr; - static const std::string fRowAggregationStr; + inline static const std::string fOrderByLimitStr = "OrderByLimit"; + inline static const std::string fRowAggregationStr = "RowAggregation"; config::Config* fConfig; static ResourceManager* fInstance; uint32_t fTraceFlags; @@ -619,7 +490,6 @@ class ResourceManager unsigned fHjNumThreads; uint32_t fJlProcessorThreadsPerScan; uint32_t fJlNumScanReceiveThreads; - uint8_t fTwNumThreads; uint32_t fJlMaxOutstandingRequests; /* old HJ support */ diff --git a/dbcon/joblist/rowestimator.cpp b/dbcon/joblist/rowestimator.cpp index 9babae7c3..fa7e8920c 100644 --- a/dbcon/joblist/rowestimator.cpp +++ b/dbcon/joblist/rowestimator.cpp @@ -269,8 +269,8 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, const m float tempFactor = 1.0; uint64_t adjustedMin = 0, adjustedMax = 0; - uint128_t adjustedBigMin, adjustedBigMax; - uint32_t distinctValuesEstimate; + uint128_t adjustedBigMin = 0, adjustedBigMax = 0; + uint32_t distinctValuesEstimate = 0; // Adjust values based on column type and estimate the if (!ct.isWideDecimalType()) diff --git a/dbcon/joblist/subquerystep.cpp b/dbcon/joblist/subquerystep.cpp index a019c24f8..fbc46912e 100644 --- a/dbcon/joblist/subquerystep.cpp +++ b/dbcon/joblist/subquerystep.cpp @@ -533,4 +533,3 @@ void SubAdapterStep::formatMiniStats() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/subquerystep.h b/dbcon/joblist/subquerystep.h index 320e4a438..3c587d170 100644 --- a/dbcon/joblist/subquerystep.h +++ b/dbcon/joblist/subquerystep.h @@ -273,4 +273,3 @@ class SubAdapterStep : public JobStep, public TupleDeliveryStep } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/subquerytransformer.cpp b/dbcon/joblist/subquerytransformer.cpp index 816b6aa72..48f1cc2d5 100644 --- a/dbcon/joblist/subquerytransformer.cpp +++ b/dbcon/joblist/subquerytransformer.cpp @@ -622,4 +622,3 @@ void SimpleScalarTransformer::getScalarResult() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/subquerytransformer.h b/dbcon/joblist/subquerytransformer.h index 26e6e0f29..5f9ea4e86 100644 --- a/dbcon/joblist/subquerytransformer.h +++ b/dbcon/joblist/subquerytransformer.h @@ -223,4 +223,3 @@ class SimpleScalarTransformer : public SubQueryTransformer } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/tuple-bps.cpp b/dbcon/joblist/tuple-bps.cpp index f9a6d7361..f59f31b9b 100644 --- a/dbcon/joblist/tuple-bps.cpp +++ b/dbcon/joblist/tuple-bps.cpp @@ -75,6 +75,7 @@ using namespace rowgroup; #include "querytele.h" using namespace querytele; +#include "columnwidth.h" #include "pseudocolumn.h" //#define DEBUG 1 @@ -990,6 +991,7 @@ void TupleBPS::storeCasualPartitionInfo(const bool estimateRowCounts) vector cpColVec; vector lbidListVec; ColumnCommandJL* colCmd = 0; + bool defaultScanFlag = true; // @bug 2123. We call this earlier in the process for the hash join estimation process now. Return if // we've already done the work. @@ -1001,7 +1003,9 @@ void TupleBPS::storeCasualPartitionInfo(const bool estimateRowCounts) fCPEvaluated = true; if (colCmdVec.size() == 0) - return; + { + defaultScanFlag = false; // no reason to scan if there are no commands. + } for (uint32_t i = 0; i < colCmdVec.size(); i++) { @@ -1027,30 +1031,28 @@ void TupleBPS::storeCasualPartitionInfo(const bool estimateRowCounts) } if (cpColVec.size() == 0) - return; + { + defaultScanFlag = true; // no reason to scan if there are no predicates to evaluate. + } const bool ignoreCP = ((fTraceFlags & CalpontSelectExecutionPlan::IGNORE_CP) != 0); for (uint32_t idx = 0; idx < numExtents; idx++) { - scanFlags[idx] = true; + scanFlags[idx] = defaultScanFlag; - for (uint32_t i = 0; i < cpColVec.size(); i++) + for (uint32_t i = 0; scanFlags[idx] && i < cpColVec.size(); i++) { colCmd = cpColVec[i]; const EMEntry& extent = colCmd->getExtents()[idx]; /* If any column filter eliminates an extent, it doesn't get scanned */ - scanFlags[idx] = - scanFlags[idx] && (ignoreCP || extent.partition.cprange.isValid != BRM::CP_VALID || - lbidListVec[i]->CasualPartitionPredicate( - extent.partition.cprange, &(colCmd->getFilterString()), - colCmd->getFilterCount(), colCmd->getColType(), colCmd->getBOP())); - - if (!scanFlags[idx]) - { - break; - } + scanFlags[idx] = scanFlags[idx] && (extent.colWid <= utils::MAXCOLUMNWIDTH) && // XXX: change to named constant. + (ignoreCP || extent.partition.cprange.isValid != BRM::CP_VALID || + colCmd->getColType().colWidth != extent.colWid || + lbidListVec[i]->CasualPartitionPredicate( + extent.partition.cprange, &(colCmd->getFilterString()), colCmd->getFilterCount(), + colCmd->getColType(), colCmd->getBOP(), colCmd->getIsDict())); } } @@ -2134,9 +2136,10 @@ void TupleBPS::processByteStreamVector(vectorgetRowGroupData(*bs, &fromPrimProc, &validCPData, &lbid, &min, &max, &cachedIO, &physIO, - &touchedBlocks, &unused, threadID, &hasBinaryColumn, fColType); + fBPP->getRowGroupData(*bs, &fromPrimProc, &validCPData, &lbid, &fromDictScan, &min, &max, &cachedIO, + &physIO, &touchedBlocks, &unused, threadID, &hasBinaryColumn, fColType); // Another layer of messiness. Need to refactor this fcn. while (!fromPrimProc.empty() && !cancelled()) @@ -2291,7 +2294,7 @@ void TupleBPS::processByteStreamVector(vectorread_some(uniqueID, fNumThreads, bsv, &flowControlOn); @@ -2469,11 +2474,13 @@ void TupleBPS::receiveMultiPrimitiveMessages() { if (fColType.colWidth > 8) { - lbidList->UpdateMinMax(cpv[i].bigMin, cpv[i].bigMax, cpv[i].LBID, fColType, cpv[i].valid); + lbidList->UpdateMinMax(cpv[i].bigMin, cpv[i].bigMax, cpv[i].LBID, cpv[i].dictScan, fColType, + cpv[i].valid); } else { - lbidList->UpdateMinMax(cpv[i].min, cpv[i].max, cpv[i].LBID, fColType, cpv[i].valid); + lbidList->UpdateMinMax(cpv[i].min, cpv[i].max, cpv[i].LBID, cpv[i].dictScan, fColType, + cpv[i].valid); } } } @@ -3334,4 +3341,3 @@ template bool TupleBPS::compareSingleValue(uint8_t COP, int64_t val1, i template bool TupleBPS::compareSingleValue(uint8_t COP, int128_t val1, int128_t val2) const; } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index dfb8cc4f3..172abd861 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -5962,4 +5962,3 @@ void TupleAggregateStep::formatMiniStats() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/tupleaggregatestep.h b/dbcon/joblist/tupleaggregatestep.h index 92ff9a1eb..b81c8497d 100644 --- a/dbcon/joblist/tupleaggregatestep.h +++ b/dbcon/joblist/tupleaggregatestep.h @@ -224,4 +224,3 @@ class TupleAggregateStep : public JobStep, public TupleDeliveryStep } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/tupleannexstep.cpp b/dbcon/joblist/tupleannexstep.cpp index f6a7d70c2..b93e41378 100644 --- a/dbcon/joblist/tupleannexstep.cpp +++ b/dbcon/joblist/tupleannexstep.cpp @@ -1258,4 +1258,3 @@ void TupleAnnexStep::formatMiniStats() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/tupleannexstep.h b/dbcon/joblist/tupleannexstep.h index 3d043afce..dcdc064c3 100644 --- a/dbcon/joblist/tupleannexstep.h +++ b/dbcon/joblist/tupleannexstep.h @@ -193,4 +193,3 @@ class reservablePQ : private std::priority_queue } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/tupleconstantstep.cpp b/dbcon/joblist/tupleconstantstep.cpp index ee252ec0f..065b087df 100644 --- a/dbcon/joblist/tupleconstantstep.cpp +++ b/dbcon/joblist/tupleconstantstep.cpp @@ -842,4 +842,3 @@ const string TupleConstantBooleanStep::toString() const } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/tupleconstantstep.h b/dbcon/joblist/tupleconstantstep.h index be3480594..ad36b5c9a 100644 --- a/dbcon/joblist/tupleconstantstep.h +++ b/dbcon/joblist/tupleconstantstep.h @@ -189,4 +189,3 @@ class TupleConstantBooleanStep : public TupleConstantStep } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/tuplehashjoin.cpp b/dbcon/joblist/tuplehashjoin.cpp index 191cd9c12..b333db630 100644 --- a/dbcon/joblist/tuplehashjoin.cpp +++ b/dbcon/joblist/tuplehashjoin.cpp @@ -2017,4 +2017,3 @@ void TupleHashJoinStep::abort() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/tuplehashjoin.h b/dbcon/joblist/tuplehashjoin.h index 3bdffb3d9..b5149a6ee 100644 --- a/dbcon/joblist/tuplehashjoin.h +++ b/dbcon/joblist/tuplehashjoin.h @@ -652,4 +652,3 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/tuplehavingstep.cpp b/dbcon/joblist/tuplehavingstep.cpp index 701e77b39..60496895a 100644 --- a/dbcon/joblist/tuplehavingstep.cpp +++ b/dbcon/joblist/tuplehavingstep.cpp @@ -408,4 +408,3 @@ void TupleHavingStep::formatMiniStats() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/tuplehavingstep.h b/dbcon/joblist/tuplehavingstep.h index 1d456b609..b4c185e31 100644 --- a/dbcon/joblist/tuplehavingstep.h +++ b/dbcon/joblist/tuplehavingstep.h @@ -115,4 +115,3 @@ class TupleHavingStep : public ExpressionStep, public TupleDeliveryStep } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/tupleunion.h b/dbcon/joblist/tupleunion.h index 2ac031381..23a00b273 100644 --- a/dbcon/joblist/tupleunion.h +++ b/dbcon/joblist/tupleunion.h @@ -199,7 +199,7 @@ class TupleUnion : public JobStep, public TupleDeliveryStep bool runRan, joinRan; boost::shared_ptr sessionMemLimit; - std::string fTimeZone; + long fTimeZone; }; } // namespace joblist diff --git a/dbcon/joblist/virtualtable.cpp b/dbcon/joblist/virtualtable.cpp index dc9bf275c..8fe1ec674 100644 --- a/dbcon/joblist/virtualtable.cpp +++ b/dbcon/joblist/virtualtable.cpp @@ -158,4 +158,3 @@ const CalpontSystemCatalog::ColType& VirtualTable::columnType(uint32_t i) const } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/windowfunctionstep.cpp b/dbcon/joblist/windowfunctionstep.cpp index 1ab8d3984..ab7b8ca89 100644 --- a/dbcon/joblist/windowfunctionstep.cpp +++ b/dbcon/joblist/windowfunctionstep.cpp @@ -1599,4 +1599,3 @@ void WindowFunctionStep::formatMiniStats() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/windowfunctionstep.h b/dbcon/joblist/windowfunctionstep.h index 2c839a5e6..d1da2aafd 100644 --- a/dbcon/joblist/windowfunctionstep.h +++ b/dbcon/joblist/windowfunctionstep.h @@ -225,4 +225,3 @@ class WindowFunctionStep : public JobStep, public TupleDeliveryStep } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/mysql/ha_autoi.cpp b/dbcon/mysql/ha_autoi.cpp index ddfbcdc22..ac71d0f80 100644 --- a/dbcon/mysql/ha_autoi.cpp +++ b/dbcon/mysql/ha_autoi.cpp @@ -18,38 +18,42 @@ /* * $Id$ */ +#include +#include +#include +#include bool parseAutoincrementTableComment(std::string comment, uint64_t& startValue, std::string& columnName) { - algorithm::to_upper(comment); - regex compat("[[:space:]]*AUTOINCREMENT[[:space:]]*=[[:space:]]*", regex_constants::extended); + boost::algorithm::to_upper(comment); + std::regex compat("[[:space:]]*AUTOINCREMENT[[:space:]]*=[[:space:]]*", std::regex_constants::extended); bool autoincrement = false; columnName = ""; - boost::match_results what; + std::match_results what; std::string::const_iterator start, end; start = comment.begin(); end = comment.end(); - boost::match_flag_type flags = boost::match_default; + std::regex_constants::match_flag_type flags = std::regex_constants::match_default; - if (boost::regex_search(start, end, what, compat, flags)) + if (std::regex_search(start, end, what, compat, flags)) { if (what[0].matched) { // string params (what[0].first, what[0].second); - string params(&(*(what[0].second))); + std::string params(&(*(what[0].second))); unsigned i = params.find_first_of(","); if (i <= params.length()) { // check whether there is more autoincrement column - string restComment = params.substr(i + 1, params.length()); + std::string restComment = params.substr(i + 1, params.length()); start = restComment.begin(); end = restComment.end(); - if (boost::regex_search(start, end, what, compat, flags)) + if (std::regex_search(start, end, what, compat, flags)) { if (what[0].matched) - throw runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_INVALID_NUMBER_AUTOINCREMENT)); + throw runtime_error(logging::IDBErrorInfo::instance()->errorMsg(ERR_INVALID_NUMBER_AUTOINCREMENT)); } columnName = params.substr(0, i); @@ -112,25 +116,25 @@ bool parseAutoincrementTableComment(std::string comment, uint64_t& startValue, s bool parseAutoincrementColumnComment(std::string comment, uint64_t& startValue) { - algorithm::to_upper(comment); - regex compat("[[:space:]]*AUTOINCREMENT[[:space:]]*", regex_constants::extended); + boost::algorithm::to_upper(comment); + std::regex compat("[[:space:]]*AUTOINCREMENT[[:space:]]*", std::regex_constants::extended); bool autoincrement = false; - boost::match_results what; + std::match_results what; std::string::const_iterator start, end; start = comment.begin(); end = comment.end(); - boost::match_flag_type flags = boost::match_default; + std::regex_constants::match_flag_type flags = std::regex_constants::match_default; - if (boost::regex_search(start, end, what, compat, flags)) + if (std::regex_search(start, end, what, compat, flags)) { if (what[0].matched) { - string params(&(*(what[0].second))); + std::string params(&(*(what[0].second))); unsigned i = params.find_first_of(","); if (i <= params.length()) { - string startVal = params.substr(i + 1, params.length()); + std::string startVal = params.substr(i + 1, params.length()); // get rid of possible empty space i = startVal.find_first_not_of(" "); @@ -160,7 +164,7 @@ bool parseAutoincrementColumnComment(std::string comment, uint64_t& startValue) // (no digits) || (more chars) || (other errors & value = 0) if ((ep == str) || (*ep != '\0') || (errno != 0)) { - throw runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_INVALID_START_VALUE)); + throw runtime_error(logging::IDBErrorInfo::instance()->errorMsg(ERR_INVALID_START_VALUE)); } } } @@ -176,4 +180,3 @@ bool parseAutoincrementColumnComment(std::string comment, uint64_t& startValue) return autoincrement; } -// vim:ts=4 sw=4: diff --git a/dbcon/mysql/ha_exists_sub.cpp b/dbcon/mysql/ha_exists_sub.cpp index 36c44f13d..da934318d 100644 --- a/dbcon/mysql/ha_exists_sub.cpp +++ b/dbcon/mysql/ha_exists_sub.cpp @@ -96,7 +96,7 @@ execplan::ParseTree* ExistsSub::transform() csep->subType(CalpontSelectExecutionPlan::EXISTS_SUBS); // gwi for the sub query - gp_walk_info gwi; + gp_walk_info gwi(fGwip.timeZone); gwi.thd = fGwip.thd; gwi.subQuery = this; diff --git a/dbcon/mysql/ha_from_sub.cpp b/dbcon/mysql/ha_from_sub.cpp index a3065d6c9..c836a5cd6 100644 --- a/dbcon/mysql/ha_from_sub.cpp +++ b/dbcon/mysql/ha_from_sub.cpp @@ -44,7 +44,7 @@ using namespace execplan; namespace cal_impl_if { -void derivedTableOptimization(THD* thd, SCSEP& csep) +void derivedTableOptimization(gp_walk_info* gwip, SCSEP& csep) { // @bug5634. replace the unused column with ConstantColumn from derived table column list, // ExeMgr will not project ConstantColumn. Only count for local derived column. @@ -135,8 +135,7 @@ void derivedTableOptimization(THD* thd, SCSEP& csep) else { cols[i].reset(new ConstantColumn(val)); - (reinterpret_cast(cols[i].get())) - ->timeZone(thd->variables.time_zone->get_name()->ptr()); + (reinterpret_cast(cols[i].get()))->timeZone(gwip->timeZone); } for (uint j = 0; j < unionColVec.size(); j++) @@ -156,8 +155,7 @@ void derivedTableOptimization(THD* thd, SCSEP& csep) else { unionColVec[j][i].reset(new ConstantColumn(val)); - (reinterpret_cast(unionColVec[j][i].get())) - ->timeZone(thd->variables.time_zone->get_name()->ptr()); + (reinterpret_cast(unionColVec[j][i].get()))->timeZone(gwip->timeZone); } } } @@ -173,15 +171,13 @@ void derivedTableOptimization(THD* thd, SCSEP& csep) if (!cols.empty()) { cols[0].reset(new ConstantColumn(val)); - (reinterpret_cast(cols[0].get())) - ->timeZone(thd->variables.time_zone->get_name()->ptr()); + (reinterpret_cast(cols[0].get()))->timeZone(gwip->timeZone); nonConstCols.push_back(cols[0]); for (uint j = 0; j < unionColVec.size(); j++) { unionColVec[j][0].reset(new ConstantColumn(val)); - (reinterpret_cast(unionColVec[j][0].get())) - ->timeZone(thd->variables.time_zone->get_name()->ptr()); + (reinterpret_cast(unionColVec[j][0].get()))->timeZone(gwip->timeZone); nonConstUnionColVec[j].push_back(unionColVec[j][0]); } } @@ -229,7 +225,7 @@ void derivedTableOptimization(THD* thd, SCSEP& csep) if (horizontalOptimization && pt) { pt->walk(setDerivedTable); - setDerivedFilter(thd, pt, derivedTbFilterMap, derivedTbList); + setDerivedFilter(gwip, pt, derivedTbFilterMap, derivedTbList); csep->filters(pt); } @@ -301,7 +297,7 @@ void derivedTableOptimization(THD* thd, SCSEP& csep) for (uint i = 0; i < csep->subSelectList().size(); i++) { SCSEP subselect(boost::dynamic_pointer_cast(csep->subSelectList()[i])); - derivedTableOptimization(thd, subselect); + derivedTableOptimization(gwip, subselect); } } @@ -339,7 +335,7 @@ void setDerivedTable(execplan::ParseTree* n) } } -ParseTree* setDerivedFilter(THD* thd, ParseTree*& n, map& filterMap, +ParseTree* setDerivedFilter(gp_walk_info* gwip, ParseTree*& n, map& filterMap, CalpontSelectExecutionPlan::SelectList& derivedTbList) { if (!(n->derivedTable().empty())) @@ -381,7 +377,7 @@ ParseTree* setDerivedFilter(THD* thd, ParseTree*& n, map& fi int64_t val = 1; n = new ParseTree(new ConstantColumn(val)); - (dynamic_cast(n->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(n->data()))->timeZone(gwip->timeZone); } else { @@ -397,10 +393,10 @@ ParseTree* setDerivedFilter(THD* thd, ParseTree*& n, map& fi ParseTree* rhs = n->right(); if (lhs) - n->left(setDerivedFilter(thd, lhs, filterMap, derivedTbList)); + n->left(setDerivedFilter(gwip, lhs, filterMap, derivedTbList)); if (rhs) - n->right(setDerivedFilter(thd, rhs, filterMap, derivedTbList)); + n->right(setDerivedFilter(gwip, rhs, filterMap, derivedTbList)); } } @@ -428,7 +424,7 @@ SCSEP FromSubQuery::transform() csep->subType(CalpontSelectExecutionPlan::FROM_SUBS); // gwi for the sub query - gp_walk_info gwi; + gp_walk_info gwi(fGwip.timeZone); gwi.thd = fGwip.thd; gwi.subQuery = this; gwi.viewName = fGwip.viewName; diff --git a/dbcon/mysql/ha_in_sub.cpp b/dbcon/mysql/ha_in_sub.cpp index cfb4fb3be..df4017165 100644 --- a/dbcon/mysql/ha_in_sub.cpp +++ b/dbcon/mysql/ha_in_sub.cpp @@ -151,7 +151,7 @@ execplan::ParseTree* InSub::transform() csep->subType(CalpontSelectExecutionPlan::IN_SUBS); // gwi for the sub query - gp_walk_info gwi; + gp_walk_info gwi(fGwip.timeZone); gwi.thd = fGwip.thd; gwi.subQuery = this; diff --git a/dbcon/mysql/ha_mcs.cpp b/dbcon/mysql/ha_mcs.cpp index dd10c9f36..a061c5abb 100644 --- a/dbcon/mysql/ha_mcs.cpp +++ b/dbcon/mysql/ha_mcs.cpp @@ -163,6 +163,8 @@ ha_mcs::ha_mcs(handlerton* hton, TABLE_SHARE* table_arg) HA_CAN_TABLE_CONDITION_PUSHDOWN | HA_CAN_DIRECT_UPDATE_AND_DELETE) , m_lock_type(F_UNLCK) { + const char* timeZone = current_thd->variables.time_zone->get_name()->ptr(); + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &time_zone); } /** @@ -318,7 +320,7 @@ int ha_mcs::write_row(const uchar* buf) int rc; try { - rc = ha_mcs_impl_write_row(buf, table, rows_changed); + rc = ha_mcs_impl_write_row(buf, table, rows_changed, time_zone); } catch (std::runtime_error& e) { @@ -652,7 +654,7 @@ int ha_mcs::rnd_next(uchar* buf) int rc; try { - rc = ha_mcs_impl_rnd_next(buf, table); + rc = ha_mcs_impl_rnd_next(buf, table, time_zone); } catch (std::runtime_error& e) { diff --git a/dbcon/mysql/ha_mcs.h b/dbcon/mysql/ha_mcs.h index b87414d81..6e6301564 100644 --- a/dbcon/mysql/ha_mcs.h +++ b/dbcon/mysql/ha_mcs.h @@ -51,6 +51,7 @@ class ha_mcs : public handler // call on Ubuntu18. std::vector condStack; int m_lock_type; + long time_zone; int impl_external_lock(THD* thd, TABLE* table, int lock_type); int impl_rnd_init(TABLE* table, const std::vector& condStack); diff --git a/dbcon/mysql/ha_mcs_client_udfs.cpp b/dbcon/mysql/ha_mcs_client_udfs.cpp index 2ea1a68a2..bdaec9ed4 100644 --- a/dbcon/mysql/ha_mcs_client_udfs.cpp +++ b/dbcon/mysql/ha_mcs_client_udfs.cpp @@ -1,5 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2016 MariaDB Corporation + Copyright (C) 2016-2022 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -52,6 +52,8 @@ extern "C" const char* SetParmsPrelude = "Updated "; const char* SetParmsError = "Invalid parameter: "; const char* InvalidParmSize = "Invalid parameter size: Input value cannot be larger than "; + const char* MsgEMIndexSizeInitErrMsg = "mcs_emindex_size() takes no arguments"; + const char* MsgEMIndexFreeInitErrMsg = "mcs_emindex_free() takes no arguments"; const size_t Plen = strlen(SetParmsPrelude); const size_t Elen = strlen(SetParmsError); @@ -66,10 +68,53 @@ extern "C" return str; } + my_bool setparms_init(UDF_INIT* initid, UDF_ARGS* args, char* message, const char* funcname) + { + if (args->arg_count != 2 || args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT) + { + sprintf(message, "%s() requires two string arguments", funcname); + return 1; + } + + initid->max_length = MAXSTRINGLENGTH; + + char valuestr[MAXSTRINGLENGTH]; + size_t vlen = args->lengths[1]; + + memcpy(valuestr, args->args[1], vlen--); + + for (size_t i = 0; i < vlen; ++i) + if (!isdigit(valuestr[i])) + { + sprintf(message, "%s() second argument must be numeric or end in G, M or K", funcname); + return 1; + } + + if (!isdigit(valuestr[vlen])) + { + switch (valuestr[vlen]) + { + case 'G': + case 'g': + case 'M': + case 'm': + case 'K': + case 'k': + case '\0': break; + + default: + sprintf(message, "%s() second argument must be numeric or end in G, M or K", funcname); + return 1; + } + } + + return 0; + } + #ifdef _MSC_VER __declspec(dllexport) #endif - const char* calsetparms(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + const char* mcssetparms(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, char* error) { char parameter[MAXSTRINGLENGTH]; @@ -135,50 +180,38 @@ extern "C" return result; } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcssetparms_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return setparms_init(initid, args, message, "MCSSETPARMS"); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcssetparms_deinit(UDF_INIT* initid) + { + } + + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* calsetparms(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + return mcssetparms(initid, args, result, length, is_null, error); + } + #ifdef _MSC_VER __declspec(dllexport) #endif my_bool calsetparms_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - if (args->arg_count != 2 || args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT) - { - strcpy(message, "CALSETPARMS() requires two string arguments"); - return 1; - } - - initid->max_length = MAXSTRINGLENGTH; - - char valuestr[MAXSTRINGLENGTH]; - size_t vlen = args->lengths[1]; - - memcpy(valuestr, args->args[1], vlen--); - - for (size_t i = 0; i < vlen; ++i) - if (!isdigit(valuestr[i])) - { - strcpy(message, "CALSETPARMS() second argument must be numeric or end in G, M or K"); - return 1; - } - - if (!isdigit(valuestr[vlen])) - { - switch (valuestr[vlen]) - { - case 'G': - case 'g': - case 'M': - case 'm': - case 'K': - case 'k': - case '\0': break; - - default: - strcpy(message, "CALSETPARMS() second argument must be numeric or end in G, M or K"); - return 1; - } - } - - return 0; + return setparms_init(initid, args, message, "CALSETPARMS"); } #ifdef _MSC_VER @@ -188,10 +221,24 @@ extern "C" { } + my_bool getstats_init(UDF_INIT* initid, UDF_ARGS* args, char* message, const char* funcname) + { + if (args->arg_count != 0) + { + sprintf(message, "%s() takes no arguments", funcname); + return 1; + } + + initid->maybe_null = 1; + initid->max_length = 255; + + return 0; + } + #ifdef _MSC_VER __declspec(dllexport) #endif - const char* calgetstats(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + const char* mcsgetstats(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, char* error) { if (get_fe_conn_info_ptr() == NULL) @@ -215,21 +262,37 @@ extern "C" return result; } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcsgetstats_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return getstats_init(initid, args, message, "MCSGETSTATS"); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcsgetstats_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* calgetstats(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + return mcsgetstats(initid, args, result, length, is_null, error); + } + #ifdef _MSC_VER __declspec(dllexport) #endif my_bool calgetstats_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - if (args->arg_count != 0) - { - strcpy(message, "CALGETSTATS() takes no arguments"); - return 1; - } - - initid->maybe_null = 1; - initid->max_length = 255; - - return 0; + return getstats_init(initid, args, message, "CALGETSTATS"); } #ifdef _MSC_VER @@ -239,10 +302,21 @@ extern "C" { } + my_bool settrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message, const char* funcname) + { + if (args->arg_count != 1 || args->arg_type[0] != INT_RESULT) + { + sprintf(message, "%s() requires one INTEGER argument", funcname); + return 1; + } + + return 0; + } + #ifdef _MSC_VER __declspec(dllexport) #endif - long long calsettrace(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + long long mcssettrace(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) { if (get_fe_conn_info_ptr() == NULL) set_fe_conn_info_ptr((void*)new cal_connection_info()); @@ -257,18 +331,35 @@ extern "C" return oldTrace; } +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcssettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return settrace_init(initid, args, message, "MCSSETTRACE"); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcssettrace_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + long long calsettrace(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + return mcssettrace(initid, args, is_null, error); + } + #ifdef _MSC_VER __declspec(dllexport) #endif my_bool calsettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - if (args->arg_count != 1 || args->arg_type[0] != INT_RESULT) - { - strcpy(message, "CALSETTRACE() requires one INTEGER argument"); - return 1; - } - - return 0; + return settrace_init(initid, args, message, "CALSETTRACE"); } #ifdef _MSC_VER @@ -290,7 +381,7 @@ extern "C" try { - if (dbrm.getSystemReady() && dbrm.getSystemQueryReady()) + if (dbrm.getSystemReady() > 0 && dbrm.getSystemQueryReady() > 0) { return 1; } @@ -414,29 +505,26 @@ extern "C" { } -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool calviewtablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool viewtablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message, const char* funcname) { if (args->arg_count == 2 && (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT)) { - strcpy(message, "CALVIEWTABLELOCK() requires two string arguments"); + sprintf(message, "%s() requires two string arguments", funcname); return 1; } else if ((args->arg_count == 1) && (args->arg_type[0] != STRING_RESULT)) { - strcpy(message, "CALVIEWTABLELOCK() requires one string argument"); + sprintf(message, "%s() requires one string argument", funcname); return 1; } else if (args->arg_count > 2) { - strcpy(message, "CALVIEWTABLELOCK() takes one or two arguments only"); + sprintf(message, "%s() takes one or two arguments only", funcname); return 1; } else if (args->arg_count == 0) { - strcpy(message, "CALVIEWTABLELOCK() requires at least one argument"); + sprintf(message, "%s() requires at least one argument", funcname); return 1; } @@ -449,7 +537,15 @@ extern "C" #ifdef _MSC_VER __declspec(dllexport) #endif - const char* calviewtablelock(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + my_bool mcsviewtablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return viewtablelock_init(initid, args, message, "MCSVIEWTABLELOCK"); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* mcsviewtablelock(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, char* error) { THD* thd = current_thd; @@ -504,18 +600,39 @@ extern "C" #ifdef _MSC_VER __declspec(dllexport) #endif - void calviewtablelock_deinit(UDF_INIT* initid) + void mcsviewtablelock_deinit(UDF_INIT* initid) { } #ifdef _MSC_VER __declspec(dllexport) #endif - my_bool calcleartablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool calviewtablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return viewtablelock_init(initid, args, message, "CALVIEWTABLELOCK"); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* calviewtablelock(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + return mcsviewtablelock(initid, args, result, length, is_null, error); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void calviewtablelock_deinit(UDF_INIT* initid) + { + } + + my_bool cleartablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message, const char* funcname) { if ((args->arg_count != 1) || (args->arg_type[0] != INT_RESULT)) { - strcpy(message, "CALCLEARTABLELOCK() requires one integer argument (the lockID)"); + sprintf(message, "%s() requires one integer argument (the lockID)", funcname); return 1; } @@ -525,10 +642,19 @@ extern "C" return 0; } + #ifdef _MSC_VER __declspec(dllexport) #endif - const char* calcleartablelock(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + my_bool mcscleartablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return cleartablelock_init(initid, args, message, "MCSCLEARTABLELOCK"); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* mcscleartablelock(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, char* error) { if (get_fe_conn_info_ptr() == NULL) @@ -555,33 +681,54 @@ extern "C" #ifdef _MSC_VER __declspec(dllexport) #endif - void calcleartablelock_deinit(UDF_INIT* initid) + void mcscleartablelock_deinit(UDF_INIT* initid) { } #ifdef _MSC_VER __declspec(dllexport) #endif - my_bool callastinsertid_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool calcleartablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return cleartablelock_init(initid, args, message, "CALCLEARTABLELOCK"); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* calcleartablelock(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + return mcscleartablelock(initid, args, result, length, is_null, error); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void calcleartablelock_deinit(UDF_INIT* initid) + { + } + + my_bool lastinsertid_init(UDF_INIT* initid, UDF_ARGS* args, char* message, const char* funcname) { if (args->arg_count == 2 && (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT)) { - strcpy(message, "CALLASTINSRTID() requires two string arguments"); + sprintf(message, "%s() requires two string arguments", funcname); return 1; } else if ((args->arg_count == 1) && (args->arg_type[0] != STRING_RESULT)) { - strcpy(message, "CALLASTINSERTID() requires one string argument"); + sprintf(message, "%s() requires one string argument", funcname); return 1; } else if (args->arg_count > 2) { - strcpy(message, "CALLASTINSERTID() takes one or two arguments only"); + sprintf(message, "%s() takes one or two arguments only", funcname); return 1; } else if (args->arg_count == 0) { - strcpy(message, "CALLASTINSERTID() requires at least one argument"); + sprintf(message, "%s() requires at least one argument", funcname); return 1; } @@ -591,10 +738,19 @@ extern "C" return 0; } + #ifdef _MSC_VER __declspec(dllexport) #endif - long long callastinsertid(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + my_bool mcslastinsertid_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return lastinsertid_init(initid, args, message, "MCSLASTINSERTID"); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + long long mcslastinsertid(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) { THD* thd = current_thd; @@ -656,6 +812,29 @@ extern "C" return (nextVal - 1); } +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcslastinsertid_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool callastinsertid_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return lastinsertid_init(initid, args, message, "CALLASTINSERTID"); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + long long callastinsertid(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + return mcslastinsertid(initid, args, is_null, error); + } + #ifdef _MSC_VER __declspec(dllexport) #endif @@ -663,6 +842,40 @@ extern "C" { } + my_bool flushcache_init(UDF_INIT* initid, UDF_ARGS* args, char* message, const char* funcname) + { + if (args->arg_count != 0) + { + sprintf(message, "%s() takes no arguments", funcname); + return 1; + } + + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcsflushcache_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + long long mcsflushcache(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + return static_cast(cacheutils::flushPrimProcCache()); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcsflushcache_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return flushcache_init(initid, args, message, "MCSFLUSHCACHE"); + } + #ifdef _MSC_VER __declspec(dllexport) #endif @@ -675,7 +888,7 @@ extern "C" #endif long long calflushcache(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) { - return static_cast(cacheutils::flushPrimProcCache()); + return mcsflushcache(initid, args, is_null, error); } #ifdef _MSC_VER @@ -683,13 +896,7 @@ extern "C" #endif my_bool calflushcache_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - if (args->arg_count != 0) - { - strcpy(message, "CALFLUSHCACHE() takes no arguments"); - return 1; - } - - return 0; + return flushcache_init(initid, args, message, "CALFLUSHCACHE"); } static const unsigned long TraceSize = 16 * 1024; @@ -700,7 +907,7 @@ extern "C" #ifdef _MSC_VER __declspec(dllexport) #endif - const char* calgettrace(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + const char* mcsgettrace(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, char* error) { const std::string* msgp; @@ -741,16 +948,13 @@ extern "C" return msgp->c_str(); } -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool calgettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool gettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message, const char* funcname) { #if 0 if (args->arg_count != 0) { - strcpy(message, "CALGETTRACE() takes no arguments"); + sprintf(message, "%s() takes no arguments", funcname); return 1; } @@ -764,14 +968,57 @@ extern "C" #ifdef _MSC_VER __declspec(dllexport) #endif - void calgettrace_deinit(UDF_INIT* initid) + my_bool mcsgettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return gettrace_init(initid, args, message, "MCSGETTRACE"); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcsgettrace_deinit(UDF_INIT* initid) { } #ifdef _MSC_VER __declspec(dllexport) #endif - const char* calgetversion(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + const char* calgettrace(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + return mcsgettrace(initid, args, result, length, is_null, error); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool calgettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return gettrace_init(initid, args, message, "CALGETTRACE"); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void calgettrace_deinit(UDF_INIT* initid) + { + } + + my_bool getversion_init(UDF_INIT* initid, UDF_ARGS* args, char* message, const char* funcname) + { + if (args->arg_count != 0) + { + sprintf(message, "%s() takes no arguments", funcname); + return 1; + } + + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* mcsgetversion(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, char* error) { std::string version(columnstore_version); @@ -780,18 +1027,36 @@ extern "C" return result; } +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcsgetversion_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return getversion_init(initid, args, message, "MCSGETVERSION"); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcsgetversion_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* calgetversion(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + return mcsgetversion(initid, args, result, length, is_null, error); + } + #ifdef _MSC_VER __declspec(dllexport) #endif my_bool calgetversion_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - if (args->arg_count != 0) - { - strcpy(message, "CALGETVERSION() takes no arguments"); - return 1; - } - - return 0; + return getversion_init(initid, args, message, "CALGETVERSION"); } #ifdef _MSC_VER @@ -801,10 +1066,21 @@ extern "C" { } + my_bool getsqlcount_init(UDF_INIT* initid, UDF_ARGS* args, char* message, const char* funcname) + { + if (args->arg_count != 0) + { + sprintf(message, "%s() takes no arguments", funcname); + return 1; + } + + return 0; + } + #ifdef _MSC_VER __declspec(dllexport) #endif - const char* calgetsqlcount(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + const char* mcsgetsqlcount(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, char* error) { if (get_fe_conn_info_ptr() == NULL) @@ -843,18 +1119,36 @@ extern "C" return result; } +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcsgetsqlcount_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return getstats_init(initid, args, message, "MCSGETSQLCOUNT"); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcsgetsqlcount_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* calgetsqlcount(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + return mcsgetsqlcount(initid, args, result, length, is_null, error); + } + #ifdef _MSC_VER __declspec(dllexport) #endif my_bool calgetsqlcount_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - if (args->arg_count != 0) - { - strcpy(message, "CALGETSQLCOUNT() takes no arguments"); - return 1; - } - - return 0; + return getstats_init(initid, args, message, "CALGETSQLCOUNT"); } #ifdef _MSC_VER @@ -864,4 +1158,64 @@ extern "C" { } +#ifdef _MSC_VER + __declspec(dllexport) +#endif + long long mcs_emindex_size(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + DBRM dbrm; + return dbrm.EMIndexShmemSize(); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcs_emindex_size_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 0) + { + strcpy(message, MsgEMIndexSizeInitErrMsg); + return 1; + } + + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcs_emindex_size_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + long long mcs_emindex_free(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + DBRM dbrm; + return dbrm.EMIndexShmemFree(); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcs_emindex_free_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 0) + { + strcpy(message, MsgEMIndexFreeInitErrMsg); + return 1; + } + + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcs_emindex_free_deinit(UDF_INIT* initid) + { + } + } // extern "C" diff --git a/dbcon/mysql/ha_mcs_datatype.h b/dbcon/mysql/ha_mcs_datatype.h index 1e9079530..17aa7d17e 100644 --- a/dbcon/mysql/ha_mcs_datatype.h +++ b/dbcon/mysql/ha_mcs_datatype.h @@ -29,9 +29,11 @@ class StoreFieldMariaDB : public StoreField { Field* m_field; const CalpontSystemCatalog::ColType& m_type; + long m_timeZone; public: - StoreFieldMariaDB(Field* f, const CalpontSystemCatalog::ColType& type) : m_field(f), m_type(type) + StoreFieldMariaDB(Field* f, const CalpontSystemCatalog::ColType& type, const long timeZone) + : m_field(f), m_type(type), m_timeZone(timeZone) { } @@ -76,8 +78,7 @@ class StoreFieldMariaDB : public StoreField int store_timestamp(int64_t val) override { char tmp[256]; - DataConvert::timestampToString(val, tmp, sizeof(tmp), current_thd->variables.time_zone->get_name()->ptr(), - m_type.precision); + DataConvert::timestampToString(val, tmp, sizeof(tmp), m_timeZone, m_type.precision); return store_string(tmp, strlen(tmp)); } @@ -212,8 +213,10 @@ class WriteBatchFieldMariaDB : public WriteBatchField Field* m_field; const CalpontSystemCatalog::ColType& m_type; uint32_t m_mbmaxlen; - WriteBatchFieldMariaDB(Field* field, const CalpontSystemCatalog::ColType& type, uint32_t mbmaxlen) - : m_field(field), m_type(type), m_mbmaxlen(mbmaxlen) + long m_timeZone; + WriteBatchFieldMariaDB(Field* field, const CalpontSystemCatalog::ColType& type, uint32_t mbmaxlen, + const long timeZone) + : m_field(field), m_type(type), m_mbmaxlen(mbmaxlen), m_timeZone(timeZone) { } size_t ColWriteBatchDate(const uchar* buf, bool nullVal, ColBatchWriter& ci) override @@ -325,7 +328,7 @@ class WriteBatchFieldMariaDB : public WriteBatchField my_timestamp_from_binary(&tm, buf, m_field->decimals()); MySQLTime time; - gmtSecToMySQLTime(tm.tv_sec, time, current_thd->variables.time_zone->get_name()->ptr()); + gmtSecToMySQLTime(tm.tv_sec, time, m_timeZone); if (!tm.tv_usec) { @@ -838,4 +841,3 @@ class WriteBatchFieldMariaDB : public WriteBatchField } // end of namespace datatypes -// vim:ts=2 sw=2: diff --git a/dbcon/mysql/ha_mcs_ddl.cpp b/dbcon/mysql/ha_mcs_ddl.cpp index 465fdf47d..92c9c9ceb 100644 --- a/dbcon/mysql/ha_mcs_ddl.cpp +++ b/dbcon/mysql/ha_mcs_ddl.cpp @@ -25,27 +25,19 @@ #include #include #include -#ifdef _MSC_VER -#include -#else #include -#endif #include #include #include #include -#ifdef _MSC_VER -#include -#else +#include #include -#endif #include #include using namespace std; #include #include -#include #include using namespace boost; @@ -142,16 +134,16 @@ CalpontSystemCatalog::ColDataType convertDataType(const ddlpackage::ColumnType& int parseCompressionComment(std::string comment) { - algorithm::to_upper(comment); - regex compat("[[:space:]]*COMPRESSION[[:space:]]*=[[:space:]]*", regex_constants::extended); + boost::algorithm::to_upper(comment); + std::regex compat("[[:space:]]*COMPRESSION[[:space:]]*=[[:space:]]*", std::regex_constants::extended); int compressiontype = 0; - boost::match_results what; + std::match_results what; std::string::const_iterator start, end; start = comment.begin(); end = comment.end(); - boost::match_flag_type flags = boost::match_default; + std::regex_constants::match_flag_type flags = std::regex_constants::match_default; - if (boost::regex_search(start, end, what, compat, flags)) + if (std::regex_search(start, end, what, compat, flags)) { // Find the pattern, now get the compression type string compType(&(*(what[0].second))); @@ -649,11 +641,14 @@ bool anyNullInTheColumn(THD* thd, string& schema, string& table, string& columnN csep.returnedCols(returnedColumnList); SimpleFilter* sf = new SimpleFilter(); - sf->timeZone(thd->variables.time_zone->get_name()->ptr()); + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + sf->timeZone(timeZoneOffset); boost::shared_ptr sop(new PredicateOperator("isnull")); sf->op(sop); ConstantColumn* rhs = new ConstantColumn("", ConstantColumn::NULLDATA); - rhs->timeZone(thd->variables.time_zone->get_name()->ptr()); + rhs->timeZone(timeZoneOffset); sf->lhs(col[0]->clone()); sf->rhs(rhs); @@ -799,6 +794,10 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl if (valConfig.compare("YES") == 0) isVarbinaryAllowed = true; + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + //@Bug 1771. error out for not supported feature. if (typeid(stmt) == typeid(CreateTableStatement)) { @@ -901,9 +900,9 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl try { - convertedVal = colType.convertColumnData( - createTable->fTableDef->fColumns[i]->fDefaultValue->fValue, pushWarning, - thd->variables.time_zone->get_name()->ptr(), false, false, false); + convertedVal = + colType.convertColumnData(createTable->fTableDef->fColumns[i]->fDefaultValue->fValue, + pushWarning, timeZoneOffset, false, false, false); } catch (std::exception&) { @@ -1143,7 +1142,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl algorithm::to_lower(alterTable->fTableName->fName); } - alterTable->fTimeZone.assign(thd->variables.time_zone->get_name()->ptr()); + alterTable->setTimeZone(timeZoneOffset); if (schema.length() == 0) { @@ -1313,9 +1312,8 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl try { - convertedVal = colType.convertColumnData( - addColumnPtr->fColumnDef->fDefaultValue->fValue, pushWarning, - thd->variables.time_zone->get_name()->ptr(), false, false, false); + convertedVal = colType.convertColumnData(addColumnPtr->fColumnDef->fDefaultValue->fValue, + pushWarning, timeZoneOffset, false, false, false); } catch (std::exception&) { @@ -1383,10 +1381,10 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl if (comment.length() > 0) { //@Bug 3782 This is for synchronization after calonlinealter to use - algorithm::to_upper(comment); - regex pat("[[:space:]]*SCHEMA[[:space:]]+SYNC[[:space:]]+ONLY", regex_constants::extended); + boost::algorithm::to_upper(comment); + std::regex pat("[[:space:]]*SCHEMA[[:space:]]+SYNC[[:space:]]+ONLY", std::regex_constants::extended); - if (regex_search(comment, pat)) + if (std::regex_search(comment, pat)) { return 0; } @@ -1698,9 +1696,8 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl try { - convertedVal = colType.convertColumnData( - addColumnsPtr->fColumns[0]->fDefaultValue->fValue, pushWarning, - thd->variables.time_zone->get_name()->ptr(), false, false, false); + convertedVal = colType.convertColumnData(addColumnsPtr->fColumns[0]->fDefaultValue->fValue, + pushWarning, timeZoneOffset, false, false, false); } catch (std::exception&) { @@ -2353,14 +2350,14 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea bool schemaSyncOnly = false; bool isCreate = true; - regex pat("[[:space:]]*SCHEMA[[:space:]]+SYNC[[:space:]]+ONLY", regex_constants::extended); + std::regex pat("[[:space:]]*SCHEMA[[:space:]]+SYNC[[:space:]]+ONLY", std::regex_constants::extended); - if (regex_search(tablecomment, pat)) + if (std::regex_search(tablecomment, pat)) { schemaSyncOnly = true; pat = createpatstr; - if (!regex_search(stmt, pat)) + if (!std::regex_search(stmt, pat)) { isCreate = false; } @@ -2390,7 +2387,7 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea pat = alterpatstr; - if (regex_search(stmt, pat)) + if (std::regex_search(stmt, pat)) { ci.isAlter = true; ci.alterTableState = cal_connection_info::ALTER_FIRST_RENAME; @@ -2802,4 +2799,3 @@ extern "C" } } -// vim:ts=4 sw=4: diff --git a/dbcon/mysql/ha_mcs_dml.cpp b/dbcon/mysql/ha_mcs_dml.cpp index 3c7bd5573..a3f97db70 100644 --- a/dbcon/mysql/ha_mcs_dml.cpp +++ b/dbcon/mysql/ha_mcs_dml.cpp @@ -337,7 +337,10 @@ int doProcessInsertValues(TABLE* table, uint32_t size, cal_connection_info& ci, pDMLPackage->set_TableName(name); name = table->s->db.str; pDMLPackage->set_SchemaName(name); - pDMLPackage->set_TimeZone(thd->variables.time_zone->get_name()->ptr()); + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + pDMLPackage->set_TimeZone(timeZoneOffset); if (thd->lex->sql_command == SQLCOM_INSERT_SELECT) pDMLPackage->set_isInsertSelect(true); @@ -676,7 +679,8 @@ int ha_mcs_impl_write_row_(const uchar* buf, TABLE* table, cal_connection_info& } } -int ha_mcs_impl_write_batch_row_(const uchar* buf, TABLE* table, cal_impl_if::cal_connection_info& ci) +int ha_mcs_impl_write_batch_row_(const uchar* buf, TABLE* table, cal_impl_if::cal_connection_info& ci, + long timeZone) { ByteStream rowData; int rc = 0; @@ -746,7 +750,7 @@ int ha_mcs_impl_write_batch_row_(const uchar* buf, TABLE* table, cal_impl_if::ca Field* fieldPtr = table->field[colpos]; uint32_t mbmaxlen = (fieldPtr->charset() && fieldPtr->charset()->mbmaxlen) ? fieldPtr->charset()->mbmaxlen : 0; - datatypes::WriteBatchFieldMariaDB field(fieldPtr, colType, mbmaxlen); + datatypes::WriteBatchFieldMariaDB field(fieldPtr, colType, mbmaxlen, timeZone); idbassert(table == table->field[colpos]->table); buf += h->ColWriteBatch(&field, buf, nullVal, writer); } @@ -997,4 +1001,3 @@ int ha_mcs_impl_close_connection_(handlerton* hton, THD* thd, cal_connection_inf return rc; } -// vim:ts=4 sw=4: diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 7a7a8978b..c9975947c 100644 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -45,7 +45,6 @@ using namespace std; #include #include #include -#include #include #include "errorids.h" @@ -1509,7 +1508,7 @@ uint32_t buildJoin(gp_walk_info& gwi, List& join_list, return 0; } -ParseTree* buildRowPredicate(THD* thd, RowColumn* lhs, RowColumn* rhs, string predicateOp) +ParseTree* buildRowPredicate(gp_walk_info* gwip, RowColumn* lhs, RowColumn* rhs, string predicateOp) { PredicateOperator* po = new PredicateOperator(predicateOp); boost::shared_ptr sop(po); @@ -1523,7 +1522,7 @@ ParseTree* buildRowPredicate(THD* thd, RowColumn* lhs, RowColumn* rhs, string pr ParseTree* pt = new ParseTree(lo); sop->setOpType(lhs->columnVec()[0]->resultType(), rhs->columnVec()[0]->resultType()); SimpleFilter* sf = new SimpleFilter(sop, lhs->columnVec()[0].get(), rhs->columnVec()[0].get()); - sf->timeZone(thd->variables.time_zone->get_name()->ptr()); + sf->timeZone(gwip->timeZone); pt->left(new ParseTree(sf)); for (uint32_t i = 1; i < lhs->columnVec().size(); i++) @@ -1531,7 +1530,7 @@ ParseTree* buildRowPredicate(THD* thd, RowColumn* lhs, RowColumn* rhs, string pr sop.reset(po->clone()); sop->setOpType(lhs->columnVec()[i]->resultType(), rhs->columnVec()[i]->resultType()); SimpleFilter* sf = new SimpleFilter(sop, lhs->columnVec()[i].get(), rhs->columnVec()[i].get()); - sf->timeZone(thd->variables.time_zone->get_name()->ptr()); + sf->timeZone(gwip->timeZone); pt->right(new ParseTree(sf)); if (i + 1 < lhs->columnVec().size()) @@ -1551,7 +1550,7 @@ bool buildRowColumnFilter(gp_walk_info* gwip, RowColumn* rhs, RowColumn* lhs, It { // (c1,c2,..) = (v1,v2,...) transform to: c1=v1 and c2=v2 and ... assert(!lhs->columnVec().empty() && lhs->columnVec().size() == rhs->columnVec().size()); - gwip->ptWorkStack.push(buildRowPredicate(gwip->thd, rhs, lhs, ifp->func_name())); + gwip->ptWorkStack.push(buildRowPredicate(gwip, rhs, lhs, ifp->func_name())); } else if (ifp->functype() == Item_func::IN_FUNC) { @@ -1595,7 +1594,7 @@ bool buildRowColumnFilter(gp_walk_info* gwip, RowColumn* rhs, RowColumn* lhs, It RowColumn* vals = dynamic_cast(tmpStack.top()); valVec.push_back(vals); tmpStack.pop(); - ParseTree* pt = buildRowPredicate(gwip->thd, columns, vals, predicateOp); + ParseTree* pt = buildRowPredicate(gwip, columns, vals, predicateOp); while (!tmpStack.empty()) { @@ -1604,7 +1603,7 @@ bool buildRowColumnFilter(gp_walk_info* gwip, RowColumn* rhs, RowColumn* lhs, It vals = dynamic_cast(tmpStack.top()); valVec.push_back(vals); tmpStack.pop(); - pt1->right(buildRowPredicate(gwip->thd, columns->clone(), vals, predicateOp)); + pt1->right(buildRowPredicate(gwip, columns->clone(), vals, predicateOp)); pt = pt1; } @@ -1646,8 +1645,8 @@ bool buildRowColumnFilter(gp_walk_info* gwip, RowColumn* rhs, RowColumn* lhs, It break; sop->setOpType(sc->resultType(), valVec[j]->columnVec()[i]->resultType()); - cf->pushFilter(new SimpleFilter(sop, sc->clone(), valVec[j]->columnVec()[i]->clone(), - gwip->thd->variables.time_zone->get_name()->ptr())); + cf->pushFilter( + new SimpleFilter(sop, sc->clone(), valVec[j]->columnVec()[i]->clone(), gwip->timeZone)); } if (j < valVec.size()) @@ -1792,7 +1791,7 @@ bool buildEqualityPredicate(execplan::ReturnedColumn* lhs, execplan::ReturnedCol } SimpleFilter* sf = new SimpleFilter(); - sf->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + sf->timeZone(gwip->timeZone); //@bug 2101 for when there are only constants in a delete or update where clause (eg "where 5 < 6"). // There will be no field column and it will get here only if the comparison is true. @@ -1906,11 +1905,11 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) sop.reset(new PredicateOperator(">")); sop->setOpType(filterCol->resultType(), rhs->resultType()); sfr = new SimpleFilter(sop, filterCol, rhs); - sfr->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + sfr->timeZone(gwip->timeZone); sop.reset(new PredicateOperator("<")); sop->setOpType(filterCol->resultType(), lhs->resultType()); sfl = new SimpleFilter(sop, filterCol->clone(), lhs); - sfl->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + sfl->timeZone(gwip->timeZone); ParseTree* ptp = new ParseTree(new LogicOperator("or")); ptp->left(sfr); ptp->right(sfl); @@ -1921,11 +1920,11 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) sop.reset(new PredicateOperator("<=")); sop->setOpType(filterCol->resultType(), rhs->resultType()); sfr = new SimpleFilter(sop, filterCol, rhs); - sfr->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + sfr->timeZone(gwip->timeZone); sop.reset(new PredicateOperator(">=")); sop->setOpType(filterCol->resultType(), lhs->resultType()); sfl = new SimpleFilter(sop, filterCol->clone(), lhs); - sfl->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + sfl->timeZone(gwip->timeZone); ParseTree* ptp = new ParseTree(new LogicOperator("and")); ptp->left(sfr); ptp->right(sfl); @@ -1985,8 +1984,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) cf->op(sop); sop.reset(new PredicateOperator(eqop)); sop->setOpType(gwip->scsp->resultType(), lhs->resultType()); - cf->pushFilter( - new SimpleFilter(sop, gwip->scsp->clone(), lhs, gwip->thd->variables.time_zone->get_name()->ptr())); + cf->pushFilter(new SimpleFilter(sop, gwip->scsp->clone(), lhs, gwip->timeZone)); while (!gwip->rcWorkStack.empty()) { @@ -1998,8 +1996,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) gwip->rcWorkStack.pop(); sop.reset(new PredicateOperator(eqop)); sop->setOpType(gwip->scsp->resultType(), lhs->resultType()); - cf->pushFilter( - new SimpleFilter(sop, gwip->scsp->clone(), lhs, gwip->thd->variables.time_zone->get_name()->ptr())); + cf->pushFilter(new SimpleFilter(sop, gwip->scsp->clone(), lhs, gwip->timeZone)); } if (!gwip->rcWorkStack.empty()) @@ -2065,8 +2062,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) { gwip->rcWorkStack.push(new ConstantColumn((int64_t)udf->val_int())); } - (dynamic_cast(gwip->rcWorkStack.top())) - ->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(gwip->rcWorkStack.top()))->timeZone(gwip->timeZone); } else { @@ -2088,8 +2084,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) { gwip->rcWorkStack.push(new ConstantColumn(buf.ptr(), ConstantColumn::NUM)); } - (dynamic_cast(gwip->rcWorkStack.top())) - ->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(gwip->rcWorkStack.top()))->timeZone(gwip->timeZone); return false; } @@ -2265,19 +2260,19 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) SimpleFilter* sfo = 0; // b IS NULL ConstantColumn* nlhs1 = new ConstantColumn("", ConstantColumn::NULLDATA); - nlhs1->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + nlhs1->timeZone(gwip->timeZone); sop.reset(new PredicateOperator("isnull")); sop->setOpType(lhs->resultType(), rhs->resultType()); sfn1 = new SimpleFilter(sop, rhs, nlhs1); - sfn1->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + sfn1->timeZone(gwip->timeZone); ParseTree* ptpl = new ParseTree(sfn1); // a IS NULL ConstantColumn* nlhs2 = new ConstantColumn("", ConstantColumn::NULLDATA); - nlhs2->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + nlhs2->timeZone(gwip->timeZone); sop.reset(new PredicateOperator("isnull")); sop->setOpType(lhs->resultType(), rhs->resultType()); sfn2 = new SimpleFilter(sop, lhs, nlhs2); - sfn2->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + sfn2->timeZone(gwip->timeZone); ParseTree* ptpr = new ParseTree(sfn2); // AND them both ParseTree* ptpn = new ParseTree(new LogicOperator("and")); @@ -2287,7 +2282,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) sop.reset(new PredicateOperator("=")); sop->setOpType(lhs->resultType(), rhs->resultType()); sfo = new SimpleFilter(sop, lhs->clone(), rhs->clone()); - sfo->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + sfo->timeZone(gwip->timeZone); // OR with the NULL comparison tree ParseTree* ptp = new ParseTree(new LogicOperator("or")); ptp->left(sfo); @@ -2337,7 +2332,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) bool buildConstPredicate(Item_func* ifp, ReturnedColumn* rhs, gp_walk_info* gwip) { SimpleFilter* sf = new SimpleFilter(); - sf->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + sf->timeZone(gwip->timeZone); boost::shared_ptr sop(new PredicateOperator(ifp->func_name())); ConstantColumn* lhs = 0; @@ -2356,7 +2351,7 @@ bool buildConstPredicate(Item_func* ifp, ReturnedColumn* rhs, gp_walk_info* gwip lhs = new ConstantColumn((int64_t)0, ConstantColumn::NUM); sop.reset(new PredicateOperator("=")); } - lhs->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + lhs->timeZone(gwip->timeZone); CalpontSystemCatalog::ColType opType = rhs->resultType(); @@ -2431,7 +2426,7 @@ SimpleColumn* buildSimpleColFromDerivedTable(gp_walk_info& gwi, Item_field* ifp) sc->tableAlias(gwi.tbList[i].alias); sc->viewName(viewName, lower_case_table_names); sc->resultType(ct); - sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sc->timeZone(gwi.timeZone); break; } } @@ -2490,7 +2485,7 @@ SimpleColumn* buildSimpleColFromDerivedTable(gp_walk_info& gwi, Item_field* ifp) sc->tableName(csep->derivedTbAlias()); sc->colPosition(j); sc->tableAlias(csep->derivedTbAlias()); - sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sc->timeZone(gwi.timeZone); if (!viewName.empty()) { sc->viewName(viewName, lower_case_table_names); @@ -2613,7 +2608,7 @@ void collectAllCols(gp_walk_info& gwi, Item_field* ifp) sc->tableAlias(csep->derivedTbAlias()); sc->viewName(gwi.tbList[i].view); sc->resultType(cols[j]->resultType()); - sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sc->timeZone(gwi.timeZone); // @bug5634 derived table optimization cols[j]->incRefCount(); @@ -2661,7 +2656,7 @@ void collectAllCols(gp_walk_info& gwi, Item_field* ifp) sc->resultType(ct); sc->tableAlias(gwi.tbList[i].alias, lower_case_table_names); sc->viewName(viewName, lower_case_table_names); - sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sc->timeZone(gwi.timeZone); srcp.reset(sc); gwi.returnedCols.push_back(srcp); gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); @@ -2994,7 +2989,7 @@ SimpleColumn* getSmallestColumn(boost::shared_ptr csc, sc->columnName(rc->alias()); sc->sequence(0); sc->tableAlias(tan.alias); - sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sc->timeZone(gwi.timeZone); sc->derivedTable(csep->derivedTbAlias()); sc->derivedRefCol(rc); return sc; @@ -3013,7 +3008,7 @@ SimpleColumn* getSmallestColumn(boost::shared_ptr csc, tan.fisColumnStore, gwi.sessionid, lower_case_table_names); sc->tableAlias(table->alias.ptr(), lower_case_table_names); sc->isColumnStore(false); - sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sc->timeZone(gwi.timeZone); sc->resultType(fieldType_MysqlToIDB(field)); sc->oid(field->field_index + 1); return sc; @@ -3043,7 +3038,7 @@ SimpleColumn* getSmallestColumn(boost::shared_ptr csc, SimpleColumn* sc = new SimpleColumn(tcn.schema, tcn.table, tcn.column, csc->sessionID()); sc->tableAlias(tan.alias); sc->viewName(tan.view); - sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sc->timeZone(gwi.timeZone); sc->resultType(csc->colType(oidlist[minWidthColOffset].objnum)); sc->charsetNumber(table->field[minWidthColOffset]->charset()->number); return sc; @@ -3209,7 +3204,7 @@ ReturnedColumn* buildReturnedColumnNull(gp_walk_info& gwi) return new SimpleColumn("noop"); ConstantColumn* rc = new ConstantColumnNull(); if (rc) - rc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + rc->timeZone(gwi.timeZone); return rc; } @@ -3346,7 +3341,7 @@ static ConstantColumn* buildConstantColumnMaybeNullFromValStr(const Item* item, { ConstantColumn* rc = newConstantColumnMaybeNullFromValStrNoTz(item, valStr, gwi); if (rc) - rc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + rc->timeZone(gwi.timeZone); return rc; } @@ -3365,7 +3360,7 @@ static ConstantColumn* buildConstantColumnNotNullUsingValNative(Item* item, gp_w { ConstantColumn* rc = newConstantColumnNotNullUsingValNativeNoTz(item, gwi); if (rc) - rc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + rc->timeZone(gwi.timeZone); return rc; } @@ -3565,7 +3560,7 @@ ArithmeticColumn* buildArithmeticColumn(Item_func* item, gp_walk_info& gwi, bool ArithmeticColumn* ac = new ArithmeticColumn(); Item** sfitempp = item->arguments(); ArithmeticOperator* aop = new ArithmeticOperator(item->func_name()); - aop->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + aop->timeZone(gwi.timeZone); aop->setOverflowCheck(get_decimal_overflow_check(gwi.thd)); ParseTree* pt = new ParseTree(aop); // ReturnedColumn *lhs = 0, *rhs = 0; @@ -3692,7 +3687,7 @@ ArithmeticColumn* buildArithmeticColumn(Item_func* item, gp_walk_info& gwi, bool else { ConstantColumn* cc = new ConstantColumn(string("0"), (int64_t)0); - cc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + cc->timeZone(gwi.timeZone); if (gwi.clauseType == SELECT || gwi.clauseType == HAVING || gwi.clauseType == GROUP_BY) // select clause { @@ -4065,15 +4060,14 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non THD* thd = current_thd; sptp.reset( new ParseTree(new ConstantColumn(static_cast(thd->variables.default_week_format)))); - (dynamic_cast(sptp->data())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwi.timeZone); funcParms.push_back(sptp); } // add the keyword unit argument for interval function if (funcName == "date_add_interval" || funcName == "extract" || funcName == "timestampdiff") { - addIntervalArgs(gwi.thd, ifp, funcParms); + addIntervalArgs(&gwi, ifp, funcParms); } // check for unsupported arguments add the keyword unit argument for extract functions @@ -4123,19 +4117,19 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non // add the keyword unit argument and char length for cast functions if (funcName == "cast_as_char") { - castCharArgs(gwi.thd, ifp, funcParms); + castCharArgs(&gwi, ifp, funcParms); } // add the length and scale arguments if (funcName == "decimal_typecast") { - castDecimalArgs(gwi.thd, ifp, funcParms); + castDecimalArgs(&gwi, ifp, funcParms); } // add the type argument if (funcName == "get_format") { - castTypeArgs(gwi.thd, ifp, funcParms); + castTypeArgs(&gwi, ifp, funcParms); } // add my_time_zone @@ -4150,8 +4144,7 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non // FIXME: Get GMT offset (in seconds east of GMT) in Windows... sptp.reset(new ParseTree(new ConstantColumn(static_cast(0), ConstantColumn::NUM))); #endif - (dynamic_cast(sptp->data())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwi.timeZone); funcParms.push_back(sptp); } @@ -4161,12 +4154,10 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non if (funcParms.size() == 0) { sptp.reset(new ParseTree(new ConstantColumn((int64_t)gwi.thd->rand.seed1, ConstantColumn::NUM))); - (dynamic_cast(sptp->data())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwi.timeZone); funcParms.push_back(sptp); sptp.reset(new ParseTree(new ConstantColumn((int64_t)gwi.thd->rand.seed2, ConstantColumn::NUM))); - (dynamic_cast(sptp->data())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwi.timeZone); funcParms.push_back(sptp); gwi.no_parm_func_list.push_back(fc); } @@ -4220,8 +4211,7 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non tzinfo = string((char*)buf, length); } sptp.reset(new ParseTree(new ConstantColumn(tzinfo))); - (dynamic_cast(sptp->data())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwi.timeZone); funcParms.push_back(sptp); tzinfo.clear(); if (to_tzinfo) @@ -4233,8 +4223,7 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non tzinfo = string((char*)buf, length); } sptp.reset(new ParseTree(new ConstantColumn(tzinfo))); - (dynamic_cast(sptp->data())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwi.timeZone); funcParms.push_back(sptp); tzinfo.clear(); } @@ -4253,8 +4242,7 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non sign = -1; } sptp.reset(new ParseTree(new ConstantColumn(sign))); - (dynamic_cast(sptp->data())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwi.timeZone); funcParms.push_back(sptp); } @@ -4327,16 +4315,23 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non #endif - fc->operationType(functor->operationType(funcParms, fc->resultType())); - // For some reason, MDB has MYSQL_TYPE_DATETIME2 for functions on a TIMESTAMP - if (fc->operationType().colDataType == CalpontSystemCatalog::TIMESTAMP) + execplan::CalpontSystemCatalog::ColType& resultType = fc->resultType(); + resultType.setTimeZone(gwi.timeZone); + fc->operationType(functor->operationType(funcParms, resultType)); + + // For floor/ceiling/truncate/round functions applied on TIMESTAMP columns, set the + // function result type to TIMESTAMP + if ((funcName == "floor" || funcName == "ceiling" || funcName == "truncate" || funcName == "round") && + fc->operationType().colDataType == CalpontSystemCatalog::TIMESTAMP) { CalpontSystemCatalog::ColType ct = fc->resultType(); ct.colDataType = CalpontSystemCatalog::TIMESTAMP; ct.colWidth = 8; fc->resultType(ct); } + fc->expressionId(ci->expressionId++); + // A few functions use a different collation than that found in // the base ifp class if (funcName == "locate" || funcName == "find_in_set" || funcName == "strcmp") @@ -4409,7 +4404,7 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non } } - fc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + fc->timeZone(gwi.timeZone); return fc; } @@ -4549,11 +4544,13 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS Func* functor = funcexp->getFunctor(funcName); fc->resultType(colType_MysqlToIDB(item)); - fc->operationType(functor->operationType(funcParms, fc->resultType())); + execplan::CalpontSystemCatalog::ColType& resultType = fc->resultType(); + resultType.setTimeZone(gwi.timeZone); + fc->operationType(functor->operationType(funcParms, resultType)); fc->functionName(funcName); fc->functionParms(funcParms); fc->expressionId(ci->expressionId++); - fc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + fc->timeZone(gwi.timeZone); // For function join. If any argument has non-zero joininfo, set it to the function. fc->setSimpleColumnList(); @@ -4695,7 +4692,7 @@ SimpleColumn* buildSimpleColumn(Item_field* ifp, gp_walk_info& gwi) sc->alias(ifp->name.str); sc->isColumnStore(prm.columnStore()); - sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sc->timeZone(gwi.timeZone); if (!prm.columnStore() && ifp->field) sc->oid(ifp->field->field_index + 1); // ExeMgr requires offset started from 1 @@ -4803,7 +4800,7 @@ static void processAggregateColumnConstArg(gp_walk_info& gwi, SRCP& parm, Aggreg { // Explicit NULL or a const function that evaluated to NULL cc = new ConstantColumnNull(); - cc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + cc->timeZone(gwi.timeZone); parm.reset(cc); ac->constCol(SRCP(rt)); return; @@ -4878,7 +4875,7 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) ac = new AggregateColumn(gwi.sessionid); } - ac->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + ac->timeZone(gwi.timeZone); if (isp->name.length) ac->alias(isp->name.str); @@ -5397,7 +5394,7 @@ because it has multiple arguments."; return ac; } -void addIntervalArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) +void addIntervalArgs(gp_walk_info* gwip, Item_func* ifp, FunctionParm& functionParms) { string funcName = ifp->func_name(); int interval_type = -1; @@ -5409,7 +5406,7 @@ void addIntervalArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) else if (funcName == "extract") interval_type = ((Item_extract*)ifp)->int_type; - functionParms.push_back(getIntervalType(thd, interval_type)); + functionParms.push_back(getIntervalType(gwip, interval_type)); SPTP sptp; if (funcName == "date_add_interval") @@ -5417,42 +5414,42 @@ void addIntervalArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) if (((Item_date_add_interval*)ifp)->date_sub_interval) { sptp.reset(new ParseTree(new ConstantColumn((int64_t)OP_SUB))); - (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwip->timeZone); functionParms.push_back(sptp); } else { sptp.reset(new ParseTree(new ConstantColumn((int64_t)OP_ADD))); - (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwip->timeZone); functionParms.push_back(sptp); } } } -SPTP getIntervalType(THD* thd, int interval_type) +SPTP getIntervalType(gp_walk_info* gwip, int interval_type) { SPTP sptp; sptp.reset(new ParseTree(new ConstantColumn((int64_t)interval_type))); - (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwip->timeZone); return sptp; } -void castCharArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) +void castCharArgs(gp_walk_info* gwip, Item_func* ifp, FunctionParm& functionParms) { Item_char_typecast* idai = (Item_char_typecast*)ifp; SPTP sptp; sptp.reset(new ParseTree(new ConstantColumn((int64_t)idai->get_cast_length()))); - (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwip->timeZone); functionParms.push_back(sptp); } -void castDecimalArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) +void castDecimalArgs(gp_walk_info* gwip, Item_func* ifp, FunctionParm& functionParms) { Item_decimal_typecast* idai = (Item_decimal_typecast*)ifp; SPTP sptp; sptp.reset(new ParseTree(new ConstantColumn((int64_t)idai->decimals))); - (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwip->timeZone); functionParms.push_back(sptp); // max length including sign and/or decimal points @@ -5460,12 +5457,12 @@ void castDecimalArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) sptp.reset(new ParseTree(new ConstantColumn((int64_t)idai->max_length - 1))); else sptp.reset(new ParseTree(new ConstantColumn((int64_t)idai->max_length - 2))); - (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwip->timeZone); functionParms.push_back(sptp); } -void castTypeArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) +void castTypeArgs(gp_walk_info* gwip, Item_func* ifp, FunctionParm& functionParms) { Item_func_get_format* get_format = (Item_func_get_format*)ifp; SPTP sptp; @@ -5474,7 +5471,7 @@ void castTypeArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) sptp.reset(new ParseTree(new ConstantColumn("DATE"))); else sptp.reset(new ParseTree(new ConstantColumn("DATETIME"))); - (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwip->timeZone); functionParms.push_back(sptp); } @@ -5575,7 +5572,7 @@ void gp_walk(const Item* item, void* arg) Item_hex_hybrid* hip = reinterpret_cast(const_cast(item)); gwip->rcWorkStack.push(new ConstantColumn((int64_t)hip->val_int(), ConstantColumn::NUM)); ConstantColumn* cc = dynamic_cast(gwip->rcWorkStack.top()); - cc->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + cc->timeZone(gwip->timeZone); break; } @@ -5594,8 +5591,7 @@ void gp_walk(const Item* item, void* arg) } gwip->rcWorkStack.push(new ConstantColumn(cval)); - (dynamic_cast(gwip->rcWorkStack.top())) - ->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(gwip->rcWorkStack.top()))->timeZone(gwip->timeZone); break; } @@ -5630,7 +5626,7 @@ void gp_walk(const Item* item, void* arg) { // push noop for unhandled item SimpleColumn* rc = new SimpleColumn("noop"); - rc->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + rc->timeZone(gwip->timeZone); gwip->rcWorkStack.push(rc); break; } @@ -5650,14 +5646,13 @@ void gp_walk(const Item* item, void* arg) { // push noop for unhandled item SimpleColumn* rc = new SimpleColumn("noop"); - rc->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + rc->timeZone(gwip->timeZone); gwip->rcWorkStack.push(rc); break; } gwip->rcWorkStack.push(new ConstantColumn("", ConstantColumn::NULLDATA)); - (dynamic_cast(gwip->rcWorkStack.top())) - ->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(gwip->rcWorkStack.top()))->timeZone(gwip->timeZone); break; } @@ -6175,7 +6170,7 @@ void gp_walk(const Item* item, void* arg) { // push noop for unhandled item SimpleColumn* rc = new SimpleColumn("noop"); - rc->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + rc->timeZone(gwip->timeZone); gwip->rcWorkStack.push(rc); break; } @@ -6664,7 +6659,7 @@ int processFrom(bool& isUnion, SELECT_LEX& select_lex, gp_walk_info& gwi, SCSEP& plan->data(csep->data()); // gwi for the union unit - gp_walk_info union_gwi; + gp_walk_info union_gwi(gwi.timeZone); union_gwi.thd = gwi.thd; uint32_t err = 0; @@ -6777,8 +6772,7 @@ int processWhere(SELECT_LEX& select_lex, gp_walk_info& gwi, SCSEP& csep, const s else if (join && join->zero_result_cause) { gwi.rcWorkStack.push(new ConstantColumn((int64_t)0, ConstantColumn::NUM)); - (dynamic_cast(gwi.rcWorkStack.top())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(gwi.rcWorkStack.top()))->timeZone(gwi.timeZone); } for (Item* item : gwi.condList) @@ -7226,7 +7220,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i gwi.sessionid = sessionID; boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); csc->identity(CalpontSystemCatalog::FE); - csep->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + csep->timeZone(gwi.timeZone); gwi.csc = csc; CalpontSelectExecutionPlan::SelectList derivedTbList; @@ -7607,7 +7601,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i selectSubList.push_back(ssub); SimpleColumn* rc = new SimpleColumn(); rc->colSource(rc->colSource() | SELECT_SUB); - rc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + rc->timeZone(gwi.timeZone); if (sub->get_select_lex()->get_table_list()) { @@ -8414,7 +8408,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i sc1->tableAlias(sc->tableAlias()); sc1->viewName(sc->viewName()); sc1->colPosition(0); - sc1->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sc1->timeZone(gwi.timeZone); minSc.reset(sc1); } } @@ -8473,12 +8467,12 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i return 0; } -int cp_get_table_plan(THD* thd, SCSEP& csep, cal_table_info& ti) +int cp_get_table_plan(THD* thd, SCSEP& csep, cal_table_info& ti, long timeZone) { gp_walk_info* gwi = ti.condInfo; if (!gwi) - gwi = new gp_walk_info(); + gwi = new gp_walk_info(timeZone); gwi->thd = thd; LEX* lex = thd->lex; @@ -8506,7 +8500,7 @@ int cp_get_table_plan(THD* thd, SCSEP& csep, cal_table_info& ti) boost::algorithm::to_lower(alias); } sc->tableAlias(alias); - sc->timeZone(gwi->thd->variables.time_zone->get_name()->ptr()); + sc->timeZone(gwi->timeZone); assert(sc); boost::shared_ptr spsc(sc); gwi->returnedCols.push_back(spsc); @@ -8581,7 +8575,10 @@ int cp_get_table_plan(THD* thd, SCSEP& csep, cal_table_info& ti) int cp_get_group_plan(THD* thd, SCSEP& csep, cal_impl_if::cal_group_info& gi) { SELECT_LEX* select_lex = gi.groupByTables->select_lex; - gp_walk_info gwi; + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + gp_walk_info gwi(timeZoneOffset); gwi.thd = thd; gwi.isGroupByHandler = true; int status = getGroupPlan(gwi, *select_lex, csep, gi); @@ -8597,7 +8594,7 @@ int cp_get_group_plan(THD* thd, SCSEP& csep, cal_impl_if::cal_group_info& gi) else if (status < 0) return status; // Derived table projection and filter optimization. - derivedTableOptimization(thd, csep); + derivedTableOptimization(&gwi, csep); return 0; } @@ -8618,7 +8615,7 @@ int cs_get_derived_plan(ha_columnstore_derived_handler* handler, THD* thd, SCSEP cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; #endif // Derived table projection and filter optimization. - derivedTableOptimization(thd, csep); + derivedTableOptimization(&gwi, csep); return 0; } @@ -8649,7 +8646,7 @@ int cs_get_select_plan(ha_columnstore_select_handler* handler, THD* thd, SCSEP& cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; #endif // Derived table projection and filter optimization. - derivedTableOptimization(thd, csep); + derivedTableOptimization(&gwi, csep); return 0; } @@ -8954,8 +8951,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro else if (join && join->zero_result_cause) { gwi.rcWorkStack.push(new ConstantColumn((int64_t)0, ConstantColumn::NUM)); - (dynamic_cast(gwi.rcWorkStack.top())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(gwi.rcWorkStack.top()))->timeZone(gwi.timeZone); } SELECT_LEX tmp_select_lex; @@ -9443,7 +9439,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro selectSubList.push_back(ssub); SimpleColumn* rc = new SimpleColumn(); rc->colSource(rc->colSource() | SELECT_SUB); - rc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + rc->timeZone(gwi.timeZone); if (sub->get_select_lex()->get_table_list()) { @@ -10401,7 +10397,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro sc1->tableName(sc->tableName()); sc1->tableAlias(sc->tableAlias()); sc1->viewName(sc->viewName()); - sc1->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sc1->timeZone(gwi.timeZone); sc1->colPosition(0); minSc.reset(sc1); } @@ -10462,4 +10458,3 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro } } // namespace cal_impl_if -// vim:ts=4 sw=4: diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 5ef57cf43..853da476f 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -54,7 +54,6 @@ using namespace std; #include #include -#include #include #include "mcs_basic_types.h" @@ -295,7 +294,8 @@ bool onlyOneTableinTM(cal_impl_if::cal_connection_info* ci) return true; } -int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool handler_flag = false) +int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, long timeZone, + bool handler_flag = false) { int rc = HA_ERR_END_OF_FILE; int num_attr = ti.msTablePtr->s->fields; @@ -436,7 +436,7 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h { // fetch and store data (*f)->set_notnull(); - datatypes::StoreFieldMariaDB mf(*f, colType); + datatypes::StoreFieldMariaDB mf(*f, colType, timeZone); h->storeValueToField(row, s, &mf); } } @@ -891,6 +891,10 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c bool isFromSameTable = true; execplan::SCSEP updateCP(new execplan::CalpontSelectExecutionPlan()); + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + updateCP->isDML(true); //@Bug 2753. the memory already freed by destructor of UpdateSqlStatement @@ -1014,9 +1018,9 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c // sysdate() etc. if (!hasNonSupportItem && !cal_impl_if::nonConstFunc(ifp) && tmpVec.size() == 0) { - gp_walk_info gwi; - gwi.thd = thd; - SRCP srcp(buildReturnedColumn(value, gwi, gwi.fatalParseError)); + gp_walk_info gwi2(gwi.timeZone); + gwi2.thd = thd; + SRCP srcp(buildReturnedColumn(value, gwi2, gwi2.fatalParseError)); ConstantColumn* constCol = dynamic_cast(srcp.get()); if (constCol) @@ -1163,7 +1167,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c char buf[64]; gettimeofday(&tv, 0); MySQLTime time; - gmtSecToMySQLTime(tv.tv_sec, time, thd->variables.time_zone->get_name()->ptr()); + gmtSecToMySQLTime(tv.tv_sec, time, timeZoneOffset); sprintf(buf, "%04d-%02d-%02d %02d:%02d:%02d.%06ld", time.year, time.month, time.day, time.hour, time.minute, time.second, tv.tv_usec); columnAssignmentPtr->fScalarExpression = buf; @@ -1314,7 +1318,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c pDMLPackage->set_TableName(tableName); pDMLPackage->set_SchemaName(schemaName); - pDMLPackage->set_TimeZone(thd->variables.time_zone->get_name()->ptr()); + pDMLPackage->set_TimeZone(timeZoneOffset); pDMLPackage->set_IsFromCol(true); // cout << " setting isFromCol to " << isFromCol << endl; @@ -1516,7 +1520,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c CalpontSystemCatalog::TableColName tcn = csc->colName(colrids[minWidthColOffset].objnum); SimpleColumn* sc = new SimpleColumn(tcn.schema, tcn.table, tcn.column, csc->sessionID()); sc->tableAlias(aliasName); - sc->timeZone(thd->variables.time_zone->get_name()->ptr()); + sc->timeZone(timeZoneOffset); sc->resultType(csc->colType(colrids[minWidthColOffset].objnum)); SRCP srcp; srcp.reset(sc); @@ -1660,7 +1664,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c // << endl; VendorDMLStatement cmdStmt("CTRL+C", DML_COMMAND, sessionID); CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(cmdStmt); - pDMLPackage->set_TimeZone(thd->variables.time_zone->get_name()->ptr()); + pDMLPackage->set_TimeZone(timeZoneOffset); ByteStream bytestream; bytestream << static_cast(sessionID); pDMLPackage->write(bytestream); @@ -1783,7 +1787,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c { VendorDMLStatement cmdStmt(command, DML_COMMAND, sessionID); CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(cmdStmt); - pDMLPackage->set_TimeZone(thd->variables.time_zone->get_name()->ptr()); + pDMLPackage->set_TimeZone(timeZoneOffset); pDMLPackage->setTableOid(ci->tableOid); ByteStream bytestream; bytestream << static_cast(sessionID); @@ -2008,6 +2012,10 @@ int ha_mcs_impl_analyze(THD* thd, TABLE* table) execplan::MCSAnalyzeTableExecutionPlan::ReturnedColumnList returnedColumnList; execplan::MCSAnalyzeTableExecutionPlan::ColumnMap columnMap; + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + // Iterate over table oid list and create a `SimpleColumn` for every column with supported type. for (uint32_t i = 0, e = oidlist.size(); i < e; ++i) { @@ -2026,7 +2034,7 @@ int ha_mcs_impl_analyze(THD* thd, TABLE* table) simpleColumn->oid(objNum); simpleColumn->alias(tableColName.column); simpleColumn->resultType(colType); - simpleColumn->timeZone(thd->variables.time_zone->get_name()->ptr()); + simpleColumn->timeZone(timeZoneOffset); returnedColumn.reset(simpleColumn); returnedColumnList.push_back(returnedColumn); @@ -2040,7 +2048,7 @@ int ha_mcs_impl_analyze(THD* thd, TABLE* table) caep->schemaName(table->s->db.str, lower_case_table_names); caep->tableName(table->s->table_name.str, lower_case_table_names); - caep->timeZone(thd->variables.time_zone->get_name()->ptr()); + caep->timeZone(timeZoneOffset); SessionManager sm; BRM::TxnID txnID; @@ -2160,7 +2168,10 @@ int ha_mcs_impl_direct_update_delete_rows(bool execute, ha_rows* affected_rows, const std::vector& condStack) { THD* thd = current_thd; - cal_impl_if::gp_walk_info gwi; + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + cal_impl_if::gp_walk_info gwi(timeZoneOffset); gwi.thd = thd; int rc = 0; @@ -2189,8 +2200,10 @@ int ha_mcs::impl_rnd_init(TABLE* table, const std::vector& condStack) { IDEBUG(cout << "rnd_init for table " << table->s->table_name.str << endl); THD* thd = current_thd; - - gp_walk_info gwi; + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + gp_walk_info gwi(timeZoneOffset); gwi.thd = thd; if (thd->slave_thread && !get_replication_slave(thd) && isDMLStatement(thd->lex->sql_command)) @@ -2333,7 +2346,7 @@ int ha_mcs::impl_rnd_init(TABLE* table, const std::vector& condStack) ti.msTablePtr = table; // send plan whenever rnd_init is called - cp_get_table_plan(thd, ti.csep, ti); + cp_get_table_plan(thd, ti.csep, ti, timeZoneOffset); } IDEBUG(cerr << table->s->table_name.str << " send plan:" << endl); @@ -2563,7 +2576,7 @@ internal_error: return ER_INTERNAL_ERROR; } -int ha_mcs_impl_rnd_next(uchar* buf, TABLE* table) +int ha_mcs_impl_rnd_next(uchar* buf, TABLE* table, long timeZone) { THD* thd = current_thd; @@ -2606,7 +2619,7 @@ int ha_mcs_impl_rnd_next(uchar* buf, TABLE* table) try { - rc = fetchNextRow(buf, ti, ci); + rc = fetchNextRow(buf, ti, ci, timeZone); } catch (std::exception& e) { @@ -2846,7 +2859,7 @@ int ha_mcs_impl_delete_table(const char* name) int rc = ha_mcs_impl_delete_table_(dbName, name, *ci); return rc; } -int ha_mcs_impl_write_row(const uchar* buf, TABLE* table, uint64_t rows_changed) +int ha_mcs_impl_write_row(const uchar* buf, TABLE* table, uint64_t rows_changed, long timeZone) { THD* thd = current_thd; @@ -2893,7 +2906,7 @@ int ha_mcs_impl_write_row(const uchar* buf, TABLE* table, uint64_t rows_changed) ((thd->lex)->sql_command == SQLCOM_LOAD) || ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT) || ci->isCacheInsert)) { - rc = ha_mcs_impl_write_batch_row_(buf, table, *ci); + rc = ha_mcs_impl_write_batch_row_(buf, table, *ci, timeZone); } else { @@ -3894,7 +3907,10 @@ COND* ha_mcs_impl_cond_push(COND* cond, TABLE* table, std::vector& condSt #ifdef DEBUG_WALK_COND { - gp_walk_info gwi; + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + gp_walk_info gwi(timeZoneOffset); gwi.condPush = true; gwi.sessionid = tid2sid(thd->thread_id); cout << "------------------ cond push -----------------------" << endl; @@ -3906,7 +3922,12 @@ COND* ha_mcs_impl_cond_push(COND* cond, TABLE* table, std::vector& condSt if (!ti.csep) { if (!ti.condInfo) - ti.condInfo = new gp_walk_info(); + { + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + ti.condInfo = new gp_walk_info(timeZoneOffset); + } gp_walk_info* gwi = ti.condInfo; gwi->dropCond = false; @@ -4554,7 +4575,7 @@ internal_error: * HA_ERR_END_OF_FILE if the record set has come to an end * others if something went wrong whilst getting the result set ***********************************************************/ -int ha_mcs_impl_group_by_next(TABLE* table) +int ha_mcs_impl_group_by_next(TABLE* table, long timeZone) { THD* thd = current_thd; @@ -4594,7 +4615,7 @@ int ha_mcs_impl_group_by_next(TABLE* table) { // fetchNextRow interface forces to use buf. unsigned char buf; - rc = fetchNextRow(&buf, ti, ci, true); + rc = fetchNextRow(&buf, ti, ci, timeZone, true); } catch (std::exception& e) { @@ -4801,7 +4822,10 @@ int ha_mcs_impl_pushdown_init(mcs_handler_info* handler_info, TABLE* table) if (thd->slave_thread && !get_replication_slave(thd) && isDMLStatement(thd->lex->sql_command)) return 0; - gp_walk_info gwi; + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + gp_walk_info gwi(timeZoneOffset); gwi.thd = thd; bool err = false; @@ -5237,7 +5261,7 @@ internal_error: return ER_INTERNAL_ERROR; } -int ha_mcs_impl_select_next(uchar* buf, TABLE* table) +int ha_mcs_impl_select_next(uchar* buf, TABLE* table, long timeZone) { THD* thd = current_thd; @@ -5336,7 +5360,7 @@ int ha_mcs_impl_select_next(uchar* buf, TABLE* table) try { - rc = fetchNextRow(buf, ti, ci); + rc = fetchNextRow(buf, ti, ci, timeZone); } catch (std::exception& e) { diff --git a/dbcon/mysql/ha_mcs_impl.h b/dbcon/mysql/ha_mcs_impl.h index a776c783a..dc8da092d 100644 --- a/dbcon/mysql/ha_mcs_impl.h +++ b/dbcon/mysql/ha_mcs_impl.h @@ -29,9 +29,9 @@ extern int ha_mcs_impl_delete_table(const char* name); extern int ha_mcs_impl_analyze(THD* thd, TABLE* table); extern int ha_mcs_impl_open(const char* name, int mode, uint32_t test_if_locked); extern int ha_mcs_impl_close(void); -extern int ha_mcs_impl_rnd_next(uchar* buf, TABLE* table); +extern int ha_mcs_impl_rnd_next(uchar* buf, TABLE* table, long timeZone); extern int ha_mcs_impl_rnd_end(TABLE* table, bool is_derived_hand = false); -extern int ha_mcs_impl_write_row(const uchar* buf, TABLE* table, uint64_t rows_changed); +extern int ha_mcs_impl_write_row(const uchar* buf, TABLE* table, uint64_t rows_changed, long timeZone); extern void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table, bool is_cache_insert = false); extern int ha_mcs_impl_end_bulk_insert(bool abort, TABLE* table); extern int ha_mcs_impl_rename_table(const char* from, const char* to); @@ -45,9 +45,9 @@ extern int ha_mcs_impl_direct_update_delete_rows(bool execute, ha_rows* affected extern int ha_mcs_impl_delete_row(); extern int ha_mcs_impl_rnd_pos(uchar* buf, uchar* pos); extern int ha_mcs_impl_pushdown_init(mcs_handler_info* handler_info, TABLE* table); -extern int ha_mcs_impl_select_next(uchar* buf, TABLE* table); +extern int ha_mcs_impl_select_next(uchar* buf, TABLE* table, long timeZone); extern int ha_mcs_impl_group_by_init(mcs_handler_info* handler_info, TABLE* table); -extern int ha_mcs_impl_group_by_next(TABLE* table); +extern int ha_mcs_impl_group_by_next(TABLE* table, long timeZone); extern int ha_mcs_impl_group_by_end(TABLE* table); #endif @@ -59,7 +59,8 @@ extern int ha_mcs_impl_group_by_end(TABLE* table); extern int ha_mcs_impl_rename_table_(const char* from, const char* to, cal_impl_if::cal_connection_info& ci); extern int ha_mcs_impl_write_row_(const uchar* buf, TABLE* table, cal_impl_if::cal_connection_info& ci, ha_rows& rowsInserted); -extern int ha_mcs_impl_write_batch_row_(const uchar* buf, TABLE* table, cal_impl_if::cal_connection_info& ci); +extern int ha_mcs_impl_write_batch_row_(const uchar* buf, TABLE* table, cal_impl_if::cal_connection_info& ci, + long timeZone); extern int ha_mcs_impl_write_last_batch(TABLE* table, cal_impl_if::cal_connection_info& ci, bool abort); extern int ha_mcs_impl_commit_(handlerton* hton, THD* thd, bool all, cal_impl_if::cal_connection_info& ci); extern int ha_mcs_impl_rollback_(handlerton* hton, THD* thd, bool all, cal_impl_if::cal_connection_info& ci); diff --git a/dbcon/mysql/ha_mcs_impl_if.h b/dbcon/mysql/ha_mcs_impl_if.h index e3ac09932..f3ebd7f42 100644 --- a/dbcon/mysql/ha_mcs_impl_if.h +++ b/dbcon/mysql/ha_mcs_impl_if.h @@ -165,6 +165,7 @@ struct gp_walk_info bool cs_vtable_impossible_where_on_union; bool isGroupByHandler; + long timeZone; // MCOL-4617 The below 2 fields are used for in-to-exists // predicate creation and injection. See usage in InSub::transform() @@ -176,7 +177,7 @@ struct gp_walk_info TableOnExprList tableOnExprList; std::vector condList; - gp_walk_info() + gp_walk_info(long timeZone_) : sessionid(0) , fatalParseError(false) , condPush(false) @@ -197,6 +198,7 @@ struct gp_walk_info , cs_vtable_is_update_with_derive(false) , cs_vtable_impossible_where_on_union(false) , isGroupByHandler(false) + , timeZone(timeZone_) , inSubQueryLHS(nullptr) , inSubQueryLHSItem(nullptr) { @@ -391,7 +393,7 @@ const std::string infinidb_err_msg = "distributed syntax or consider changing the MariaDB Columnstore Operating Mode (infinidb_vtable_mode)."; int cp_get_plan(THD* thd, execplan::SCSEP& csep); -int cp_get_table_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_table_info& ti); +int cp_get_table_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_table_info& ti, long timeZone); int cp_get_group_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_group_info& gi); int cs_get_derived_plan(ha_columnstore_derived_handler* handler, THD* thd, execplan::SCSEP& csep, gp_walk_info& gwi); @@ -428,14 +430,14 @@ execplan::ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi); execplan::ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& nonSupport); execplan::ReturnedColumn* buildPseudoColumn(Item* item, gp_walk_info& gwi, bool& nonSupport, uint32_t pseudoType); -void addIntervalArgs(THD* thd, Item_func* ifp, funcexp::FunctionParm& functionParms); -void castCharArgs(THD* thd, Item_func* ifp, funcexp::FunctionParm& functionParms); -void castDecimalArgs(THD* thd, Item_func* ifp, funcexp::FunctionParm& functionParms); -void castTypeArgs(THD* thd, Item_func* ifp, funcexp::FunctionParm& functionParms); +void addIntervalArgs(gp_walk_info* gwip, Item_func* ifp, funcexp::FunctionParm& functionParms); +void castCharArgs(gp_walk_info* gwip, Item_func* ifp, funcexp::FunctionParm& functionParms); +void castDecimalArgs(gp_walk_info* gwip, Item_func* ifp, funcexp::FunctionParm& functionParms); +void castTypeArgs(gp_walk_info* gwip, Item_func* ifp, funcexp::FunctionParm& functionParms); // void parse_item (Item* item, std::vector& field_vec, bool& hasNonSupportItem, uint16& // parseInfo); bool isPredicateFunction(Item* item, gp_walk_info* gwip); -execplan::ParseTree* buildRowPredicate(THD* thd, execplan::RowColumn* lhs, execplan::RowColumn* rhs, +execplan::ParseTree* buildRowPredicate(gp_walk_info* gwip, execplan::RowColumn* lhs, execplan::RowColumn* rhs, std::string predicateOp); bool buildRowColumnFilter(gp_walk_info* gwip, execplan::RowColumn* rhs, execplan::RowColumn* lhs, Item_func* ifp); @@ -448,13 +450,13 @@ std::string getViewName(TABLE_LIST* table_ptr); bool buildConstPredicate(Item_func* ifp, execplan::ReturnedColumn* rhs, gp_walk_info* gwip); execplan::CalpontSystemCatalog::ColType fieldType_MysqlToIDB(const Field* field); execplan::CalpontSystemCatalog::ColType colType_MysqlToIDB(const Item* item); -execplan::SPTP getIntervalType(THD* thd, int interval_type); +execplan::SPTP getIntervalType(gp_walk_info* gwip, int interval_type); uint32_t isPseudoColumn(std::string funcName); void setDerivedTable(execplan::ParseTree* n); -execplan::ParseTree* setDerivedFilter(THD* thd, execplan::ParseTree*& n, +execplan::ParseTree* setDerivedFilter(gp_walk_info* gwip, execplan::ParseTree*& n, std::map& obj, execplan::CalpontSelectExecutionPlan::SelectList& derivedTbList); -void derivedTableOptimization(THD* thd, execplan::SCSEP& csep); +void derivedTableOptimization(gp_walk_info* gwip, execplan::SCSEP& csep); bool buildEqualityPredicate(execplan::ReturnedColumn* lhs, execplan::ReturnedColumn* rhs, gp_walk_info* gwip, boost::shared_ptr& sop, const Item_func::Functype& funcType, const std::vector& itemList, bool isInSubs = false); diff --git a/dbcon/mysql/ha_mcs_partition.cpp b/dbcon/mysql/ha_mcs_partition.cpp index 8ed8c59f3..c65c79f46 100644 --- a/dbcon/mysql/ha_mcs_partition.cpp +++ b/dbcon/mysql/ha_mcs_partition.cpp @@ -385,7 +385,10 @@ void partitionByValue_common(UDF_ARGS* args, // inp csc->identity(CalpontSystemCatalog::FE); OID_t oid = csc->lookupOID(tcn); CalpontSystemCatalog::ColType ct = csc->colType(oid); - datatypes::SessionParam sp(current_thd->variables.time_zone->get_name()->ptr()); + const char* timeZone = current_thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + datatypes::SessionParam sp(timeZoneOffset); datatypes::SimpleValue startVal; datatypes::SimpleValue endVal; datatypes::round_style_t rfMin = datatypes::round_style_t::NONE; @@ -1252,7 +1255,10 @@ extern "C" string schema, table, column; CalpontSystemCatalog::ColType ct; string errMsg; - datatypes::SessionParam sp(current_thd->variables.time_zone->get_name()->ptr()); + const char* timeZone = current_thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + datatypes::SessionParam sp(timeZoneOffset); datatypes::SimpleValue startVal; datatypes::SimpleValue endVal; datatypes::round_style_t rfMin = datatypes::round_style_t::NONE; diff --git a/dbcon/mysql/ha_mcs_pushdown.cpp b/dbcon/mysql/ha_mcs_pushdown.cpp index cefe23986..619caff4c 100644 --- a/dbcon/mysql/ha_mcs_pushdown.cpp +++ b/dbcon/mysql/ha_mcs_pushdown.cpp @@ -581,6 +581,8 @@ ha_columnstore_derived_handler::ha_columnstore_derived_handler(THD* thd, TABLE_L : derived_handler(thd, mcs_hton) { derived = dt; + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &time_zone); } /*********************************************************** @@ -625,7 +627,7 @@ int ha_columnstore_derived_handler::next_row() { DBUG_ENTER("ha_columnstore_derived_handler::next_row"); - int rc = ha_mcs_impl_rnd_next(table->record[0], table); + int rc = ha_mcs_impl_rnd_next(table->record[0], table, time_zone); DBUG_RETURN(rc); } @@ -670,6 +672,8 @@ ha_mcs_group_by_handler::ha_mcs_group_by_handler(THD* thd_arg, Query* query) , order_by(query->order_by) , having(query->having) { + const char* timeZone = thd_arg->variables.time_zone->get_name()->ptr(); + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &time_zone); } /*********************************************************** @@ -705,7 +709,7 @@ int ha_mcs_group_by_handler::init_scan() int ha_mcs_group_by_handler::next_row() { DBUG_ENTER("ha_mcs_group_by_handler::next_row"); - int rc = ha_mcs_impl_group_by_next(table); + int rc = ha_mcs_impl_group_by_next(table, time_zone); DBUG_RETURN(rc); } @@ -985,6 +989,8 @@ ha_columnstore_select_handler::ha_columnstore_select_handler(THD* thd, SELECT_LE , pushdown_init_rc(0) { select = select_lex; + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &time_zone); } /*********************************************************** @@ -1028,7 +1034,7 @@ int ha_columnstore_select_handler::next_row() { DBUG_ENTER("ha_columnstore_select_handler::next_row"); - int rc = ha_mcs_impl_select_next(table->record[0], table); + int rc = ha_mcs_impl_select_next(table->record[0], table, time_zone); DBUG_RETURN(rc); } diff --git a/dbcon/mysql/ha_mcs_pushdown.h b/dbcon/mysql/ha_mcs_pushdown.h index dc127bb2e..80ca3cff8 100644 --- a/dbcon/mysql/ha_mcs_pushdown.h +++ b/dbcon/mysql/ha_mcs_pushdown.h @@ -76,6 +76,9 @@ struct mcs_handler_info ***********************************************************/ class ha_mcs_group_by_handler : public group_by_handler { + private: + long time_zone; + public: ha_mcs_group_by_handler(THD* thd_arg, Query* query); ~ha_mcs_group_by_handler(); @@ -109,6 +112,7 @@ class ha_columnstore_derived_handler : public derived_handler { private: COLUMNSTORE_SHARE* share; + long time_zone; public: ha_columnstore_derived_handler(THD* thd_arg, TABLE_LIST* tbl); @@ -138,6 +142,7 @@ class ha_columnstore_select_handler : public select_handler COLUMNSTORE_SHARE* share; bool prepared; bool scan_ended; + long time_zone; public: bool scan_initialized; diff --git a/dbcon/mysql/ha_pseudocolumn.cpp b/dbcon/mysql/ha_pseudocolumn.cpp index fb87a76cc..a9341049a 100644 --- a/dbcon/mysql/ha_pseudocolumn.cpp +++ b/dbcon/mysql/ha_pseudocolumn.cpp @@ -494,7 +494,7 @@ execplan::ReturnedColumn* buildPseudoColumn(Item* item, gp_walk_info& gwi, bool& cc = new ConstantColumn(localPm); else cc = new ConstantColumn("", ConstantColumn::NULLDATA); - cc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + cc->timeZone(gwi.timeZone); cc->alias(ifp->full_name() ? ifp->full_name() : ""); return cc; @@ -556,7 +556,7 @@ execplan::ReturnedColumn* buildPseudoColumn(Item* item, gp_walk_info& gwi, bool& parms.push_back(sptp); fc->functionParms(parms); fc->expressionId(ci->expressionId++); - fc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + fc->timeZone(gwi.timeZone); // string result type CalpontSystemCatalog::ColType ct; @@ -579,4 +579,3 @@ execplan::ReturnedColumn* buildPseudoColumn(Item* item, gp_walk_info& gwi, bool& } } // namespace cal_impl_if -// vim:ts=4 sw=4: diff --git a/dbcon/mysql/ha_scalar_sub.cpp b/dbcon/mysql/ha_scalar_sub.cpp index 4a4f08fca..b12ea4791 100644 --- a/dbcon/mysql/ha_scalar_sub.cpp +++ b/dbcon/mysql/ha_scalar_sub.cpp @@ -109,8 +109,7 @@ execplan::ParseTree* ScalarSub::transform() { fSub = (Item_subselect*)(fFunc->arguments()[0]); fColumn.reset(new ConstantColumn("", ConstantColumn::NULLDATA)); - (dynamic_cast(fColumn.get())) - ->timeZone(fGwip.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(fColumn.get()))->timeZone(fGwip.timeZone); delete rhs; return buildParseTree(op); } @@ -176,7 +175,7 @@ execplan::ParseTree* ScalarSub::transform_between() SOP sop; sop.reset(op_LE); rhs = new ParseTree(new SimpleFilter(sop, fColumn.get(), op3)); - (dynamic_cast(rhs->data()))->timeZone(fGwip.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(rhs->data()))->timeZone(fGwip.timeZone); } SubSelect* sub1 = dynamic_cast(op2); @@ -192,7 +191,7 @@ execplan::ParseTree* ScalarSub::transform_between() SOP sop; sop.reset(op_GE); lhs = new ParseTree(new SimpleFilter(sop, fColumn.get(), op2)); - (dynamic_cast(lhs->data()))->timeZone(fGwip.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(lhs->data()))->timeZone(fGwip.timeZone); } if (!rhs || !lhs) @@ -245,7 +244,7 @@ execplan::ParseTree* ScalarSub::buildParseTree(PredicateOperator* op) csep->subType(CalpontSelectExecutionPlan::SINGLEROW_SUBS); // gwi for the sub query - gp_walk_info gwi; + gp_walk_info gwi(fGwip.timeZone); gwi.thd = fGwip.thd; gwi.subQuery = this; diff --git a/dbcon/mysql/ha_select_sub.cpp b/dbcon/mysql/ha_select_sub.cpp index 897da428e..2bce37765 100644 --- a/dbcon/mysql/ha_select_sub.cpp +++ b/dbcon/mysql/ha_select_sub.cpp @@ -67,7 +67,7 @@ SCSEP SelectSubQuery::transform() csep->subType(CalpontSelectExecutionPlan::SELECT_SUBS); // gwi for the sub query - gp_walk_info gwi; + gp_walk_info gwi(fGwip.timeZone); gwi.thd = fGwip.thd; gwi.subQuery = this; diff --git a/dbcon/mysql/ha_view.cpp b/dbcon/mysql/ha_view.cpp index c37907e98..4912fd71d 100644 --- a/dbcon/mysql/ha_view.cpp +++ b/dbcon/mysql/ha_view.cpp @@ -65,7 +65,7 @@ void View::transform() csep->sessionID(fParentGwip->sessionid); // gwi for the sub query - gp_walk_info gwi; + gp_walk_info gwi(fParentGwip->timeZone); gwi.thd = fParentGwip->thd; uint32_t sessionID = csep->sessionID(); diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index 8111e14b0..e94372cea 100644 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -145,27 +145,25 @@ ReturnedColumn* buildBoundExp(WF_Boundary& bound, SRCP& order, gp_walk_info& gwi // put interval val column to bound (dynamic_cast(rc))->functionName(funcName); - (dynamic_cast(rc))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(rc))->timeZone(gwi.timeZone); sptp.reset(new ParseTree(order->clone())); funcParms.push_back(sptp); sptp.reset(new ParseTree(intervalCol->val()->clone())); funcParms.push_back(sptp); - funcParms.push_back(getIntervalType(gwi.thd, intervalCol->intervalType())); + funcParms.push_back(getIntervalType(&gwi, intervalCol->intervalType())); SRCP srcp(intervalCol->val()); bound.fVal = srcp; if (addOp) { sptp.reset(new ParseTree(new ConstantColumn("ADD"))); - (dynamic_cast(sptp->data())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwi.timeZone); funcParms.push_back(sptp); } else { sptp.reset(new ParseTree(new ConstantColumn("SUB"))); - (dynamic_cast(sptp->data())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwi.timeZone); funcParms.push_back(sptp); } @@ -187,7 +185,7 @@ ReturnedColumn* buildBoundExp(WF_Boundary& bound, SRCP& order, gp_walk_info& gwi else aop = new ArithmeticOperator("-"); - aop->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + aop->timeZone(gwi.timeZone); ParseTree* pt = new ParseTree(aop); ParseTree *lhs = 0, *rhs = 0; lhs = new ParseTree(order->clone()); @@ -314,7 +312,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n Item_sum* item_sum = wf->window_func(); string funcName = ConvertFuncName(item_sum); WindowFunctionColumn* ac = new WindowFunctionColumn(funcName); - ac->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + ac->timeZone(gwi.timeZone); ac->distinct(item_sum->has_with_distinct()); Window_spec* win_spec = wf->window_spec; SRCP srcp; @@ -407,45 +405,45 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n sprintf(sRespectNulls, "%lu", bRespectNulls); srcp.reset(new ConstantColumn(sRespectNulls, (uint64_t)bRespectNulls, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); funcParms.push_back(srcp); break; } case Item_sum::FIRST_VALUE_FUNC: srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // OFFSET (always one) - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); funcParms.push_back(srcp); srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // FROM_FIRST - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); funcParms.push_back(srcp); srcp.reset( new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); funcParms.push_back(srcp); break; case Item_sum::LAST_VALUE_FUNC: srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // OFFSET (always one) - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); funcParms.push_back(srcp); srcp.reset(new ConstantColumn("0", (uint64_t)0, ConstantColumn::NUM)); // FROM_LAST - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); funcParms.push_back(srcp); srcp.reset( new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); funcParms.push_back(srcp); break; case Item_sum::NTH_VALUE_FUNC: // When the front end supports these paramters, this needs modification srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // FROM FIRST/LAST 1 => FIRST - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); funcParms.push_back(srcp); srcp.reset( new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); funcParms.push_back(srcp); break; @@ -453,11 +451,11 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n case Item_sum::LAG_FUNC: // When the front end supports these paramters, this needs modification srcp.reset(new ConstantColumn("", ConstantColumn::NULLDATA)); // Default to fill in for NULL values - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); funcParms.push_back(srcp); srcp.reset( new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); funcParms.push_back(srcp); break; @@ -802,8 +800,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n bound = 1; srcp.reset(new ConstantColumn((int64_t)bound)); - (dynamic_cast(srcp.get())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); frm.fStart.fVal = srcp; frm.fStart.fBound.reset(buildBoundExp(frm.fStart, srcp, gwi)); @@ -819,8 +816,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n bound = 1; srcp.reset(new ConstantColumn((int64_t)bound)); - (dynamic_cast(srcp.get())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); frm.fEnd.fVal = srcp; frm.fEnd.fBound.reset(buildBoundExp(frm.fEnd, srcp, gwi)); diff --git a/dbcon/mysql/idb_mysql.h b/dbcon/mysql/idb_mysql.h index 47f74f89a..6798e6fca 100644 --- a/dbcon/mysql/idb_mysql.h +++ b/dbcon/mysql/idb_mysql.h @@ -118,4 +118,3 @@ inline char* idb_mysql_query_str(THD* thd) } } // namespace -// vim:ts=4 sw=4: diff --git a/dbcon/mysql/install_mcs_mysql.sh.in b/dbcon/mysql/install_mcs_mysql.sh.in index 1bd6e371a..9b2dddf4e 100755 --- a/dbcon/mysql/install_mcs_mysql.sh.in +++ b/dbcon/mysql/install_mcs_mysql.sh.in @@ -19,16 +19,25 @@ fi # DELETE libcalmysql.so entries first as they are in ha_columnstore.so in 1.4.2 onwards $MDB 2> ${tmpdir}/mysql_install.log <fTimeZone = alterTableStmt.fTimeZone; + processor->fTimeZone = alterTableStmt.getTimeZone(); result = processor->processPackage(alterTableStmt); @@ -876,4 +876,3 @@ int DDLProcessor::commitTransaction(uint32_t txnID, std::string& errorMsg) return rc; } } // namespace ddlprocessor -// vim:ts=4 sw=4: diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh deleted file mode 100755 index 75df5e2bb..000000000 --- a/debian/autobake-deb.sh +++ /dev/null @@ -1,130 +0,0 @@ -#!/bin/bash -# -# Build MariaDB .deb packages for test and release at mariadb.org -# -# Purpose of this script: -# Always keep the actual packaging as up-to-date as possible following the latest -# Debian policy and targeting Debian Sid. Then case-by-case run in autobake-deb.sh -# tests for backwards compatibility and strip away parts on older builders or -# specfic build environments. - -# Exit immediately on any error -set -e - -# This file is invocated from Buildbot and Travis-CI to build deb packages. -# As both of those CI systems have many parallel jobs that include different -# parts of the test suite, we don't need to run the mysql-test-run at all when -# building the deb packages here. -export DEB_BUILD_OPTIONS="nocheck $DEB_BUILD_OPTIONS" - -if [[ -d storage/columnstore/columnstore/debian ]]; then - cp -v storage/columnstore/columnstore/debian/mariadb-plugin-columnstore.* debian/ - echo >> debian/control - cat storage/columnstore/columnstore/debian/control >> debian/control -fi - -# General CI optimizations to keep build output smaller -if [[ $TRAVIS ]] || [[ $GITLAB_CI ]] -then - # On both Travis and Gitlab the output log must stay under 4MB so make the - # build less verbose - sed '/Add support for verbose builds/,/^$/d' -i debian/rules - - # MCOL-4149: ColumnStore builds are so slow and big that they must be skipped on - # both Travis-CI and Gitlab-CI - sed 's|-DPLUGIN_COLUMNSTORE=YES|-DPLUGIN_COLUMNSTORE=NO|' -i debian/rules - sed "/Package: mariadb-plugin-columnstore/,/^$/d" -i debian/control -fi - -# Don't build or try to put files in a package for selected plugins and compontents on Travis-CI -# in order to keep build small (in both duration and disk space) -if [[ $TRAVIS ]] -then - # Test suite package not relevant on Travis-CI - sed 's|DINSTALL_MYSQLTESTDIR=share/mysql/mysql-test|DINSTALL_MYSQLTESTDIR=false|' -i debian/rules - sed '/Package: mariadb-test-data/,/^$/d' -i debian/control - sed '/Package: mariadb-test$/,/^$/d' -i debian/control - - # Extra plugins such as Mroonga, Spider, OQgraph, Sphinx and the embedded build can safely be skipped - sed 's|-DDEB|-DPLUGIN_MROONGA=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_PERFSCHEMA=NO -DPLUGIN_SPHINX=NO -DWITH_EMBEDDED_SERVER=OFF -DDEB|' -i debian/rules - sed "/Package: mariadb-plugin-mroonga/,/^$/d" -i debian/control - sed "/Package: mariadb-plugin-rocksdb/,/^$/d" -i debian/control - sed "/Package: mariadb-plugin-spider/,/^$/d" -i debian/control - sed "/Package: mariadb-plugin-oqgraph/,/^$/d" -i debian/control - sed "/ha_sphinx.so/d" -i debian/mariadb-server-10.*.install - sed "/Package: libmariadbd19/,/^$/d" -i debian/control - sed "/Package: libmariadbd-dev/,/^$/d" -i debian/control -fi - -if [[ $(arch) =~ i[346]86 ]] -then - sed "/Package: mariadb-plugin-rocksdb/,/^$/d" -i debian/control -fi - -# If rocksdb-tools is not available (before Debian Buster and Ubuntu Disco) -# remove the dependency from the RocksDB plugin so it can install properly -# and instead ship the one built from MariaDB sources -if ! apt-cache madison rocksdb-tools | grep 'rocksdb-tools' >/dev/null 2>&1 -then - sed '/rocksdb-tools/d' -i debian/control - sed '/sst_dump/d' -i debian/not-installed - echo "usr/bin/sst_dump" >> debian/mariadb-plugin-rocksdb.install -fi - -# From Debian Buster/Ubuntu Bionic, libcurl4 replaces libcurl3. -if ! apt-cache madison libcurl4 | grep 'libcurl4' >/dev/null 2>&1 -then - sed 's/libcurl4/libcurl3/g' -i debian/control -fi - -# Adjust changelog, add new version -echo "Incrementing changelog and starting build scripts" - -# Find major.minor version -source ./VERSION -UPSTREAM="${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}" -PATCHLEVEL="+maria" -LOGSTRING="MariaDB build" -CODENAME="$(lsb_release -sc)" -EPOCH="1:" - -dch -b -D "${CODENAME}" -v "${EPOCH}${UPSTREAM}${PATCHLEVEL}~${CODENAME}" "Automatic build with ${LOGSTRING}." - -echo "Creating package version ${EPOCH}${UPSTREAM}${PATCHLEVEL}~${CODENAME} ... " - -# On Travis CI and Gitlab-CI, use -b to build binary only packages as there is -# no need to waste time on generating the source package. -if [[ $TRAVIS ]] -then - BUILDPACKAGE_FLAGS="-b" -fi - -# Use eatmydata is available to build faster with less I/O, skipping fsync() -# during the entire build process (safe because a build can always be restarted) -if which eatmydata > /dev/null -then - BUILDPACKAGE_PREPEND=eatmydata -fi - -# Build the package -# Pass -I so that .git and other unnecessary temporary and source control files -# will be ignored by dpkg-source when creating the tar.gz source package. -fakeroot $BUILDPACKAGE_PREPEND dpkg-buildpackage -us -uc -I $BUILDPACKAGE_FLAGS || sleep 600 - -# If the step above fails due to missing dependencies, you can manually run -# sudo mk-build-deps debian/control -r -i - -# Don't log package contents on Travis-CI or Gitlab-CI to save time and log size -if [[ ! $TRAVIS ]] && [[ ! $GITLAB_CI ]] -then - echo "List package contents ..." - cd .. - for package in *.deb - do - echo "$package" | cut -d '_' -f 1 - dpkg-deb -c "$package" | awk '{print $1 " " $2 " " $6 " " $7 " " $8}' | sort -k 3 - echo "------------------------------------------------" - done -fi - -echo "Build complete" diff --git a/debian/control b/debian/control index 9df70ecb8..20c09024a 100644 --- a/debian/control +++ b/debian/control @@ -4,7 +4,7 @@ Depends: binutils, libjemalloc1 | libjemalloc2, libsnappy1 | libsnappy1v5, liblz4-1, - mariadb-server-10.7 (= ${server:Version}), + mariadb-server-10.8 (= ${server:Version}), net-tools, python3, ${misc:Depends}, diff --git a/debian/mariadb-plugin-columnstore.install b/debian/mariadb-plugin-columnstore.install index 39f4e8c72..a921e36f2 100644 --- a/debian/mariadb-plugin-columnstore.install +++ b/debian/mariadb-plugin-columnstore.install @@ -6,7 +6,6 @@ etc/mysql/mariadb.conf.d/columnstore.cnf usr/bin/mcsRebuildEM usr/bin/DDLProc usr/bin/DMLProc -usr/bin/ExeMgr usr/bin/PrimProc usr/bin/StorageManager usr/bin/WriteEngineServer @@ -106,7 +105,6 @@ usr/share/columnstore/mariadb-columnstore.service usr/share/columnstore/mcs-controllernode.service usr/share/columnstore/mcs-ddlproc.service usr/share/columnstore/mcs-dmlproc.service -usr/share/columnstore/mcs-exemgr.service usr/share/columnstore/mcs-loadbrm.service usr/share/columnstore/mcs-primproc.service usr/share/columnstore/mcs-storagemanager.service diff --git a/dmlproc/batchinsertprocessor.cpp b/dmlproc/batchinsertprocessor.cpp index 1a61e1eb6..483074179 100644 --- a/dmlproc/batchinsertprocessor.cpp +++ b/dmlproc/batchinsertprocessor.cpp @@ -547,4 +547,3 @@ void BatchInsertProc::getError(int& errorCode, std::string& errMsg) errMsg = fErrMsg; } } // namespace dmlprocessor -// vim:ts=4 sw=4: diff --git a/dmlproc/batchinsertprocessor.h b/dmlproc/batchinsertprocessor.h index 270bd592a..725581d20 100644 --- a/dmlproc/batchinsertprocessor.h +++ b/dmlproc/batchinsertprocessor.h @@ -93,4 +93,3 @@ class BatchInsertProc }; } // namespace dmlprocessor -// vim:ts=4 sw=4: diff --git a/dmlproc/dmlproc.cpp b/dmlproc/dmlproc.cpp index f1fa9cc4c..fb79ae59f 100644 --- a/dmlproc/dmlproc.cpp +++ b/dmlproc/dmlproc.cpp @@ -639,14 +639,7 @@ int ServiceDMLProc::Child() // Couldn't check the return code b/c // fuser returns 1 for unused port. -#if defined(__GNUC__) && __GNUC__ >= 5 -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-result" - (void)::system(cmd.c_str()); -#pragma GCC diagnostic pop -#else - (void)::system(cmd.c_str()); -#endif + std::ignore = ::system(cmd.c_str()); } catch (...) { @@ -698,4 +691,3 @@ int main(int argc, char** argv) return ServiceDMLProc(opt).Run(); } -// vim:ts=4 sw=4: diff --git a/dmlproc/dmlprocessor.cpp b/dmlproc/dmlprocessor.cpp index 33e9b3d14..0cc5337a5 100644 --- a/dmlproc/dmlprocessor.cpp +++ b/dmlproc/dmlprocessor.cpp @@ -2001,4 +2001,3 @@ void DMLProcessor::log(const std::string& msg, logging::LOG_TYPE level) } } // namespace dmlprocessor -// vim:ts=4 sw=4: diff --git a/dmlproc/dmlprocessor.h b/dmlproc/dmlprocessor.h index 43b1850ba..fbc7c32d6 100644 --- a/dmlproc/dmlprocessor.h +++ b/dmlproc/dmlprocessor.h @@ -330,4 +330,3 @@ class RollbackTransactionProcessor : public dmlpackageprocessor::DMLPackageProce } // namespace dmlprocessor -// vim:ts=4 sw=4: diff --git a/engine.vpw b/engine.vpw new file mode 100644 index 000000000..159f94b06 --- /dev/null +++ b/engine.vpw @@ -0,0 +1,6 @@ + + + + + + diff --git a/engine.vpwhistu b/engine.vpwhistu new file mode 100644 index 000000000..0462e4325 --- /dev/null +++ b/engine.vpwhistu @@ -0,0 +1,6 @@ +[Global] +CurrentProject=engine.vpj +[ProjectDates] +engine.vpj=20220606154459230 +[ActiveConfig] +engine.vpj=Debug diff --git a/engine.vtg b/engine.vtg new file mode 100644 index 000000000..97a7b702d Binary files /dev/null and b/engine.vtg differ diff --git a/exemgr/CMakeLists.txt b/exemgr/CMakeLists.txt deleted file mode 100644 index e869e4371..000000000 --- a/exemgr/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ - -include_directories( ${ENGINE_COMMON_INCLUDES} ) - - -########### next target ############### - -set(ExeMgr_SRCS main.cpp activestatementcounter.cpp femsghandler.cpp ../utils/common/crashtrace.cpp) - -add_executable(ExeMgr ${ExeMgr_SRCS}) - -target_link_libraries(ExeMgr ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} cacheutils threadpool) - -target_include_directories(ExeMgr PRIVATE ${Boost_INCLUDE_DIRS}) - -install(TARGETS ExeMgr DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) - - -########### install files ############### - - diff --git a/exemgr/ExeMgr.rc b/exemgr/ExeMgr.rc deleted file mode 100644 index 02d4d6817..000000000 --- a/exemgr/ExeMgr.rc +++ /dev/null @@ -1,102 +0,0 @@ -// Microsoft Visual C++ generated resource script. -// -#include "resource.h" - -#define APSTUDIO_READONLY_SYMBOLS -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 2 resource. -// -#include "afxres.h" - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS - -///////////////////////////////////////////////////////////////////////////// -// English (U.S.) resources - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -#ifdef _WIN32 -LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US -#pragma code_page(1252) -#endif //_WIN32 - -#ifdef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// TEXTINCLUDE -// - -1 TEXTINCLUDE -BEGIN - "resource.h\0" -END - -2 TEXTINCLUDE -BEGIN - "#include ""afxres.h""\r\n" - "\0" -END - -3 TEXTINCLUDE -BEGIN - "\r\n" - "\0" -END - -#endif // APSTUDIO_INVOKED - - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,0,0 - PRODUCTVERSION 4,6,0,0 - FILEFLAGSMASK 0x17L -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x4L - FILETYPE 0x1L - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "CompanyName", "InfiniDB, Inc." - VALUE "FileDescription", "InfiniDB Execution Manager" - VALUE "FileVersion", "4.6.0-0" - VALUE "InternalName", "ExeMgr" - VALUE "LegalCopyright", "Copyright (C) 2014" - VALUE "OriginalFilename", "ExeMgr.exe" - VALUE "ProductName", "InfiniDB" - VALUE "ProductVersion", "4.6.0.0 Beta" - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END - -#endif // English (U.S.) resources -///////////////////////////////////////////////////////////////////////////// - - - -#ifndef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 3 resource. -// - - -///////////////////////////////////////////////////////////////////////////// -#endif // not APSTUDIO_INVOKED - diff --git a/exemgr/main.cpp b/exemgr/main.cpp deleted file mode 100644 index 513cc075e..000000000 --- a/exemgr/main.cpp +++ /dev/null @@ -1,1828 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2019 MariaDB Corporation - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/********************************************************************** - * $Id: main.cpp 1000 2013-07-24 21:05:51Z pleblanc $ - * - * - ***********************************************************************/ -/** - * @brief execution plan manager main program - * - * This is the ?main? program for dealing with execution plans and - * result sets. It sits in a loop waiting for CalpontExecutionPlan - * from the FEP. It then passes the CalpontExecutionPlan to the - * JobListFactory from which a JobList is obtained. This is passed - * to to the Query Manager running in the real-time portion of the - * EC. The ExecutionPlanManager waits until the Query Manager - * returns a result set for the job list. These results are passed - * into the CalpontResultFactory, which outputs a CalpontResultSet. - * The ExecutionPlanManager passes the CalpontResultSet into the - * VendorResultFactory which produces a result set tailored to the - * specific DBMS front end in use. The ExecutionPlanManager then - * sends the VendorResultSet back to the Calpont Database Connector - * on the Front-End Processor where it is returned to the DBMS - * front-end. - */ -#include -#include -#include -#include - -#undef root_name -#include - -#include "calpontselectexecutionplan.h" -#include "mcsanalyzetableexecutionplan.h" -#include "activestatementcounter.h" -#include "distributedenginecomm.h" -#include "resourcemanager.h" -#include "configcpp.h" -#include "queryteleserverparms.h" -#include "iosocket.h" -#include "joblist.h" -#include "joblistfactory.h" -#include "oamcache.h" -#include "simplecolumn.h" -#include "bytestream.h" -#include "telestats.h" -#include "messageobj.h" -#include "messagelog.h" -#include "sqllogger.h" -#include "femsghandler.h" -#include "idberrorinfo.h" -#include "MonitorProcMem.h" -#include "liboamcpp.h" -#include "crashtrace.h" -#include "service.h" - -#include -#include -#include - -#include "dbrm.h" - -#include "mariadb_my_sys.h" -#include "statistics.h" -#include "threadnaming.h" - -class Opt -{ - public: - int m_debug; - bool m_e; - bool m_fg; - Opt(int argc, char* argv[]) : m_debug(0), m_e(false), m_fg(false) - { - int c; - while ((c = getopt(argc, argv, "edf")) != EOF) - { - switch (c) - { - case 'd': m_debug++; break; - - case 'e': m_e = true; break; - - case 'f': m_fg = true; break; - - case '?': - default: break; - } - } - } -}; - -class ServiceExeMgr : public Service, public Opt -{ - protected: - void log(logging::LOG_TYPE type, const std::string& str) - { - logging::LoggingID logid(16); - logging::Message::Args args; - logging::Message message(8); - args.add(strerror(errno)); - message.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(type, message, logid); - } - - public: - ServiceExeMgr(const Opt& opt) : Service("ExeMgr"), Opt(opt) - { - } - void LogErrno() override - { - log(logging::LOG_TYPE_CRITICAL, std::string(strerror(errno))); - } - void ParentLogChildMessage(const std::string& str) override - { - log(logging::LOG_TYPE_INFO, str); - } - int Child() override; - int Run() - { - return m_fg ? Child() : RunForking(); - } -}; - -namespace -{ -// If any flags other than the table mode flags are set, produce output to screeen -const uint32_t flagsWantOutput = (0xffffffff & ~execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_AUTOSWITCH & - ~execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF); - -int gDebug; - -const unsigned logDefaultMsg = logging::M0000; -const unsigned logDbProfStartStatement = logging::M0028; -const unsigned logDbProfEndStatement = logging::M0029; -const unsigned logStartSql = logging::M0041; -const unsigned logEndSql = logging::M0042; -const unsigned logRssTooBig = logging::M0044; -const unsigned logDbProfQueryStats = logging::M0047; -const unsigned logExeMgrExcpt = logging::M0055; - -logging::Logger msgLog(16); - -typedef std::map SessionMemMap_t; -SessionMemMap_t sessionMemMap; // track memory% usage during a query -std::mutex sessionMemMapMutex; - -//...The FrontEnd may establish more than 1 connection (which results in -// more than 1 ExeMgr thread) per session. These threads will share -// the same CalpontSystemCatalog object for that session. Here, we -// define a std::map to track how many threads are sharing each session, so -// that we know when we can safely delete a CalpontSystemCatalog object -// shared by multiple threads per session. -typedef std::map ThreadCntPerSessionMap_t; -ThreadCntPerSessionMap_t threadCntPerSessionMap; -std::mutex threadCntPerSessionMapMutex; - -// This var is only accessed using thread-safe inc/dec calls -ActiveStatementCounter* statementsRunningCount; - -joblist::DistributedEngineComm* ec; - -auto rm = joblist::ResourceManager::instance(true); - -int toInt(const std::string& val) -{ - if (val.length() == 0) - return -1; - - return static_cast(config::Config::fromText(val)); -} - -const std::string ExeMgr("ExeMgr1"); - -const std::string prettyPrintMiniInfo(const std::string& in) -{ - // 1. take the std::string and tok it by '\n' - // 2. for each part in each line calc the longest part - // 3. padding to each longest value, output a header and the lines - typedef boost::tokenizer > my_tokenizer; - boost::char_separator sep1("\n"); - my_tokenizer tok1(in, sep1); - std::vector lines; - std::string header = "Desc Mode Table TableOID ReferencedColumns PIO LIO PBE Elapsed Rows"; - const int header_parts = 10; - lines.push_back(header); - - for (my_tokenizer::const_iterator iter1 = tok1.begin(); iter1 != tok1.end(); ++iter1) - { - if (!iter1->empty()) - lines.push_back(*iter1); - } - - std::vector lens; - - for (int i = 0; i < header_parts; i++) - lens.push_back(0); - - std::vector > lineparts; - std::vector::iterator iter2; - int j; - - for (iter2 = lines.begin(), j = 0; iter2 != lines.end(); ++iter2, j++) - { - boost::char_separator sep2(" "); - my_tokenizer tok2(*iter2, sep2); - int i; - std::vector parts; - my_tokenizer::iterator iter3; - - for (iter3 = tok2.begin(), i = 0; iter3 != tok2.end(); ++iter3, i++) - { - if (i >= header_parts) - break; - - std::string part(*iter3); - - if (j != 0 && i == 8) - part.resize(part.size() - 3); - - assert(i < header_parts); - - if (part.size() > lens[i]) - lens[i] = part.size(); - - parts.push_back(part); - } - - assert(i == header_parts); - lineparts.push_back(parts); - } - - std::ostringstream oss; - - std::vector >::iterator iter1 = lineparts.begin(); - std::vector >::iterator end1 = lineparts.end(); - - oss << "\n"; - - while (iter1 != end1) - { - std::vector::iterator iter2 = iter1->begin(); - std::vector::iterator end2 = iter1->end(); - assert(distance(iter2, end2) == header_parts); - int i = 0; - - while (iter2 != end2) - { - assert(i < header_parts); - oss << std::setw(lens[i]) << std::left << *iter2 << " "; - ++iter2; - i++; - } - - oss << "\n"; - ++iter1; - } - - return oss.str(); -} - -const std::string timeNow() -{ - time_t outputTime = time(0); - struct tm ltm; - char buf[32]; // ctime(3) says at least 26 - size_t len = 0; -#ifdef _MSC_VER - asctime_s(buf, 32, localtime_r(&outputTime, <m)); -#else - asctime_r(localtime_r(&outputTime, <m), buf); -#endif - len = strlen(buf); - - if (len > 0) - --len; - - if (buf[len] == '\n') - buf[len] = 0; - - return buf; -} - -querytele::QueryTeleServerParms gTeleServerParms; - -class SessionThread -{ - public: - SessionThread(const messageqcpp::IOSocket& ios, joblist::DistributedEngineComm* ec, - joblist::ResourceManager* rm) - : fIos(ios) - , fEc(ec) - , fRm(rm) - , fStatsRetrieved(false) - , fTeleClient(gTeleServerParms) - , fOamCachePtr(oam::OamCache::makeOamCache()) - { - } - - private: - messageqcpp::IOSocket fIos; - joblist::DistributedEngineComm* fEc; - joblist::ResourceManager* fRm; - querystats::QueryStats fStats; - - // Variables used to store return stats - bool fStatsRetrieved; - - querytele::QueryTeleClient fTeleClient; - - oam::OamCache* fOamCachePtr; // this ptr is copyable... - - //...Reinitialize stats for start of a new query - void initStats(uint32_t sessionId, std::string& sqlText) - { - initMaxMemPct(sessionId); - - fStats.reset(); - fStats.setStartTime(); - fStats.fSessionID = sessionId; - fStats.fQuery = sqlText; - fStatsRetrieved = false; - } - - //...Get % memory usage during latest query for sesssionId. - //...SessionId >= 0x80000000 is system catalog query we can ignore. - static uint64_t getMaxMemPct(uint32_t sessionId) - { - uint64_t maxMemoryPct = 0; - - if (sessionId < 0x80000000) - { - std::lock_guard lk(sessionMemMapMutex); - SessionMemMap_t::iterator mapIter = sessionMemMap.find(sessionId); - - if (mapIter != sessionMemMap.end()) - { - maxMemoryPct = (uint64_t)mapIter->second; - } - } - - return maxMemoryPct; - } - - //...Delete sessionMemMap entry for the specified session's memory % use. - //...SessionId >= 0x80000000 is system catalog query we can ignore. - static void deleteMaxMemPct(uint32_t sessionId) - { - if (sessionId < 0x80000000) - { - std::lock_guard lk(sessionMemMapMutex); - SessionMemMap_t::iterator mapIter = sessionMemMap.find(sessionId); - - if (mapIter != sessionMemMap.end()) - { - sessionMemMap.erase(sessionId); - } - } - } - - //...Get and log query stats to specified output stream - const std::string formatQueryStats( - joblist::SJLP& jl, // joblist associated with query - const std::string& label, // header label to print in front of log output - bool includeNewLine, // include line breaks in query stats std::string - bool vtableModeOn, bool wantExtendedStats, uint64_t rowsReturned) - { - std::ostringstream os; - - // Get stats if not already acquired for current query - if (!fStatsRetrieved) - { - if (wantExtendedStats) - { - // wait for the ei data to be written by another thread (brain-dead) - struct timespec req = {0, 250000}; // 250 usec -#ifdef _MSC_VER - Sleep(20); // 20ms on Windows -#else - nanosleep(&req, 0); -#endif - } - - // Get % memory usage during current query for sessionId - jl->querySummary(wantExtendedStats); - fStats = jl->queryStats(); - fStats.fMaxMemPct = getMaxMemPct(fStats.fSessionID); - fStats.fRows = rowsReturned; - fStatsRetrieved = true; - } - - std::string queryMode; - queryMode = (vtableModeOn ? "Distributed" : "Standard"); - - // Log stats to specified output stream - os << label << ": MaxMemPct-" << fStats.fMaxMemPct << "; NumTempFiles-" << fStats.fNumFiles - << "; TempFileSpace-" << roundBytes(fStats.fFileBytes) << "; ApproxPhyI/O-" << fStats.fPhyIO - << "; CacheI/O-" << fStats.fCacheIO << "; BlocksTouched-" << fStats.fMsgRcvCnt; - - if (includeNewLine) - os << std::endl << " "; // insert line break - else - os << "; "; // continue without line break - - os << "PartitionBlocksEliminated-" << fStats.fCPBlocksSkipped << "; MsgBytesIn-" - << roundBytes(fStats.fMsgBytesIn) << "; MsgBytesOut-" << roundBytes(fStats.fMsgBytesOut) << "; Mode-" - << queryMode; - - return os.str(); - } - - //...Increment the number of threads using the specified sessionId - static void incThreadCntPerSession(uint32_t sessionId) - { - std::lock_guard lk(threadCntPerSessionMapMutex); - ThreadCntPerSessionMap_t::iterator mapIter = threadCntPerSessionMap.find(sessionId); - - if (mapIter == threadCntPerSessionMap.end()) - threadCntPerSessionMap.insert(ThreadCntPerSessionMap_t::value_type(sessionId, 1)); - else - mapIter->second++; - } - - //...Decrement the number of threads using the specified sessionId. - //...When the thread count for a sessionId reaches 0, the corresponding - //...CalpontSystemCatalog objects are deleted. - //...The user query and its associated catalog query have a different - //...session Id where the highest bit is flipped. - //...The object with id(sessionId | 0x80000000) cannot be removed before - //...user query session completes because the sysdata may be used for - //...debugging/stats purpose, such as result graph, etc. - static void decThreadCntPerSession(uint32_t sessionId) - { - std::lock_guard lk(threadCntPerSessionMapMutex); - ThreadCntPerSessionMap_t::iterator mapIter = threadCntPerSessionMap.find(sessionId); - - if (mapIter != threadCntPerSessionMap.end()) - { - if (--mapIter->second == 0) - { - threadCntPerSessionMap.erase(mapIter); - execplan::CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId); - execplan::CalpontSystemCatalog::removeCalpontSystemCatalog((sessionId ^ 0x80000000)); - } - } - } - - //...Init sessionMemMap entry for specified session to 0 memory %. - //...SessionId >= 0x80000000 is system catalog query we can ignore. - static void initMaxMemPct(uint32_t sessionId) - { - if (sessionId < 0x80000000) - { - // std::cout << "Setting pct to 0 for session " << sessionId << std::endl; - std::lock_guard lk(sessionMemMapMutex); - SessionMemMap_t::iterator mapIter = sessionMemMap.find(sessionId); - - if (mapIter == sessionMemMap.end()) - { - sessionMemMap[sessionId] = 0; - } - else - { - mapIter->second = 0; - } - } - } - - //... Round off to human readable format (KB, MB, or GB). - const std::string roundBytes(uint64_t value) const - { - const char* units[] = {"B", "KB", "MB", "GB", "TB"}; - uint64_t i = 0, up = 0; - uint64_t roundedValue = value; - - while (roundedValue > 1024 && i < 4) - { - up = (roundedValue & 512); - roundedValue /= 1024; - i++; - } - - if (up) - roundedValue++; - - std::ostringstream oss; - oss << roundedValue << units[i]; - return oss.str(); - } - - //...Round off to nearest (1024*1024) MB - uint64_t roundMB(uint64_t value) const - { - uint64_t roundedValue = value >> 20; - - if (value & 524288) - roundedValue++; - - return roundedValue; - } - - void setRMParms(const execplan::CalpontSelectExecutionPlan::RMParmVec& parms) - { - for (execplan::CalpontSelectExecutionPlan::RMParmVec::const_iterator it = parms.begin(); - it != parms.end(); ++it) - { - switch (it->id) - { - case execplan::PMSMALLSIDEMEMORY: - { - fRm->addHJPmMaxSmallSideMap(it->sessionId, it->value); - break; - } - - case execplan::UMSMALLSIDEMEMORY: - { - fRm->addHJUmMaxSmallSideMap(it->sessionId, it->value); - break; - } - - default:; - } - } - } - - void buildSysCache(const execplan::CalpontSelectExecutionPlan& csep, - boost::shared_ptr csc) - { - const execplan::CalpontSelectExecutionPlan::ColumnMap& colMap = csep.columnMap(); - execplan::CalpontSelectExecutionPlan::ColumnMap::const_iterator it; - std::string schemaName; - - for (it = colMap.begin(); it != colMap.end(); ++it) - { - const auto sc = dynamic_cast((it->second).get()); - - if (sc) - { - schemaName = sc->schemaName(); - - // only the first time a schema is got will actually query - // system catalog. System catalog keeps a schema name std::map. - // if a schema exists, the call getSchemaInfo returns without - // doing anything. - if (!schemaName.empty()) - csc->getSchemaInfo(schemaName); - } - } - - execplan::CalpontSelectExecutionPlan::SelectList::const_iterator subIt; - - for (subIt = csep.derivedTableList().begin(); subIt != csep.derivedTableList().end(); ++subIt) - { - buildSysCache(*(dynamic_cast(subIt->get())), csc); - } - } - - void writeCodeAndError(messageqcpp::ByteStream::quadbyte code, const std::string emsg) - { - messageqcpp::ByteStream emsgBs; - messageqcpp::ByteStream tbs; - tbs << code; - fIos.write(tbs); - emsgBs << emsg; - fIos.write(emsgBs); - } - - void analyzeTableExecute(messageqcpp::ByteStream& bs, joblist::SJLP& jl, bool& stmtCounted) - { - messageqcpp::ByteStream::quadbyte qb; - execplan::MCSAnalyzeTableExecutionPlan caep; - - bs = fIos.read(); - caep.unserialize(bs); - - statementsRunningCount->incr(stmtCounted); - jl = joblist::JobListFactory::makeJobList(&caep, fRm, false, true); - - // Joblist is empty. - if (jl->status() == logging::statisticsJobListEmpty) - { - if (caep.traceOn()) - std::cout << "JobList is empty " << std::endl; - - jl.reset(); - bs.restart(); - qb = ANALYZE_TABLE_SUCCESS; - bs << qb; - fIos.write(bs); - bs.reset(); - statementsRunningCount->decr(stmtCounted); - return; - } - - if (UNLIKELY(fEc->getNumConnections() != fEc->connectedPmServers())) - { - std::cout << "fEc setup " << std::endl; - fEc->Setup(); - } - - if (jl->status() == 0) - { - if (jl->putEngineComm(fEc) != 0) - throw std::runtime_error(jl->errMsg()); - } - else - { - throw std::runtime_error("ExeMgr: could not build a JobList!"); - } - - // Execute a joblist. - jl->doQuery(); - - FEMsgHandler msgHandler(jl, &fIos); - - msgHandler.start(); - auto rowCount = jl->projectTable(100, bs); - msgHandler.stop(); - - auto outRG = (static_cast(jl.get()))->getOutputRowGroup(); - - if (caep.traceOn()) - std::cout << "Row count " << rowCount << std::endl; - - // Process `RowGroup`, increase an epoch and save statistics to the file. - auto* statisticsManager = statistics::StatisticsManager::instance(); - statisticsManager->analyzeColumnKeyTypes(outRG, caep.traceOn()); - statisticsManager->incEpoch(); - statisticsManager->saveToFile(); - - // Distribute statistics across all ExeMgr clients if possible. - statistics::StatisticsDistributor::instance()->distributeStatistics(); - - // Send the signal back to front-end. - bs.restart(); - qb = ANALYZE_TABLE_SUCCESS; - bs << qb; - fIos.write(bs); - bs.reset(); - statementsRunningCount->decr(stmtCounted); - } - - void analyzeTableHandleStats(messageqcpp::ByteStream& bs) - { - messageqcpp::ByteStream::quadbyte qb; -#ifdef DEBUG_STATISTICS - std::cout << "Get distributed statistics on ExeMgr(Client) from ExeMgr(Server) " << std::endl; -#endif - bs = fIos.read(); -#ifdef DEBUG_STATISTICS - std::cout << "Read the hash from statistics on ExeMgr(Client) from ExeMgr(Server) " << std::endl; -#endif - uint64_t dataHashRec; - bs >> dataHashRec; - - uint64_t dataHash = statistics::StatisticsManager::instance()->computeHashFromStats(); - // The stats are the same. - if (dataHash == dataHashRec) - { -#ifdef DEBUG_STATISTICS - std::cout << "The hash is the same as rec hash on ExeMgr(Client) from ExeMgr(Server) " << std::endl; -#endif - qb = ANALYZE_TABLE_SUCCESS; - bs << qb; - fIos.write(bs); - bs.reset(); - return; - } - - bs.restart(); - qb = ANALYZE_TABLE_NEED_STATS; - bs << qb; - fIos.write(bs); - - bs.restart(); - bs = fIos.read(); -#ifdef DEBUG_STATISTICS - std::cout << "Read statistics on ExeMgr(Client) from ExeMgr(Server) " << std::endl; -#endif - statistics::StatisticsManager::instance()->unserialize(bs); - statistics::StatisticsManager::instance()->saveToFile(); - -#ifdef DEBUG_STATISTICS - std::cout << "Write flag on ExeMgr(Client) to ExeMgr(Server)" << std::endl; -#endif - qb = ANALYZE_TABLE_SUCCESS; - bs << qb; - fIos.write(bs); - bs.reset(); - } - - public: - void operator()() - { - utils::setThreadName("SessionThread"); - messageqcpp::ByteStream bs, inbs; - execplan::CalpontSelectExecutionPlan csep; - csep.sessionID(0); - joblist::SJLP jl; - bool incSessionThreadCnt = true; - std::mutex jlMutex; - std::condition_variable jlCleanupDone; - int destructing = 0; - - bool selfJoin = false; - bool tryTuples = false; - bool usingTuples = false; - bool stmtCounted = false; - - try - { - for (;;) - { - selfJoin = false; - tryTuples = false; - usingTuples = false; - - if (jl) - { - // puts the real destruction in another thread to avoid - // making the whole session wait. It can take several seconds. - std::unique_lock scoped(jlMutex); - destructing++; - std::thread bgdtor( - [jl, &jlMutex, &jlCleanupDone, &destructing] - { - std::unique_lock scoped(jlMutex); - const_cast(jl).reset(); // this happens second; does real destruction - if (--destructing == 0) - jlCleanupDone.notify_one(); - }); - jl.reset(); // this runs first - bgdtor.detach(); - } - - bs = fIos.read(); - - if (bs.length() == 0) - { - if (gDebug > 1 || (gDebug && !csep.isInternal())) - std::cout << "### Got a close(1) for session id " << csep.sessionID() << std::endl; - - // connection closed by client - fIos.close(); - break; - } - else if (bs.length() < 4) // Not a CalpontSelectExecutionPlan - { - if (gDebug) - std::cout << "### Got a not-a-plan for session id " << csep.sessionID() << " with length " - << bs.length() << std::endl; - - fIos.close(); - break; - } - else if (bs.length() == 4) // possible tuple flag - { - messageqcpp::ByteStream::quadbyte qb; - bs >> qb; - - if (qb == 4) // UM wants new tuple i/f - { - if (gDebug) - std::cout << "### UM wants tuples" << std::endl; - - tryTuples = true; - // now wait for the CSEP... - bs = fIos.read(); - } - else if (qb == 5) // somebody wants stats - { - bs.restart(); - qb = statementsRunningCount->cur(); - bs << qb; - qb = statementsRunningCount->waiting(); - bs << qb; - fIos.write(bs); - fIos.close(); - break; - } - else if (qb == ANALYZE_TABLE_EXECUTE) - { - analyzeTableExecute(bs, jl, stmtCounted); - continue; - } - else if (qb == ANALYZE_TABLE_REC_STATS) - { - analyzeTableHandleStats(bs); - continue; - } - else - { - if (gDebug) - std::cout << "### Got a not-a-plan value " << qb << std::endl; - - fIos.close(); - break; - } - } - - new_plan: - try - { - csep.unserialize(bs); - } - catch (logging::IDBExcept& ex) - { - // We can get here on illegal function parameter data type, e.g. - // SELECT blob_column|1 FROM t1; - statementsRunningCount->decr(stmtCounted); - writeCodeAndError(ex.errorCode(), std::string(ex.what())); - continue; - } - - querytele::QueryTeleStats qts; - - if (!csep.isInternal() && (csep.queryType() == "SELECT" || csep.queryType() == "INSERT_SELECT")) - { - qts.query_uuid = csep.uuid(); - qts.msg_type = querytele::QueryTeleStats::QT_START; - qts.start_time = querytele::QueryTeleClient::timeNowms(); - qts.query = csep.data(); - qts.session_id = csep.sessionID(); - qts.query_type = csep.queryType(); - qts.system_name = fOamCachePtr->getSystemName(); - qts.module_name = fOamCachePtr->getModuleName(); - qts.local_query = csep.localQuery(); - qts.schema_name = csep.schemaName(); - fTeleClient.postQueryTele(qts); - } - - if (gDebug > 1 || (gDebug && !csep.isInternal())) - std::cout << "### For session id " << csep.sessionID() << ", got a CSEP" << std::endl; - - setRMParms(csep.rmParms()); - // Re-establish lost PP connections. - if (UNLIKELY(fEc->getNumConnections() != fEc->connectedPmServers())) - { - fEc->Setup(); - } - // @bug 1021. try to get schema cache for a come in query. - // skip system catalog queries. - if (!csep.isInternal()) - { - boost::shared_ptr csc = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(csep.sessionID()); - buildSysCache(csep, csc); - } - - // As soon as we have a session id for this thread, update the - // thread count per session; only do this once per thread. - // Mask 0x80000000 is for associate user query and csc query - if (incSessionThreadCnt) - { - incThreadCntPerSession(csep.sessionID() | 0x80000000); - incSessionThreadCnt = false; - } - - bool needDbProfEndStatementMsg = false; - logging::Message::Args args; - std::string sqlText = csep.data(); - logging::LoggingID li(16, csep.sessionID(), csep.txnID()); - - // Initialize stats for this query, including - // init sessionMemMap entry for this session to 0 memory %. - // We will need this later for traceOn() or if we receive a - // table request with qb=3 (see below). This is also recorded - // as query start time. - initStats(csep.sessionID(), sqlText); - fStats.fQueryType = csep.queryType(); - - // Log start and end statement if tracing is enabled. Keep in - // mind the trace flag won't be set for system catalog queries. - if (csep.traceOn()) - { - args.reset(); - args.add((int)csep.statementID()); - args.add((int)csep.verID().currentScn); - args.add(sqlText); - msgLog.logMessage(logging::LOG_TYPE_DEBUG, logDbProfStartStatement, args, li); - needDbProfEndStatementMsg = true; - } - - // Don't log subsequent self joins after first. - if (selfJoin) - sqlText = ""; - - std::ostringstream oss; - oss << sqlText << "; |" << csep.schemaName() << "|"; - logging::SQLLogger sqlLog(oss.str(), li); - - statementsRunningCount->incr(stmtCounted); - - if (tryTuples) - { - try // @bug2244: try/catch around fIos.write() calls responding to makeTupleList - { - jl = joblist::JobListFactory::makeJobList(&csep, fRm, true, true); - // assign query stats - jl->queryStats(fStats); - - if ((jl->status()) == 0 && (jl->putEngineComm(fEc) == 0)) - { - usingTuples = true; - - // Tell the FE that we're sending tuples back, not TableBands - writeCodeAndError(0, "NOERROR"); - auto tjlp = dynamic_cast(jl.get()); - assert(tjlp); - messageqcpp::ByteStream tbs; - tbs << tjlp->getOutputRowGroup(); - fIos.write(tbs); - } - else - { - const std::string emsg = jl->errMsg(); - statementsRunningCount->decr(stmtCounted); - writeCodeAndError(jl->status(), emsg); - std::cerr << "ExeMgr: could not build a tuple joblist: " << emsg << std::endl; - continue; - } - } - catch (std::exception& ex) - { - std::ostringstream errMsg; - errMsg << "ExeMgr: error writing makeJoblist " - "response; " - << ex.what(); - throw std::runtime_error(errMsg.str()); - } - catch (...) - { - std::ostringstream errMsg; - errMsg << "ExeMgr: unknown error writing makeJoblist " - "response; "; - throw std::runtime_error(errMsg.str()); - } - - if (!usingTuples) - { - if (gDebug) - std::cout << "### UM wanted tuples but it didn't work out :-(" << std::endl; - } - else - { - if (gDebug) - std::cout << "### UM wanted tuples and we'll do our best;-)" << std::endl; - } - } - else - { - usingTuples = false; - jl = joblist::JobListFactory::makeJobList(&csep, fRm, false, true); - - if (jl->status() == 0) - { - std::string emsg; - - if (jl->putEngineComm(fEc) != 0) - throw std::runtime_error(jl->errMsg()); - } - else - { - throw std::runtime_error("ExeMgr: could not build a JobList!"); - } - } - - jl->doQuery(); - - execplan::CalpontSystemCatalog::OID tableOID; - bool swallowRows = false; - joblist::DeliveredTableMap tm; - uint64_t totalBytesSent = 0; - uint64_t totalRowCount = 0; - - // Project each table as the FE asks for it - for (;;) - { - bs = fIos.read(); - - if (bs.length() == 0) - { - if (gDebug > 1 || (gDebug && !csep.isInternal())) - std::cout << "### Got a close(2) for session id " << csep.sessionID() << std::endl; - - break; - } - - if (gDebug && bs.length() > 4) - std::cout << "### For session id " << csep.sessionID() << ", got too many bytes = " << bs.length() - << std::endl; - - // TODO: Holy crud! Can this be right? - //@bug 1444 Yes, if there is a self-join - if (bs.length() > 4) - { - selfJoin = true; - statementsRunningCount->decr(stmtCounted); - goto new_plan; - } - - assert(bs.length() == 4); - - messageqcpp::ByteStream::quadbyte qb; - - try // @bug2244: try/catch around fIos.write() calls responding to qb command - { - bs >> qb; - - if (gDebug > 1 || (gDebug && !csep.isInternal())) - std::cout << "### For session id " << csep.sessionID() << ", got a command = " << qb - << std::endl; - - if (qb == 0) - { - // No more tables, query is done - break; - } - else if (qb == 1) - { - // super-secret flag indicating that the UM is going to scarf down all the rows in the - // query. - swallowRows = true; - tm = jl->deliveredTables(); - continue; - } - else if (qb == 2) - { - // UM just wants any table - assert(swallowRows); - auto iter = tm.begin(); - - if (iter == tm.end()) - { - if (gDebug > 1 || (gDebug && !csep.isInternal())) - std::cout << "### For session id " << csep.sessionID() << ", returning end flag" - << std::endl; - - bs.restart(); - bs << (messageqcpp::ByteStream::byte)1; - fIos.write(bs); - continue; - } - - tableOID = iter->first; - } - else if (qb == 3) // special option-UM wants job stats std::string - { - std::string statsString; - - // Log stats std::string to be sent back to front end - statsString = formatQueryStats( - jl, "Query Stats", false, - !(csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF), - (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG), totalRowCount); - - bs.restart(); - bs << statsString; - - if ((csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG) != 0) - { - bs << jl->extendedInfo(); - bs << prettyPrintMiniInfo(jl->miniInfo()); - } - else - { - std::string empty; - bs << empty; - bs << empty; - } - - // send stats to connector for inserting to the querystats table - fStats.serialize(bs); - fIos.write(bs); - continue; - } - // for table mode handling - else if (qb == 4) - { - statementsRunningCount->decr(stmtCounted); - bs = fIos.read(); - goto new_plan; - } - else // (qb > 3) - { - // Return table bands for the requested tableOID - tableOID = static_cast(qb); - } - } - catch (std::exception& ex) - { - std::ostringstream errMsg; - errMsg << "ExeMgr: error writing qb response " - "for qb cmd " - << qb << "; " << ex.what(); - throw std::runtime_error(errMsg.str()); - } - catch (...) - { - std::ostringstream errMsg; - errMsg << "ExeMgr: unknown error writing qb response " - "for qb cmd " - << qb; - throw std::runtime_error(errMsg.str()); - } - - if (swallowRows) - tm.erase(tableOID); - - FEMsgHandler msgHandler(jl, &fIos); - - if (tableOID == 100) - msgHandler.start(); - - //...Loop serializing table bands projected for the tableOID - for (;;) - { - uint32_t rowCount; - - rowCount = jl->projectTable(tableOID, bs); - - msgHandler.stop(); - - if (jl->status()) - { - const auto errInfo = logging::IDBErrorInfo::instance(); - - if (jl->errMsg().length() != 0) - bs << jl->errMsg(); - else - bs << errInfo->errorMsg(jl->status()); - } - - try // @bug2244: try/catch around fIos.write() calls projecting rows - { - if (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS3) - { - // Skip the write to the front end until the last empty band. Used to time queries - // through without any front end waiting. - if (tableOID < 3000 || rowCount == 0) - fIos.write(bs); - } - else - { - fIos.write(bs); - } - } - catch (std::exception& ex) - { - msgHandler.stop(); - std::ostringstream errMsg; - errMsg << "ExeMgr: error projecting rows " - "for tableOID: " - << tableOID << "; rowCnt: " << rowCount << "; prevTotRowCnt: " << totalRowCount << "; " - << ex.what(); - jl->abort(); - - while (rowCount) - rowCount = jl->projectTable(tableOID, bs); - - if (tableOID == 100 && msgHandler.aborted()) - { - /* TODO: modularize the cleanup code, as well as - * the rest of this fcn */ - - decThreadCntPerSession(csep.sessionID() | 0x80000000); - statementsRunningCount->decr(stmtCounted); - fIos.close(); - return; - } - - // std::cout << "connection drop\n"; - throw std::runtime_error(errMsg.str()); - } - catch (...) - { - std::ostringstream errMsg; - msgHandler.stop(); - errMsg << "ExeMgr: unknown error projecting rows " - "for tableOID: " - << tableOID << "; rowCnt: " << rowCount << "; prevTotRowCnt: " << totalRowCount; - jl->abort(); - - while (rowCount) - rowCount = jl->projectTable(tableOID, bs); - - throw std::runtime_error(errMsg.str()); - } - - totalRowCount += rowCount; - totalBytesSent += bs.length(); - - if (rowCount == 0) - { - msgHandler.stop(); - // No more bands, table is done - bs.reset(); - - // @bug 2083 decr active statement count here for table mode. - if (!usingTuples) - statementsRunningCount->decr(stmtCounted); - - break; - } - else - { - bs.restart(); - } - } // End of loop to project and serialize table bands for a table - } // End of loop to process tables - - // @bug 828 - if (csep.traceOn()) - jl->graph(csep.sessionID()); - - if (needDbProfEndStatementMsg) - { - std::string ss; - std::ostringstream prefix; - prefix << "ses:" << csep.sessionID() << " Query Totals"; - - // Log stats std::string to standard out - ss = formatQueryStats(jl, prefix.str(), true, - !(csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF), - (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG), - totalRowCount); - //@Bug 1306. Added timing info for real time tracking. - std::cout << ss << " at " << timeNow() << std::endl; - - // log query stats to debug log file - args.reset(); - args.add((int)csep.statementID()); - args.add(fStats.fMaxMemPct); - args.add(fStats.fNumFiles); - args.add(fStats.fFileBytes); // log raw byte count instead of MB - args.add(fStats.fPhyIO); - args.add(fStats.fCacheIO); - args.add(fStats.fMsgRcvCnt); - args.add(fStats.fMsgBytesIn); - args.add(fStats.fMsgBytesOut); - args.add(fStats.fCPBlocksSkipped); - msgLog.logMessage(logging::LOG_TYPE_DEBUG, logDbProfQueryStats, args, li); - //@bug 1327 - deleteMaxMemPct(csep.sessionID()); - // Calling reset here, will cause joblist destructor to be - // called, which "joins" the threads. We need to do that - // here to make sure all syslogging from all the threads - // are complete; and that our logDbProfEndStatement will - // appear "last" in the syslog for this SQL statement. - // puts the real destruction in another thread to avoid - // making the whole session wait. It can take several seconds. - int stmtID = csep.statementID(); - std::unique_lock scoped(jlMutex); - // C7's compiler complains about the msgLog capture here - // msgLog is global scope, and passed by copy, so, unclear - // what the warning is about. - destructing++; - std::thread bgdtor( - [jl, &jlMutex, &jlCleanupDone, stmtID, &li, &destructing] - { - std::unique_lock scoped(jlMutex); - const_cast(jl).reset(); // this happens second; does real destruction - logging::Message::Args args; - args.add(stmtID); - msgLog.logMessage(logging::LOG_TYPE_DEBUG, logDbProfEndStatement, args, li); - if (--destructing == 0) - jlCleanupDone.notify_one(); - }); - jl.reset(); // this happens first - bgdtor.detach(); - } - else - // delete sessionMemMap entry for this session's memory % use - deleteMaxMemPct(csep.sessionID()); - - std::string endtime(timeNow()); - - if ((csep.traceFlags() & flagsWantOutput) && (csep.sessionID() < 0x80000000)) - { - std::cout << "For session " << csep.sessionID() << ": " << totalBytesSent << " bytes sent back at " - << endtime << std::endl; - - // @bug 663 - Implemented caltraceon(16) to replace the - // $FIFO_SINK compiler definition in pColStep. - // This option consumes rows in the project steps. - if (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS4) - { - std::cout << std::endl; - std::cout << "**** No data returned to DM. Rows consumed " - "in ProjectSteps - caltrace(16) is on (FIFO_SINK)." - " ****" - << std::endl; - std::cout << std::endl; - } - else if (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS3) - { - std::cout << std::endl; - std::cout << "**** No data returned to DM - caltrace(8) is " - "on (SWALLOW_ROWS_EXEMGR). ****" - << std::endl; - std::cout << std::endl; - } - } - - statementsRunningCount->decr(stmtCounted); - - if (!csep.isInternal() && (csep.queryType() == "SELECT" || csep.queryType() == "INSERT_SELECT")) - { - qts.msg_type = querytele::QueryTeleStats::QT_SUMMARY; - qts.max_mem_pct = fStats.fMaxMemPct; - qts.num_files = fStats.fNumFiles; - qts.phy_io = fStats.fPhyIO; - qts.cache_io = fStats.fCacheIO; - qts.msg_rcv_cnt = fStats.fMsgRcvCnt; - qts.cp_blocks_skipped = fStats.fCPBlocksSkipped; - qts.msg_bytes_in = fStats.fMsgBytesIn; - qts.msg_bytes_out = fStats.fMsgBytesOut; - qts.rows = totalRowCount; - qts.end_time = querytele::QueryTeleClient::timeNowms(); - qts.session_id = csep.sessionID(); - qts.query_type = csep.queryType(); - qts.query = csep.data(); - qts.system_name = fOamCachePtr->getSystemName(); - qts.module_name = fOamCachePtr->getModuleName(); - qts.local_query = csep.localQuery(); - fTeleClient.postQueryTele(qts); - } - } - - // Release CSC object (for sessionID) that was added by makeJobList() - // Mask 0x80000000 is for associate user query and csc query. - // (actual joblist destruction happens at the top of this loop) - decThreadCntPerSession(csep.sessionID() | 0x80000000); - } - catch (std::exception& ex) - { - decThreadCntPerSession(csep.sessionID() | 0x80000000); - statementsRunningCount->decr(stmtCounted); - std::cerr << "### ExeMgr ses:" << csep.sessionID() << " caught: " << ex.what() << std::endl; - logging::Message::Args args; - logging::LoggingID li(16, csep.sessionID(), csep.txnID()); - args.add(ex.what()); - msgLog.logMessage(logging::LOG_TYPE_CRITICAL, logExeMgrExcpt, args, li); - fIos.close(); - } - catch (...) - { - decThreadCntPerSession(csep.sessionID() | 0x80000000); - statementsRunningCount->decr(stmtCounted); - std::cerr << "### Exception caught!" << std::endl; - logging::Message::Args args; - logging::LoggingID li(16, csep.sessionID(), csep.txnID()); - args.add("ExeMgr caught unknown exception"); - msgLog.logMessage(logging::LOG_TYPE_CRITICAL, logExeMgrExcpt, args, li); - fIos.close(); - } - - // make sure we don't leave scope while joblists are being destroyed - std::unique_lock scoped(jlMutex); - while (destructing > 0) - jlCleanupDone.wait(scoped); - } -}; - -class RssMonFcn : public utils::MonitorProcMem -{ - public: - RssMonFcn(size_t maxPct, int pauseSeconds) : MonitorProcMem(maxPct, 0, 21, pauseSeconds) - { - } - - /*virtual*/ - void operator()() const - { - for (;;) - { - size_t rssMb = rss(); - size_t pct = rssMb * 100 / fMemTotal; - - if (pct > fMaxPct) - { - if (fMaxPct >= 95) - { - std::cerr << "Too much memory allocated!" << std::endl; - logging::Message::Args args; - args.add((int)pct); - args.add((int)fMaxPct); - msgLog.logMessage(logging::LOG_TYPE_CRITICAL, logRssTooBig, args, logging::LoggingID(16)); - exit(1); - } - - if (statementsRunningCount->cur() == 0) - { - std::cerr << "Too much memory allocated!" << std::endl; - logging::Message::Args args; - args.add((int)pct); - args.add((int)fMaxPct); - msgLog.logMessage(logging::LOG_TYPE_WARNING, logRssTooBig, args, logging::LoggingID(16)); - exit(1); - } - - std::cerr << "Too much memory allocated, but stmts running" << std::endl; - } - - // Update sessionMemMap entries lower than current mem % use - { - std::lock_guard lk(sessionMemMapMutex); - - for (SessionMemMap_t::iterator mapIter = sessionMemMap.begin(); mapIter != sessionMemMap.end(); - ++mapIter) - { - if (pct > mapIter->second) - { - mapIter->second = pct; - } - } - } - - pause_(); - } - } -}; - -#ifdef _MSC_VER -void exit_(int) -{ - exit(0); -} -#endif - -void added_a_pm(int) -{ - logging::LoggingID logid(21, 0, 0); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("exeMgr caught SIGHUP. Resetting connections"); - msg.format(args1); - std::cout << msg.msg().c_str() << std::endl; - logging::Logger logger(logid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_DEBUG, msg, logid); - - if (ec) - { - oam::OamCache* oamCache = oam::OamCache::makeOamCache(); - oamCache->forceReload(); - ec->Setup(); - } -} - -void printTotalUmMemory(int sig) -{ - int64_t num = rm->availableMemory(); - std::cout << "Total UM memory available: " << num << std::endl; -} - -void setupSignalHandlers() -{ -#ifdef _MSC_VER - signal(SIGINT, exit_); - signal(SIGTERM, exit_); -#else - struct sigaction ign; - - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = SIG_IGN; - - sigaction(SIGPIPE, &ign, 0); - - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = added_a_pm; - sigaction(SIGHUP, &ign, 0); - ign.sa_handler = printTotalUmMemory; - sigaction(SIGUSR1, &ign, 0); - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = fatalHandler; - sigaction(SIGSEGV, &ign, 0); - sigaction(SIGABRT, &ign, 0); - sigaction(SIGFPE, &ign, 0); -#endif -} - -int8_t setupCwd(joblist::ResourceManager* rm) -{ - std::string workdir = rm->getScWorkingDir(); - int8_t rc = chdir(workdir.c_str()); - - if (rc < 0 || access(".", W_OK) != 0) - rc = chdir("/tmp"); - - return (rc < 0) ? -5 : rc; -} - -void startRssMon(size_t maxPct, int pauseSeconds) -{ - new boost::thread(RssMonFcn(maxPct, pauseSeconds)); -} - -int setupResources() -{ -#ifdef _MSC_VER - // FIXME: -#else - struct rlimit rlim; - - if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) - { - return -1; - } - - rlim.rlim_cur = rlim.rlim_max = 65536; - - if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) - { - return -2; - } - - if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) - { - return -3; - } - - if (rlim.rlim_cur != 65536) - { - return -4; - } - -#endif - return 0; -} - -} // namespace - -void cleanTempDir() -{ - using TempDirPurpose = config::Config::TempDirPurpose; - struct Dirs - { - std::string section; - std::string allowed; - TempDirPurpose purpose; - }; - std::vector dirs{{"HashJoin", "AllowDiskBasedJoin", TempDirPurpose::Joins}, - {"RowAggregation", "AllowDiskBasedAggregation", TempDirPurpose::Aggregates}}; - const auto config = config::Config::makeConfig(); - - for (const auto& dir : dirs) - { - std::string allowStr = config->getConfig(dir.section, dir.allowed); - bool allow = (allowStr == "Y" || allowStr == "y"); - - std::string tmpPrefix = config->getTempFileDir(dir.purpose); - - if (allow && tmpPrefix.empty()) - { - std::cerr << "Empty tmp directory name for " << dir.section << std::endl; - logging::LoggingID logid(16, 0, 0); - logging::Message::Args args; - logging::Message message(8); - args.add("Empty tmp directory name for:"); - args.add(dir.section); - message.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_CRITICAL, message, logid); - } - - tmpPrefix += "/"; - - idbassert(tmpPrefix != "/"); - - /* This is quite scary as ExeMgr usually runs as root */ - try - { - if (allow) - { - boost::filesystem::remove_all(tmpPrefix); - } - boost::filesystem::create_directories(tmpPrefix); - } - catch (const std::exception& ex) - { - std::cerr << ex.what() << std::endl; - logging::LoggingID logid(16, 0, 0); - logging::Message::Args args; - logging::Message message(8); - args.add("Exception whilst cleaning tmpdir: "); - args.add(ex.what()); - message.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_WARNING, message, logid); - } - catch (...) - { - std::cerr << "Caught unknown exception during tmpdir cleanup" << std::endl; - logging::LoggingID logid(16, 0, 0); - logging::Message::Args args; - logging::Message message(8); - args.add("Unknown exception whilst cleaning tmpdir"); - message.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_WARNING, message, logid); - } - } -} - -int ServiceExeMgr::Child() -{ - gDebug = m_debug; - -#ifdef _MSC_VER - // FIXME: -#else - - // Make sure CSC thinks it's on a UM or else bucket reuse stuff below will stall - if (!m_e) - setenv("CALPONT_CSC_IDENT", "um", 1); - -#endif - setupSignalHandlers(); - int err = 0; - if (!m_debug) - err = setupResources(); - std::string errMsg; - - switch (err) - { - case -1: - case -3: errMsg = "Error getting file limits, please see non-root install documentation"; break; - - case -2: errMsg = "Error setting file limits, please see non-root install documentation"; break; - - case -4: - errMsg = "Could not install file limits to required value, please see non-root install documentation"; - break; - - default: errMsg = "Couldn't change working directory or unknown error"; break; - } - - err = setupCwd(rm); - - if (err < 0) - { - oam::Oam oam; - logging::Message::Args args; - logging::Message message; - args.add(errMsg); - message.format(args); - logging::LoggingID lid(16); - logging::MessageLog ml(lid); - ml.logCriticalMessage(message); - std::cerr << errMsg << std::endl; - - NotifyServiceInitializationFailed(); - return 2; - } - - cleanTempDir(); - - logging::MsgMap msgMap; - msgMap[logDefaultMsg] = logging::Message(logDefaultMsg); - msgMap[logDbProfStartStatement] = logging::Message(logDbProfStartStatement); - msgMap[logDbProfEndStatement] = logging::Message(logDbProfEndStatement); - msgMap[logStartSql] = logging::Message(logStartSql); - msgMap[logEndSql] = logging::Message(logEndSql); - msgMap[logRssTooBig] = logging::Message(logRssTooBig); - msgMap[logDbProfQueryStats] = logging::Message(logDbProfQueryStats); - msgMap[logExeMgrExcpt] = logging::Message(logExeMgrExcpt); - msgLog.msgMap(msgMap); - - ec = joblist::DistributedEngineComm::instance(rm, true); - ec->Open(); - - bool tellUser = true; - - messageqcpp::MessageQueueServer* mqs; - - statementsRunningCount = new ActiveStatementCounter(rm->getEmExecQueueSize()); - - for (;;) - { - try - { - mqs = new messageqcpp::MessageQueueServer(ExeMgr, rm->getConfig(), messageqcpp::ByteStream::BlockSize, - 64); - break; - } - catch (std::runtime_error& re) - { - std::string what = re.what(); - - if (what.find("Address already in use") != std::string::npos) - { - if (tellUser) - { - std::cerr << "Address already in use, retrying..." << std::endl; - tellUser = false; - } - - sleep(5); - } - else - { - throw; - } - } - } - - // class jobstepThreadPool is used by other processes. We can't call - // resourcemanaager (rm) functions during the static creation of threadpool - // because rm has a "isExeMgr" flag that is set upon creation (rm is a singleton). - // From the pools perspective, it has no idea if it is ExeMgr doing the - // creation, so it has no idea which way to set the flag. So we set the max here. - joblist::JobStep::jobstepThreadPool.setMaxThreads(rm->getJLThreadPoolSize()); - joblist::JobStep::jobstepThreadPool.setName("ExeMgrJobList"); - - if (rm->getJlThreadPoolDebug() == "Y" || rm->getJlThreadPoolDebug() == "y") - { - joblist::JobStep::jobstepThreadPool.setDebug(true); - joblist::JobStep::jobstepThreadPool.invoke( - threadpool::ThreadPoolMonitor(&joblist::JobStep::jobstepThreadPool)); - } - - int serverThreads = rm->getEmServerThreads(); - int maxPct = rm->getEmMaxPct(); - int pauseSeconds = rm->getEmSecondsBetweenMemChecks(); - int priority = rm->getEmPriority(); - - FEMsgHandler::threadPool.setMaxThreads(serverThreads); - FEMsgHandler::threadPool.setName("FEMsgHandler"); - - if (maxPct > 0) - startRssMon(maxPct, pauseSeconds); - -#ifndef _MSC_VER - setpriority(PRIO_PROCESS, 0, priority); -#endif - - std::string teleServerHost(rm->getConfig()->getConfig("QueryTele", "Host")); - - if (!teleServerHost.empty()) - { - int teleServerPort = toInt(rm->getConfig()->getConfig("QueryTele", "Port")); - - if (teleServerPort > 0) - { - gTeleServerParms.host = teleServerHost; - gTeleServerParms.port = teleServerPort; - } - } - - NotifyServiceStarted(); - - std::cout << "Starting ExeMgr: st = " << serverThreads << ", qs = " << rm->getEmExecQueueSize() - << ", mx = " << maxPct << ", cf = " << rm->getConfig()->configFile() << std::endl; - - { - BRM::DBRM* dbrm = new BRM::DBRM(); - dbrm->setSystemQueryReady(true); - delete dbrm; - } - - threadpool::ThreadPool exeMgrThreadPool(serverThreads, 0); - exeMgrThreadPool.setName("ExeMgrServer"); - - if (rm->getExeMgrThreadPoolDebug() == "Y" || rm->getExeMgrThreadPoolDebug() == "y") - { - exeMgrThreadPool.setDebug(true); - exeMgrThreadPool.invoke(threadpool::ThreadPoolMonitor(&exeMgrThreadPool)); - } - - // Load statistics. - try - { - statistics::StatisticsManager::instance()->loadFromFile(); - } - catch (...) - { - std::cerr << "Cannot load statistics from file " << std::endl; - } - - for (;;) - { - messageqcpp::IOSocket ios; - ios = mqs->accept(); - exeMgrThreadPool.invoke(SessionThread(ios, ec, rm)); - } - - exeMgrThreadPool.wait(); - - return 0; -} - -int main(int argc, char* argv[]) -{ - opterr = 0; - Opt opt(argc, argv); - - // Set locale language - setlocale(LC_ALL, ""); - setlocale(LC_NUMERIC, "C"); - - // This is unset due to the way we start it - program_invocation_short_name = const_cast("ExeMgr"); - - // Initialize the charset library - MY_INIT(argv[0]); - - return ServiceExeMgr(opt).Run(); -} - -// vim:ts=4 sw=4: diff --git a/exemgr/resource.h b/exemgr/resource.h deleted file mode 100644 index 3e2717d59..000000000 --- a/exemgr/resource.h +++ /dev/null @@ -1,14 +0,0 @@ -//{{NO_DEPENDENCIES}} -// Microsoft Visual C++ generated include file. -// Used by ExeMgr.rc - -// Next default values for new objects -// -#ifdef APSTUDIO_INVOKED -#ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NEXT_RESOURCE_VALUE 101 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 -#endif -#endif diff --git a/mysql-test/columnstore/basic/r/cal_named_udfs.result b/mysql-test/columnstore/basic/r/cal_named_udfs.result new file mode 100644 index 000000000..26b20fb8b --- /dev/null +++ b/mysql-test/columnstore/basic/r/cal_named_udfs.result @@ -0,0 +1,33 @@ +DROP DATABASE IF EXISTS cal_test_db; +CREATE DATABASE cal_test_db; +USE cal_test_db; +create table t1(a int, b int comment 'autoincrement=1') engine=columnstore; +select calflushcache(); +calflushcache() +0 +select calsettrace(0); +calsettrace(0) +0 +select calsetparms("pmmaxmemorysmallside","2048000000"); +calsetparms("pmmaxmemorysmallside","2048000000") +Updated pmmaxmemorysmallside 2048000000 +select calgettrace(); +calgettrace() +NULL +select calgetversion()=mcsgetversion(); +calgetversion()=mcsgetversion() +1 +select calviewtablelock("t1"); +calviewtablelock("t1") + Table cal_test_db.t1 is not locked by any process. +select calcleartablelock(0); +calcleartablelock(0) +No table lock found for specified table lock ID +select callastinsertid("t1"); +callastinsertid("t1") +0 +select calgetsqlcount(); +calgetsqlcount() +Running SQL statements 0, Waiting SQL statments 0 +DROP TABLE t1; +DROP DATABASE cal_test_db; diff --git a/mysql-test/columnstore/basic/r/ctype_extent_koi8u.result b/mysql-test/columnstore/basic/r/ctype_extent_koi8u.result index 231c29855..eeb715c77 100644 --- a/mysql-test/columnstore/basic/r/ctype_extent_koi8u.result +++ b/mysql-test/columnstore/basic/r/ctype_extent_koi8u.result @@ -87,7 +87,7 @@ CREATE TABLE t1 (c1 CHAR(2) CHARACTER SET koi8u COLLATE koi8u_general_ci) c1 HEX(c1) Ъ─ FF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_koi8u t1 c1 FFFFFFFFFFFF80FF FFFFFFFFFFFF80FF +mcs_ctype_extent_koi8u t1 c1 80FF 80FF c1 Ъ─ Level Code Message @@ -100,7 +100,7 @@ CREATE TABLE t1 (c1 CHAR(4) CHARACTER SET koi8u COLLATE koi8u_general_ci) c1 HEX(c1) ЪЪЪ─ FFFFFF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_koi8u t1 c1 FFFFFFFF80FFFFFF FFFFFFFF80FFFFFF +mcs_ctype_extent_koi8u t1 c1 80FFFFFF 80FFFFFF c1 ЪЪЪ─ Level Code Message @@ -127,7 +127,7 @@ CREATE TABLE t1 (c1 CHAR(2) CHARACTER SET koi8u COLLATE koi8u_general_nopad_ci) c1 HEX(c1) Ъ─ FF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_koi8u t1 c1 FFFFFFFFFFFF80FF FFFFFFFFFFFF80FF +mcs_ctype_extent_koi8u t1 c1 80FF 80FF c1 Ъ─ Level Code Message @@ -140,7 +140,7 @@ CREATE TABLE t1 (c1 CHAR(4) CHARACTER SET koi8u COLLATE koi8u_general_nopad_ci) c1 HEX(c1) ЪЪЪ─ FFFFFF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_koi8u t1 c1 FFFFFFFF80FFFFFF FFFFFFFF80FFFFFF +mcs_ctype_extent_koi8u t1 c1 80FFFFFF 80FFFFFF c1 ЪЪЪ─ Level Code Message @@ -167,7 +167,7 @@ CREATE TABLE t1 (c1 CHAR(2) CHARACTER SET koi8u COLLATE koi8u_bin) c1 HEX(c1) Ъ─ FF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_koi8u t1 c1 FFFFFFFFFFFF80FF FFFFFFFFFFFF80FF +mcs_ctype_extent_koi8u t1 c1 80FF 80FF c1 Ъ─ Level Code Message @@ -180,7 +180,7 @@ CREATE TABLE t1 (c1 CHAR(4) CHARACTER SET koi8u COLLATE koi8u_bin) c1 HEX(c1) ЪЪЪ─ FFFFFF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_koi8u t1 c1 FFFFFFFF80FFFFFF FFFFFFFF80FFFFFF +mcs_ctype_extent_koi8u t1 c1 80FFFFFF 80FFFFFF c1 ЪЪЪ─ Level Code Message @@ -207,7 +207,7 @@ CREATE TABLE t1 (c1 CHAR(2) CHARACTER SET koi8u COLLATE koi8u_nopad_bin) c1 HEX(c1) Ъ─ FF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_koi8u t1 c1 FFFFFFFFFFFF80FF FFFFFFFFFFFF80FF +mcs_ctype_extent_koi8u t1 c1 80FF 80FF c1 Ъ─ Level Code Message @@ -220,7 +220,7 @@ CREATE TABLE t1 (c1 CHAR(4) CHARACTER SET koi8u COLLATE koi8u_nopad_bin) c1 HEX(c1) ЪЪЪ─ FFFFFF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_koi8u t1 c1 FFFFFFFF80FFFFFF FFFFFFFF80FFFFFF +mcs_ctype_extent_koi8u t1 c1 80FFFFFF 80FFFFFF c1 ЪЪЪ─ Level Code Message diff --git a/mysql-test/columnstore/basic/r/ctype_extent_latin1.result b/mysql-test/columnstore/basic/r/ctype_extent_latin1.result index fb338da26..70cbb89da 100644 --- a/mysql-test/columnstore/basic/r/ctype_extent_latin1.result +++ b/mysql-test/columnstore/basic/r/ctype_extent_latin1.result @@ -99,7 +99,7 @@ CREATE TABLE t1 (c1 CHAR(1) CHARACTER SET latin1 COLLATE latin1_swedish_ci) c1 HEX(c1) é E9 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFFFFFFFFE9 FFFFFFFFFFFFFFE9 +mcs_ctype_extent_latin1 t1 c1 E9 E9 c1 é Level Code Message @@ -112,7 +112,7 @@ CREATE TABLE t1 (c1 CHAR(2) CHARACTER SET latin1 COLLATE latin1_swedish_ci) c1 HEX(c1) ÿ€ FF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFFFFFF80FF FFFFFFFFFFFF80FF +mcs_ctype_extent_latin1 t1 c1 80FF 80FF c1 ÿ€ Level Code Message @@ -125,7 +125,7 @@ CREATE TABLE t1 (c1 CHAR(4) CHARACTER SET latin1 COLLATE latin1_swedish_ci) c1 HEX(c1) ÿÿÿ€ FFFFFF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFF80FFFFFF FFFFFFFF80FFFFFF +mcs_ctype_extent_latin1 t1 c1 80FFFFFF 80FFFFFF c1 ÿÿÿ€ Level Code Message @@ -152,7 +152,7 @@ CREATE TABLE t1 (c1 CHAR(1) CHARACTER SET latin1 COLLATE latin1_swedish_nopad_ci c1 HEX(c1) é E9 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFFFFFFFFE9 FFFFFFFFFFFFFFE9 +mcs_ctype_extent_latin1 t1 c1 E9 E9 c1 é Level Code Message @@ -165,7 +165,7 @@ CREATE TABLE t1 (c1 CHAR(2) CHARACTER SET latin1 COLLATE latin1_swedish_nopad_ci c1 HEX(c1) ÿ€ FF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFFFFFF80FF FFFFFFFFFFFF80FF +mcs_ctype_extent_latin1 t1 c1 80FF 80FF c1 ÿ€ Level Code Message @@ -178,7 +178,7 @@ CREATE TABLE t1 (c1 CHAR(4) CHARACTER SET latin1 COLLATE latin1_swedish_nopad_ci c1 HEX(c1) ÿÿÿ€ FFFFFF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFF80FFFFFF FFFFFFFF80FFFFFF +mcs_ctype_extent_latin1 t1 c1 80FFFFFF 80FFFFFF c1 ÿÿÿ€ Level Code Message @@ -205,7 +205,7 @@ CREATE TABLE t1 (c1 CHAR(1) CHARACTER SET latin1 COLLATE latin1_bin) c1 HEX(c1) é E9 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFFFFFFFFE9 FFFFFFFFFFFFFFE9 +mcs_ctype_extent_latin1 t1 c1 E9 E9 c1 é Level Code Message @@ -218,7 +218,7 @@ CREATE TABLE t1 (c1 CHAR(2) CHARACTER SET latin1 COLLATE latin1_bin) c1 HEX(c1) ÿ€ FF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFFFFFF80FF FFFFFFFFFFFF80FF +mcs_ctype_extent_latin1 t1 c1 80FF 80FF c1 ÿ€ Level Code Message @@ -231,7 +231,7 @@ CREATE TABLE t1 (c1 CHAR(4) CHARACTER SET latin1 COLLATE latin1_bin) c1 HEX(c1) ÿÿÿ€ FFFFFF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFF80FFFFFF FFFFFFFF80FFFFFF +mcs_ctype_extent_latin1 t1 c1 80FFFFFF 80FFFFFF c1 ÿÿÿ€ Level Code Message @@ -258,7 +258,7 @@ CREATE TABLE t1 (c1 CHAR(1) CHARACTER SET latin1 COLLATE latin1_nopad_bin) c1 HEX(c1) é E9 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFFFFFFFFE9 FFFFFFFFFFFFFFE9 +mcs_ctype_extent_latin1 t1 c1 E9 E9 c1 é Level Code Message @@ -271,7 +271,7 @@ CREATE TABLE t1 (c1 CHAR(2) CHARACTER SET latin1 COLLATE latin1_nopad_bin) c1 HEX(c1) ÿ€ FF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFFFFFF80FF FFFFFFFFFFFF80FF +mcs_ctype_extent_latin1 t1 c1 80FF 80FF c1 ÿ€ Level Code Message @@ -284,7 +284,7 @@ CREATE TABLE t1 (c1 CHAR(4) CHARACTER SET latin1 COLLATE latin1_nopad_bin) c1 HEX(c1) ÿÿÿ€ FFFFFF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFF80FFFFFF FFFFFFFF80FFFFFF +mcs_ctype_extent_latin1 t1 c1 80FFFFFF 80FFFFFF c1 ÿÿÿ€ Level Code Message diff --git a/mysql-test/columnstore/basic/r/mcs_named_udfs.result b/mysql-test/columnstore/basic/r/mcs_named_udfs.result new file mode 100644 index 000000000..4a09eb41d --- /dev/null +++ b/mysql-test/columnstore/basic/r/mcs_named_udfs.result @@ -0,0 +1,33 @@ +DROP DATABASE IF EXISTS cal_test_db; +CREATE DATABASE cal_test_db; +USE cal_test_db; +create table t1(a int, b int comment 'autoincrement=1') engine=columnstore; +select mcsflushcache(); +mcsflushcache() +0 +select mcssettrace(0); +mcssettrace(0) +0 +select mcssetparms("pmmaxmemorysmallside","2048000000"); +mcssetparms("pmmaxmemorysmallside","2048000000") +Updated pmmaxmemorysmallside 2048000000 +select mcsgettrace(); +mcsgettrace() +NULL +select mcsgetversion()=calgetversion(); +mcsgetversion()=calgetversion() +1 +select mcsviewtablelock("t1"); +mcsviewtablelock("t1") + Table cal_test_db.t1 is not locked by any process. +select mcscleartablelock(0); +mcscleartablelock(0) +No table lock found for specified table lock ID +select mcslastinsertid("t1"); +mcslastinsertid("t1") +0 +select mcsgetsqlcount(); +mcsgetsqlcount() +Running SQL statements 0, Waiting SQL statments 0 +DROP TABLE t1; +DROP DATABASE cal_test_db; diff --git a/mysql-test/columnstore/basic/r/type_timestamp.result b/mysql-test/columnstore/basic/r/type_timestamp.result new file mode 100644 index 000000000..e56cb083f --- /dev/null +++ b/mysql-test/columnstore/basic/r/type_timestamp.result @@ -0,0 +1,141 @@ +# +# Test cases for the TIMESTAMP datatype +# +DROP DATABASE IF EXISTS timestamp_test; +CREATE DATABASE timestamp_test; +USE timestamp_test; +CREATE TABLE ctimestamp (a timestamp); +SET time_zone='-5:00'; +INSERT INTO ctimestamp VALUES ('2019-01-01 01:02:03'), ('2019-05-05 01:01:01'); +SET time_zone='+1:00'; +SELECT a FROM ctimestamp ORDER BY a; +a +2019-01-01 07:02:03 +2019-05-05 07:01:01 +SET time_zone='-2:00'; +SELECT a FROM ctimestamp ORDER BY a; +a +2019-01-01 04:02:03 +2019-05-05 04:01:01 +CREATE TABLE ctimestamp2 (a timestamp DEFAULT 0); +INSERT INTO ctimestamp2 SELECT * FROM ctimestamp; +SELECT a FROM ctimestamp2 ORDER BY a; +a +2019-01-01 04:02:03 +2019-05-05 04:01:01 +CREATE TABLE ctimestamp3 (a timestamp); +INSERT INTO ctimestamp3 VALUES (19940101), (940101), +(19940101010203), (940101010203), ('1994-01-01T01:02:03'); +SELECT a FROM ctimestamp3 ORDER BY a; +a +1994-01-01 00:00:00 +1994-01-01 00:00:00 +1994-01-01 01:02:03 +1994-01-01 01:02:03 +1994-01-01 01:02:03 +CREATE TABLE ctimestamp4 (a timestamp(6) default 0); +INSERT INTO ctimestamp4 VALUES (0), ('2019-01-01 01:01:01.123456'); +SELECT a, microsecond(a) FROM ctimestamp4 ORDER BY a; +a microsecond(a) +0000-00-00 00:00:00.000000 0 +2019-01-01 01:01:01.123456 123456 +DROP DATABASE IF EXISTS timestamp_test; +CREATE DATABASE timestamp_test; +USE timestamp_test; +CREATE TABLE ctimestamp (a timestamp); +SET time_zone='+0:00'; +INSERT INTO ctimestamp VALUES ('2019-01-02 00:02:03'), +('2019-01-02 01:02:03'), ('2019-01-02 10:11:12'); +SET time_zone='+1:00'; +SELECT a, a BETWEEN '2019-01-02 02:00:00' AND '2019-01-02 13:00:00' +FROM ctimestamp ORDER BY a; +a a BETWEEN '2019-01-02 02:00:00' AND '2019-01-02 13:00:00' +2019-01-02 01:02:03 0 +2019-01-02 02:02:03 1 +2019-01-02 11:11:12 1 +SELECT a, IF(a < '2019-01-02 02:00:00', 'yes', 'no'), +ADDTIME(a, '1:1:1'), STR_TO_DATE(a, '%Y-%m-%d %H:%i:%s'), +EXTRACT(DAY_HOUR FROM a), EXTRACT(MINUTE_SECOND FROM a), +TIME_FORMAT(a, '%H:\%i:\%s'), a RLIKE '02:03', IFNULL(NULL, a), +CASE a WHEN '2019-01-02 01:02:03' THEN 'found' WHEN '2019-01-02 11:11:12' +THEN 'found2' ELSE 'notfound' END, CHAR_LENGTH(a), +CAST(a AS UNSIGNED INT), CAST(a AS CHAR), CAST(a AS DATE), +TIME(CAST(a AS DATETIME)), TIME(COALESCE(NULL, a)), HEX(a), +NULLIF(a, '2019-01-02 01:02:03'), TIMEDIFF(a, '2019-01-01 01:02:03') +FROM ctimestamp ORDER BY a; +a IF(a < '2019-01-02 02:00:00', 'yes', 'no') ADDTIME(a, '1:1:1') STR_TO_DATE(a, '%Y-%m-%d %H:%i:%s') EXTRACT(DAY_HOUR FROM a) EXTRACT(MINUTE_SECOND FROM a) TIME_FORMAT(a, '%H:\%i:\%s') a RLIKE '02:03' IFNULL(NULL, a) CASE a WHEN '2019-01-02 01:02:03' THEN 'found' WHEN '2019-01-02 11:11:12' +THEN 'found2' ELSE 'notfound' END CHAR_LENGTH(a) CAST(a AS UNSIGNED INT) CAST(a AS CHAR) CAST(a AS DATE) TIME(CAST(a AS DATETIME)) TIME(COALESCE(NULL, a)) HEX(a) NULLIF(a, '2019-01-02 01:02:03') TIMEDIFF(a, '2019-01-01 01:02:03') +2019-01-02 01:02:03 yes 2019-01-02 02:03:04 2019-01-02 01:02:03 201 203 01:\02:\03 1 2019-01-02 01:02:03 found 19 20190102010203 2019-01-02 01:02:03 2019-01-02 01:02:03 01:02:03 323031392D30312D30322030313A30323A3033 NULL 24:00:00 +2019-01-02 02:02:03 no 2019-01-02 03:03:04 2019-01-02 02:02:03 202 203 02:\02:\03 1 2019-01-02 02:02:03 notfound 19 20190102020203 2019-01-02 02:02:03 2019-01-02 02:02:03 02:02:03 323031392D30312D30322030323A30323A3033 2019-01-02 02:02:03 25:00:00 +2019-01-02 11:11:12 no 2019-01-02 12:12:13 2019-01-02 11:11:12 211 1112 11:\11:\12 0 2019-01-02 11:11:12 found2 19 20190102111112 2019-01-02 11:11:12 2019-01-02 11:11:12 11:11:12 323031392D30312D30322031313A31313A3132 2019-01-02 11:11:12 34:09:09 +INSERT INTO ctimestamp VALUES ('2020-01-03 12:12:12'), +('2020-05-06 12:12:12'), ('2020-10-28 12:12:12'); +SELECT a, DAYNAME(a), DAYOFWEEK(a), DATE_FORMAT(a, '%W %M %Y'), +MONTHNAME(a), DATE(a), YEARWEEK(a), DAYOFYEAR(a), YEAR(a), +a + INTERVAL 1 DAY, TIMESTAMPDIFF(DAY, a, '2020-01-01'), +LAST_DAY(a), TRUNCATE(a, -2), a IN ('2019-01-02 01:02:03', a), +TO_DAYS(a), DAY(a), WEEK(a), WEEKDAY(a), GREATEST(a, '2020-07-01'), +MONTH(a), QUARTER(a), DATE_ADD(a, INTERVAL 1 SECOND) +FROM ctimestamp WHERE a > '2020-01-01' ORDER BY a; +a DAYNAME(a) DAYOFWEEK(a) DATE_FORMAT(a, '%W %M %Y') MONTHNAME(a) DATE(a) YEARWEEK(a) DAYOFYEAR(a) YEAR(a) a + INTERVAL 1 DAY TIMESTAMPDIFF(DAY, a, '2020-01-01') LAST_DAY(a) TRUNCATE(a, -2) a IN ('2019-01-02 01:02:03', a) TO_DAYS(a) DAY(a) WEEK(a) WEEKDAY(a) GREATEST(a, '2020-07-01') MONTH(a) QUARTER(a) DATE_ADD(a, INTERVAL 1 SECOND) +2020-01-03 12:12:12 Friday 6 Friday January 2020 January 2020-01-03 201952 3 2020 2020-01-04 12:12:12 -2 2020-01-31 2020-01-03 12:12:12 1 737792 3 0 4 2020-07-01 00:00:00 1 1 2020-01-03 12:12:13 +2020-05-06 12:12:12 Wednesday 4 Wednesday May 2020 May 2020-05-06 202018 127 2020 2020-05-07 12:12:12 -126 2020-05-31 2020-05-06 12:12:12 1 737916 6 18 2 2020-07-01 00:00:00 5 2 2020-05-06 12:12:13 +2020-10-28 12:12:12 Wednesday 4 Wednesday October 2020 October 2020-10-28 202043 302 2020 2020-10-29 12:12:12 -301 2020-10-31 2020-10-28 12:12:12 1 738091 28 43 2 2020-10-28 12:12:12 10 4 2020-10-28 12:12:13 +SELECT UNIX_TIMESTAMP(a), TIME_TO_SEC(a), CEIL(a), +CAST(LEAST(a, '2019-03-03 00:00:00') AS DATETIME), +ROUND(a), SECOND(a), MINUTE(a), HOUR(a), FLOOR(a) +FROM ctimestamp ORDER BY a; +UNIX_TIMESTAMP(a) TIME_TO_SEC(a) CEIL(a) CAST(LEAST(a, '2019-03-03 00:00:00') AS DATETIME) ROUND(a) SECOND(a) MINUTE(a) HOUR(a) FLOOR(a) +1546387323 3723 2019-01-02 01:02:03 2019-01-02 01:02:03 2019-01-02 01:02:03 3 2 1 2019-01-02 01:02:03 +1546390923 7323 2019-01-02 02:02:03 2019-01-02 02:02:03 2019-01-02 02:02:03 3 2 2 2019-01-02 02:02:03 +1546423872 40272 2019-01-02 11:11:12 2019-01-02 11:11:12 2019-01-02 11:11:12 12 11 11 2019-01-02 11:11:12 +1578049932 43932 2020-01-03 12:12:12 2019-03-03 00:00:00 2020-01-03 12:12:12 12 12 12 2020-01-03 12:12:12 +1588763532 43932 2020-05-06 12:12:12 2019-03-03 00:00:00 2020-05-06 12:12:12 12 12 12 2020-05-06 12:12:12 +1603883532 43932 2020-10-28 12:12:12 2019-03-03 00:00:00 2020-10-28 12:12:12 12 12 12 2020-10-28 12:12:12 +DROP DATABASE IF EXISTS timestamp_test; +CREATE DATABASE timestamp_test; +USE timestamp_test; +CREATE TABLE ctimestamp (a timestamp, b int); +SET time_zone='+0:00'; +INSERT INTO ctimestamp VALUES ('2019-01-03 12:12:12', 1), +('2019-01-04 12:12:12', 2), ('2019-01-03 12:12:12', 4), +('2019-01-03 12:12:12', 2), ('2019-01-04 12:12:12', 1); +SELECT a, b, SUM(b) over (PARTITION BY a ORDER BY a, b +ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) c +FROM ctimestamp; +a b c +2019-01-03 12:12:12 1 1 +2019-01-03 12:12:12 2 3 +2019-01-03 12:12:12 4 7 +2019-01-04 12:12:12 1 1 +2019-01-04 12:12:12 2 3 +SELECT a, b, MAX(a) over (PARTITION BY b ORDER BY a desc +ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) c +FROM ctimestamp; +a b c +2019-01-04 12:12:12 1 2019-01-04 12:12:12 +2019-01-03 12:12:12 1 2019-01-04 12:12:12 +2019-01-04 12:12:12 2 2019-01-04 12:12:12 +2019-01-03 12:12:12 2 2019-01-04 12:12:12 +2019-01-03 12:12:12 4 2019-01-03 12:12:12 +DROP DATABASE IF EXISTS timestamp_test; +CREATE DATABASE timestamp_test; +USE timestamp_test; +CREATE TABLE ctimestamp (a int, b timestamp); +INSERT INTO ctimestamp VALUES (1, 20190101), (1, 20200202), +(2, 20190202), (2, 20200202), (2, 20190101); +SELECT b, count(*) FROM ctimestamp GROUP BY b ORDER BY b; +b count(*) +2019-01-01 00:00:00 2 +2019-02-02 00:00:00 1 +2020-02-02 00:00:00 2 +SELECT b, max(a), min(a) FROM ctimestamp GROUP BY b ORDER BY b; +b max(a) min(a) +2019-01-01 00:00:00 2 1 +2019-02-02 00:00:00 2 2 +2020-02-02 00:00:00 2 1 +SELECT a, max(b), min(b) FROM ctimestamp GROUP BY a ORDER BY a; +a max(b) min(b) +1 2020-02-02 00:00:00 2019-01-01 00:00:00 +2 2020-02-02 00:00:00 2019-01-01 00:00:00 +DROP DATABASE timestamp_test; diff --git a/mysql-test/columnstore/basic/t/cal_named_udfs.test b/mysql-test/columnstore/basic/t/cal_named_udfs.test new file mode 100644 index 000000000..f127716d2 --- /dev/null +++ b/mysql-test/columnstore/basic/t/cal_named_udfs.test @@ -0,0 +1,25 @@ +-- source ../include/have_columnstore.inc + +--disable_warnings +DROP DATABASE IF EXISTS cal_test_db; +--enable_warnings + +CREATE DATABASE cal_test_db; +USE cal_test_db; + +create table t1(a int, b int comment 'autoincrement=1') engine=columnstore; + +select calflushcache(); +select calsettrace(0); +select calsetparms("pmmaxmemorysmallside","2048000000"); +select calgettrace(); +select calgetversion()=mcsgetversion(); + + +select calviewtablelock("t1"); +select calcleartablelock(0); +select callastinsertid("t1"); +select calgetsqlcount(); + +DROP TABLE t1; +DROP DATABASE cal_test_db; diff --git a/mysql-test/columnstore/basic/t/mcs_named_udfs.test b/mysql-test/columnstore/basic/t/mcs_named_udfs.test new file mode 100644 index 000000000..980c3fc69 --- /dev/null +++ b/mysql-test/columnstore/basic/t/mcs_named_udfs.test @@ -0,0 +1,24 @@ +-- source ../include/have_columnstore.inc + +--disable_warnings +DROP DATABASE IF EXISTS cal_test_db; +--enable_warnings + +CREATE DATABASE cal_test_db; +USE cal_test_db; + +create table t1(a int, b int comment 'autoincrement=1') engine=columnstore; + +select mcsflushcache(); +select mcssettrace(0); +select mcssetparms("pmmaxmemorysmallside","2048000000"); +select mcsgettrace(); +select mcsgetversion()=calgetversion(); + +select mcsviewtablelock("t1"); +select mcscleartablelock(0); +select mcslastinsertid("t1"); +select mcsgetsqlcount(); + +DROP TABLE t1; +DROP DATABASE cal_test_db; diff --git a/mysql-test/columnstore/basic/t/type_timestamp.test b/mysql-test/columnstore/basic/t/type_timestamp.test new file mode 100644 index 000000000..2789f6e7c --- /dev/null +++ b/mysql-test/columnstore/basic/t/type_timestamp.test @@ -0,0 +1,131 @@ +--source ../include/have_columnstore.inc +--source ../include/combinations.myisam-columnstore.inc + +--echo # +--echo # Test cases for the TIMESTAMP datatype +--echo # + +# Test bulk insert/literals/microseconds +--disable_warnings +DROP DATABASE IF EXISTS timestamp_test; +--enable_warnings + +CREATE DATABASE timestamp_test; +USE timestamp_test; + +## Test the effect of changing timezones on timestamp values +CREATE TABLE ctimestamp (a timestamp); +SET time_zone='-5:00'; +INSERT INTO ctimestamp VALUES ('2019-01-01 01:02:03'), ('2019-05-05 01:01:01'); +SET time_zone='+1:00'; +SELECT a FROM ctimestamp ORDER BY a; +SET time_zone='-2:00'; +SELECT a FROM ctimestamp ORDER BY a; + +## Test bulk insert using cpimport +CREATE TABLE ctimestamp2 (a timestamp DEFAULT 0); +INSERT INTO ctimestamp2 SELECT * FROM ctimestamp; +SELECT a FROM ctimestamp2 ORDER BY a; + +## Test literals +CREATE TABLE ctimestamp3 (a timestamp); +INSERT INTO ctimestamp3 VALUES (19940101), (940101), +(19940101010203), (940101010203), ('1994-01-01T01:02:03'); +SELECT a FROM ctimestamp3 ORDER BY a; + +## Test microseconds +CREATE TABLE ctimestamp4 (a timestamp(6) default 0); +INSERT INTO ctimestamp4 VALUES (0), ('2019-01-01 01:01:01.123456'); +SELECT a, microsecond(a) FROM ctimestamp4 ORDER BY a; + +# Test distributed functions +--disable_warnings +DROP DATABASE IF EXISTS timestamp_test; +--enable_warnings + +CREATE DATABASE timestamp_test; +USE timestamp_test; + +CREATE TABLE ctimestamp (a timestamp); +SET time_zone='+0:00'; +INSERT INTO ctimestamp VALUES ('2019-01-02 00:02:03'), +('2019-01-02 01:02:03'), ('2019-01-02 10:11:12'); +SET time_zone='+1:00'; + +SELECT a, a BETWEEN '2019-01-02 02:00:00' AND '2019-01-02 13:00:00' +FROM ctimestamp ORDER BY a; + +SELECT a, IF(a < '2019-01-02 02:00:00', 'yes', 'no'), +ADDTIME(a, '1:1:1'), STR_TO_DATE(a, '%Y-%m-%d %H:%i:%s'), +EXTRACT(DAY_HOUR FROM a), EXTRACT(MINUTE_SECOND FROM a), +TIME_FORMAT(a, '%H:\%i:\%s'), a RLIKE '02:03', IFNULL(NULL, a), +CASE a WHEN '2019-01-02 01:02:03' THEN 'found' WHEN '2019-01-02 11:11:12' +THEN 'found2' ELSE 'notfound' END, CHAR_LENGTH(a), +CAST(a AS UNSIGNED INT), CAST(a AS CHAR), CAST(a AS DATE), +TIME(CAST(a AS DATETIME)), TIME(COALESCE(NULL, a)), HEX(a), +NULLIF(a, '2019-01-02 01:02:03'), TIMEDIFF(a, '2019-01-01 01:02:03') +FROM ctimestamp ORDER BY a; + +INSERT INTO ctimestamp VALUES ('2020-01-03 12:12:12'), +('2020-05-06 12:12:12'), ('2020-10-28 12:12:12'); + +SELECT a, DAYNAME(a), DAYOFWEEK(a), DATE_FORMAT(a, '%W %M %Y'), +MONTHNAME(a), DATE(a), YEARWEEK(a), DAYOFYEAR(a), YEAR(a), +a + INTERVAL 1 DAY, TIMESTAMPDIFF(DAY, a, '2020-01-01'), +LAST_DAY(a), TRUNCATE(a, -2), a IN ('2019-01-02 01:02:03', a), +TO_DAYS(a), DAY(a), WEEK(a), WEEKDAY(a), GREATEST(a, '2020-07-01'), +MONTH(a), QUARTER(a), DATE_ADD(a, INTERVAL 1 SECOND) +FROM ctimestamp WHERE a > '2020-01-01' ORDER BY a; + +SELECT UNIX_TIMESTAMP(a), TIME_TO_SEC(a), CEIL(a), +CAST(LEAST(a, '2019-03-03 00:00:00') AS DATETIME), +ROUND(a), SECOND(a), MINUTE(a), HOUR(a), FLOOR(a) +FROM ctimestamp ORDER BY a; + +# Test window functions +--disable_warnings +DROP DATABASE IF EXISTS timestamp_test; +--enable_warnings + +CREATE DATABASE timestamp_test; +USE timestamp_test; + +CREATE TABLE ctimestamp (a timestamp, b int); +SET time_zone='+0:00'; +INSERT INTO ctimestamp VALUES ('2019-01-03 12:12:12', 1), +('2019-01-04 12:12:12', 2), ('2019-01-03 12:12:12', 4), +('2019-01-03 12:12:12', 2), ('2019-01-04 12:12:12', 1); + +## Test SUM + +SELECT a, b, SUM(b) over (PARTITION BY a ORDER BY a, b +ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) c +FROM ctimestamp; + +## Test MAX + +SELECT a, b, MAX(a) over (PARTITION BY b ORDER BY a desc +ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) c +FROM ctimestamp; + +# Test aggregate functions +--disable_warnings +DROP DATABASE IF EXISTS timestamp_test; +--enable_warnings + +CREATE DATABASE timestamp_test; +USE timestamp_test; + +CREATE TABLE ctimestamp (a int, b timestamp); +INSERT INTO ctimestamp VALUES (1, 20190101), (1, 20200202), +(2, 20190202), (2, 20200202), (2, 20190101); + +# Test count(*) +SELECT b, count(*) FROM ctimestamp GROUP BY b ORDER BY b; + +# Test max/min +SELECT b, max(a), min(a) FROM ctimestamp GROUP BY b ORDER BY b; +SELECT a, max(b), min(b) FROM ctimestamp GROUP BY a ORDER BY a; + +# Cleanup +DROP DATABASE timestamp_test; diff --git a/mysql-test/columnstore/bugfixes/mcol-4940.result b/mysql-test/columnstore/bugfixes/mcol-4940.result new file mode 100644 index 000000000..abb04e159 --- /dev/null +++ b/mysql-test/columnstore/bugfixes/mcol-4940.result @@ -0,0 +1,11 @@ +DROP DATABASE IF EXISTS mcol_4940; +CREATE DATABASE mcol_4940; +USE mcol_4940; +create table rounding_table ( a int, b double, c double) engine=columnstore; +insert into rounding_table values (26805, 1252, -9647); +insert into rounding_table values (26806, 573, -2804.5); +SELECT CASE a WHEN 26805 THEN ROUND(c/b, 2) WHEN 26806 THEN b END MCOL4940 FROM ( SELECT a, SUM(b) b, SUM(c) c FROM rounding_table GROUP BY a ) abc ; +MCOL4940 +-7.71 +573 +DROP DATABASE mcol_4940; diff --git a/mysql-test/columnstore/bugfixes/mcol-4940.test b/mysql-test/columnstore/bugfixes/mcol-4940.test new file mode 100644 index 000000000..82c417811 --- /dev/null +++ b/mysql-test/columnstore/bugfixes/mcol-4940.test @@ -0,0 +1,15 @@ +--source ../include/have_columnstore.inc +--disable_warnings +DROP DATABASE IF EXISTS mcol_4940; +--enable_warnings +CREATE DATABASE mcol_4940; +USE mcol_4940; + +create table rounding_table ( a int, b double, c double) engine=columnstore; +insert into rounding_table values (26805, 1252, -9647); +insert into rounding_table values (26806, 573, -2804.5); + +--sorted_result +SELECT CASE a WHEN 26805 THEN ROUND(c/b, 2) WHEN 26806 THEN b END MCOL4940 FROM ( SELECT a, SUM(b) b, SUM(c) c FROM rounding_table GROUP BY a ) abc ; + +DROP DATABASE mcol_4940; diff --git a/oam/etc/Columnstore.xml b/oam/etc/Columnstore.xml index 98014307e..aa53a018e 100644 --- a/oam/etc/Columnstore.xml +++ b/oam/etc/Columnstore.xml @@ -9,61 +9,6 @@ 8601 unassigned - - 0.0.0.0 - 8602 - - - 127.0.0.1 - 8603 - - - 127.0.0.1 - 8606 - - - 127.0.0.1 - 8604 - - - 0.0.0.0 - 8605 - - - - - 127.0.0.1 - 8800 - - - 0.0.0.0 - 8800 - - - 0.0.0.0 - 8800 - - - 127.0.0.1 - 8800 - - - 0.0.0.0 - 8622 - - - 0.0.0.0 - 8622 - - - 127.0.0.1 - 8622 - 127.0.0.1 8630 @@ -85,7 +30,6 @@ 128 10K 0 - 13 512 512 @@ -96,206 +40,40 @@ y - - - + + + 127.0.0.1 8620 - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - columnstore-1 pm1 - unassigned pm1 - 1 - 3 - 12 // 2.5 minutes - 1 - /var/lib/columnstore/data1 - /var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves - /var/lib/columnstore/data1/systemFiles/dbrm/tablelocks + /var/lib/columnstore/data1 + /var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves + /var/lib/columnstore/data1/systemFiles/dbrm/tablelocks 15 100000 - 90 - 80 - 70 - 10 - 0.0.0.0 - 128M 10 - 10 - 120 - restartSystem - n 95 OFF /rdwrscratch - /columnstore_tmp_files /tmp/columnstore_tmp_files - dm - Director Module - SIMPLEX - 0 - 0.0.0.0 - unassigned - ENABLED - 0 - 0 - 0 - 0 - 90 - 80 - 70 - 90 - 0 - 0 - 90 - 80 - 70 - / - unassigned - unassigned um User Module - SIMPLEX 0 0.0.0.0 unassigned @@ -318,7 +96,6 @@ unassigned pm Performance Module - SIMPLEX 1 127.0.0.1 localhost @@ -340,30 +117,24 @@ 1 1 - - 0 - unassigned - 0.0.0.0 - ENABLED - 1000 - /var/lib/columnstore/data1/systemFiles/dbrm/SMTxnID + /var/lib/columnstore/data1/systemFiles/dbrm/SMTxnID + One version buffer file will be put on each DB root. --> 1GB - /var/lib/columnstore/data1/systemFiles/dbrm/oidbitmap + /var/lib/columnstore/data1/systemFiles/dbrm/oidbitmap 3000 - /var/log/mariadb/columnstore/data/bulk - /var/lib/columnstore/data1/systemFiles/bulkRollback + /var/log/mariadb/columnstore/data/bulk + /var/lib/columnstore/data1/systemFiles/bulkRollback 98 1 @@ -378,55 +149,10 @@ 8700 pm1 - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - + 1 @@ -439,40 +165,13 @@ 50 - n - y - y 2 y n internal internal - /etc/rsyslog.d/49-columnstore.conf - unassigned - autoassign - unassigned - unassigned - unassigned - gp2 - unassigned - unassigned - unassigned - gp2 - unassigned - y - y - 0 - unassigned - unassigned - n - 0 - unassigned - n - 3306 - /dev/xvd - /var/lock/subsys /etc/profile.d/columnstoreAlias.sh - + - 64 1G 25% 100 N Y - Snappy + Snappy 16K @@ -511,11 +209,6 @@ 100 - - 1M - 1M - 512 - @@ -539,17 +232,17 @@ Y - Snappy + Snappy 127.0.0.1 0 - - 30 - N - - - - + + 30 + N + + + + diff --git a/oam/etc/Columnstore.xml.singleserver b/oam/etc/Columnstore.xml.singleserver deleted file mode 100644 index 483207323..000000000 --- a/oam/etc/Columnstore.xml.singleserver +++ /dev/null @@ -1,546 +0,0 @@ - - - - - 127.0.0.1 - 8601 - pm1 - - - 0.0.0.0 - 8602 - - - 127.0.0.1 - 8603 - - - 127.0.0.1 - 8606 - - - 127.0.0.1 - 8604 - - - 0.0.0.0 - 8605 - - - 127.0.0.1 - 8800 - - - 0.0.0.0 - 8800 - - - 0.0.0.0 - 8800 - - - 127.0.0.1 - 8800 - - - 0.0.0.0 - 8622 - - - 0.0.0.0 - 8622 - - - 127.0.0.1 - 8622 - - - 127.0.0.1 - 8630 - - - 127.0.0.1 - 8612 - - - 127.0.0.1 - 8614 - - - 10000 - - - 1 - 8 - 128 - - 10K - 0 - 13 - 512 - 512 - - 1 - 0 - n - - - - 95 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - C - columnstore-1 - pm1 - unassigned - - - 1 - /var/lib/columnstore/data1 - /var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves - /var/lib/columnstore/data1/systemFiles/dbrm/tablelocks - 20 - 100000 - 90 - 80 - 70 - /tmp - /tmp - 10 - 0.0.0.0 - 128M - 10 - /tmp/columnstore_tmp_files - 10 - 3 - 10 - 120 - restartSystem - n - 95 - OFF - - /tmp/rdwrscratch - - /tmp/columnstore_tmp_files - - - dm - Director Module - SIMPLEX - 0 - 0.0.0.0 - unassigned - ENABLED - 0 - 0 - 0 - 0 - 90 - 80 - 70 - 90 - 0 - 0 - 90 - 80 - 70 - / - unassigned - unassigned - um - User Module - SIMPLEX - 0 - 0.0.0.0 - unassigned - ENABLED - 0 - 0 - 0 - 0 - 90 - 80 - 70 - 90 - 0 - 0 - 90 - 80 - 70 - / - unassigned - unassigned - pm - Performance Module - SIMPLEX - 1 - 127.0.0.1 - localhost - ENABLED - 0 - 0 - 0 - 0 - 90 - 80 - 70 - 90 - 0 - 0 - 90 - 80 - 70 - / - 1 - 1 - - - 0 - unassigned - 0.0.0.0 - ENABLED - - - 1000 - /var/lib/columnstore/data1/systemFiles/dbrm/CalpontShm - /var/lib/columnstore/data1/systemFiles/dbrm/SMTxnID - - - /var/lib/columnstore/data1/systemFiles/dbrm/CalpontSessionMonitorShm - 10 - - - - 1GB - - - - - /var/lib/columnstore/data1/systemFiles/dbrm/oidbitmap - - 3000 - - - /var/lib/columnstore/data/bulk - /var/lib/columnstore/data/bulk/rollback - 98 - 1 - - - 1 - 127.0.0.1 - 8616 - - - - 127.0.0.1 - 8700 - pm1 - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - - - 50 - - 1 - 0 - 0 - 65536 - 2K - 200 - 0 - - - n - y - y - 2 - n - n - internal - internal - unassigned - unassigned - unassigned - rpm - unassigned - unassigned - us-east-1 - unassigned - unassigned - unassigned - unassigned - unassigned - unassigned - unassigned - unassigned - n - n - 0 - unassigned - unassigned - n - unassigned - unassigned - 3306 - /var/lock/subsys - - - - 4 - 0x0 - - - 128 - 128K - 64 - 1G - 25% - 10% - 100 - N - Y - - - 16K - 16 - 1 - - - - - 100 - - - 1M - 1M - 512 - - - - - - - - - 127.0.0.1 - 3306 - root - - - - - - - N - - - N - - - Y - - - 127.0.0.1 - 0 - - - 30 - N - - diff --git a/oam/install_scripts/CMakeLists.txt b/oam/install_scripts/CMakeLists.txt index b010b033e..80ca95be9 100644 --- a/oam/install_scripts/CMakeLists.txt +++ b/oam/install_scripts/CMakeLists.txt @@ -10,7 +10,6 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mariadb-columnstore-start.sh.in" "${ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-workernode.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-workernode.service" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-controllernode.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-controllernode.service" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-primproc.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-primproc.service" @ONLY) -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-exemgr.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-exemgr.service" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-writeengineserver.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-writeengineserver.service" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-dmlproc.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-dmlproc.service" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-ddlproc.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-ddlproc.service" @ONLY) @@ -21,12 +20,12 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-loadbrm.py.in" "${CMAKE_CURRENT_ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-savebrm.py.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-savebrm.py" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslog.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslog" @ONLY) -install(PROGRAMS columnstore-post-install - columnstore-pre-uninstall - columnstore_run.sh - post-mysql-install - post-mysqld-install - columnstoreSyslogSetup.sh +install(PROGRAMS columnstore-post-install + columnstore-pre-uninstall + columnstore_run.sh + post-mysql-install + post-mysqld-install + columnstoreSyslogSetup.sh mcs-stop-controllernode.sh mcs-loadbrm.py mcs-savebrm.py @@ -45,7 +44,6 @@ install(FILES mariadb-columnstore.service mcs-workernode.service mcs-controllernode.service mcs-primproc.service - mcs-exemgr.service mcs-writeengineserver.service mcs-dmlproc.service mcs-ddlproc.service diff --git a/oam/install_scripts/columnstore-post-install.in b/oam/install_scripts/columnstore-post-install.in index dea0eba12..8f8d33832 100755 --- a/oam/install_scripts/columnstore-post-install.in +++ b/oam/install_scripts/columnstore-post-install.in @@ -70,7 +70,7 @@ quiet=0 stop_mysqld=0 if [ -z "$(pgrep -x mariadbd)" ];then - # Startup mysqld + # Startup mysqld systemctl cat mariadb.service > /dev/null 2>&1 if [ $? -eq 0 ] && [ $(running_systemd) -eq 0 ]; then systemctl start mariadb.service @@ -165,8 +165,6 @@ if [ $user = "root" ]; then cp @ENGINE_SUPPORTDIR@/mcs-ddlproc.service /lib/systemd/system/. >/dev/null 2>&1 cp @ENGINE_SUPPORTDIR@/mcs-dmlproc.service /usr/lib/systemd/system/. >/dev/null 2>&1 cp @ENGINE_SUPPORTDIR@/mcs-dmlproc.service /lib/systemd/system/. >/dev/null 2>&1 - cp @ENGINE_SUPPORTDIR@/mcs-exemgr.service /usr/lib/systemd/system/. >/dev/null 2>&1 - cp @ENGINE_SUPPORTDIR@/mcs-exemgr.service /lib/systemd/system/. >/dev/null 2>&1 cp @ENGINE_SUPPORTDIR@/mcs-primproc.service /usr/lib/systemd/system/. >/dev/null 2>&1 cp @ENGINE_SUPPORTDIR@/mcs-primproc.service /lib/systemd/system/. >/dev/null 2>&1 cp @ENGINE_SUPPORTDIR@/mcs-workernode.service /usr/lib/systemd/system/mcs-workernode@.service >/dev/null 2>&1 @@ -203,7 +201,7 @@ if [ $user = "root" ]; then update-rc.d columnstore defaults 99 > /dev/null 2>&1 else echo "" - echo "Package 'systemctl', 'chkconfig' or 'update-rc.d' not installed, contact your sysadmin if you want to setup to autostart for columnstore" + echo "Package 'systemctl', 'chkconfig' or 'update-rc.d' not installed, contact your sysadmin if you want to setup to autostart for columnstore" fi fi fi @@ -225,7 +223,7 @@ if [ $user = "root" ]; then fi else chown $user:$user @ENGINE_SYSCONFDIR@/columnstore/Columnstore.xml - + cat < /dev/null 2>&1 diff --git a/oam/install_scripts/columnstore-pre-uninstall.in b/oam/install_scripts/columnstore-pre-uninstall.in index 941d9aa7a..82e2bd570 100755 --- a/oam/install_scripts/columnstore-pre-uninstall.in +++ b/oam/install_scripts/columnstore-pre-uninstall.in @@ -35,7 +35,7 @@ systemctl cat mariadb-columnstore.service > /dev/null 2>&1 if [ $? -eq 0 ] && [ $(running_systemd) -eq 0 ]; then systemctl stop mariadb-columnstore >/dev/null 2>&1 else - PROGS='PrimProc ExeMgr DMLProc DDLProc WriteEngineServer StorageManager controllernode workernode' + PROGS='PrimProc ExeMgr DMLProc DDLProc WriteEngineServer StorageManager controllernode workernode' kill $(pidof $PROGS) > /dev/null sleep 3 kill -9 $(pidof $PROGS) > /dev/null @@ -43,7 +43,7 @@ else fi if [ -n "$(pgrep -x ProcMon)" ] || [ -n "$(pgrep -x ProcMgr)" ];then - # Old system must be running, kill ProcMon/ProcMgr + # Old system must be running, kill ProcMon/ProcMgr pkill ProcMon pkill ProcMgr fi @@ -94,8 +94,12 @@ if [ -n "$systemctl" ] && [ $(running_systemd) -eq 0 ]; then rm -f /lib/systemd/system/mcs-ddlproc.service rm -f /usr/lib/systemd/system/mcs-dmlproc.service rm -f /lib/systemd/system/mcs-dmlproc.service - rm -f /usr/lib/systemd/system/mcs-exemgr.service - rm -f /lib/systemd/system/mcs-exemgr.service + if [[ -f /usr/lib/systemd/system/mcs-exemgr.service ]] + rm -f /usr/lib/systemd/system/mcs-exemgr.service + fi + if [[ -f /lib/systemd/system/mcs-exemgr.service ]] + rm -f /lib/systemd/system/mcs-exemgr.service + fi rm -f /usr/lib/systemd/system/mcs-primproc.service rm -f /lib/systemd/system/mcs-primproc.service rm -f /usr/lib/systemd/system/mcs-workernode@.service @@ -123,7 +127,7 @@ else updaterc=`which update-rc.d 2>/dev/null` if [ -n "$updaterc" ]; then update-rc.d -f columnstore remove > /dev/null 2>&1 - rm -f /etc/init.d/columnstore > /dev/null 2>&1 + rm -f /etc/init.d/columnstore > /dev/null 2>&1 fi fi fi diff --git a/oam/install_scripts/mariadb-columnstore-start.sh.in b/oam/install_scripts/mariadb-columnstore-start.sh.in index 351512d62..f41b6647d 100644 --- a/oam/install_scripts/mariadb-columnstore-start.sh.in +++ b/oam/install_scripts/mariadb-columnstore-start.sh.in @@ -12,7 +12,6 @@ flock -n "$fd_lock" || exit 0 /bin/systemctl start mcs-controllernode /bin/systemctl start mcs-primproc /bin/systemctl start mcs-writeengineserver -/bin/systemctl start mcs-exemgr /bin/systemctl start mcs-dmlproc /bin/systemctl start mcs-ddlproc su -s /bin/sh -c '@ENGINE_BINDIR@/dbbuilder 7' @DEFAULT_USER@ 1> @ENGINE_LOGDIR@/install/dbbuilder.log diff --git a/oam/install_scripts/mariadb-columnstore-stop.sh b/oam/install_scripts/mariadb-columnstore-stop.sh index 9bce80618..9e1a879ab 100644 --- a/oam/install_scripts/mariadb-columnstore-stop.sh +++ b/oam/install_scripts/mariadb-columnstore-stop.sh @@ -4,7 +4,6 @@ /bin/systemctl stop mcs-dmlproc /bin/systemctl stop mcs-ddlproc -/bin/systemctl stop mcs-exemgr /bin/systemctl stop mcs-writeengineserver /bin/systemctl stop mcs-primproc /bin/systemctl stop mcs-controllernode diff --git a/oam/install_scripts/mcs-exemgr.service.in b/oam/install_scripts/mcs-exemgr.service.in deleted file mode 100644 index b9a36fcaf..000000000 --- a/oam/install_scripts/mcs-exemgr.service.in +++ /dev/null @@ -1,20 +0,0 @@ -[Unit] -Description=mcs-exemgr - -# restart/start mcs-exemgr on restart/start of mcs-primproc -PartOf=mcs-primproc.service -After=network.target mcs-primproc.service - -[Service] -Type=forking - -User=@DEFAULT_USER@ -Group=@DEFAULT_GROUP@ -LimitNOFILE=65536 -LimitNPROC=65536 - -ExecStartPre=/usr/bin/env bash -c "ldconfig -p | grep -m1 libjemalloc > /dev/null || echo 'Please install jemalloc to avoid ColumnStore performance degradation and unexpected service interruptions.'" -ExecStart=/usr/bin/env bash -c "LD_PRELOAD=$(ldconfig -p | grep -m1 libjemalloc | awk '{print $1}') exec @ENGINE_BINDIR@/ExeMgr" - -Restart=on-failure -TimeoutStopSec=2 diff --git a/oam/install_scripts/mcs-writeengineserver.service.in b/oam/install_scripts/mcs-writeengineserver.service.in index 32564fc61..2e1470022 100644 --- a/oam/install_scripts/mcs-writeengineserver.service.in +++ b/oam/install_scripts/mcs-writeengineserver.service.in @@ -1,9 +1,9 @@ [Unit] Description=WriteEngineServer -# restart/stop mcs-writeengineserver on restart/stop of mcs-exemgr -PartOf=mcs-exemgr.service -After=network.target mcs-exemgr.service +# restart/stop mcs-writeengineserver on restart/stop of mcs-primproc +PartOf=mcs-primproc.service +After=network.target mcs-primproc.service [Service] Type=forking diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index cfbd00886..471535848 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -167,7 +167,6 @@ void Oam::getSystemConfig(const std::string& moduletype, ModuleTypeConfig& modul const string Section = "SystemModuleConfig"; const string MODULE_TYPE = "ModuleType"; const string MODULE_DESC = "ModuleDesc"; - const string MODULE_RUN_TYPE = "RunType"; const string MODULE_COUNT = "ModuleCount"; const string MODULE_DISABLE_STATE = "ModuleDisableState"; const string MODULE_CPU_CRITICAL = "ModuleCPUCriticalThreshold"; @@ -198,7 +197,6 @@ void Oam::getSystemConfig(const std::string& moduletype, ModuleTypeConfig& modul string ModuleCount = MODULE_COUNT + itoa(moduleTypeID); string ModuleType = MODULE_TYPE + itoa(moduleTypeID); string ModuleDesc = MODULE_DESC + itoa(moduleTypeID); - string ModuleRunType = MODULE_RUN_TYPE + itoa(moduleTypeID); string ModuleCPUCriticalThreshold = MODULE_CPU_CRITICAL + itoa(moduleTypeID); string ModuleCPUMajorThreshold = MODULE_CPU_MAJOR + itoa(moduleTypeID); string ModuleCPUMinorThreshold = MODULE_CPU_MINOR + itoa(moduleTypeID); @@ -216,7 +214,6 @@ void Oam::getSystemConfig(const std::string& moduletype, ModuleTypeConfig& modul moduletypeconfig.ModuleCount = strtol(sysConfig->getConfig(Section, ModuleCount).c_str(), 0, 0); moduletypeconfig.ModuleType = sysConfig->getConfig(Section, ModuleType); moduletypeconfig.ModuleDesc = sysConfig->getConfig(Section, ModuleDesc); - moduletypeconfig.RunType = sysConfig->getConfig(Section, ModuleRunType); moduletypeconfig.ModuleCPUCriticalThreshold = strtol(sysConfig->getConfig(Section, ModuleCPUCriticalThreshold).c_str(), 0, 0); moduletypeconfig.ModuleCPUMajorThreshold = @@ -1066,4 +1063,3 @@ std::string Oam::itoa(const int i) return ss.str(); } } // namespace oam -// vim:ts=4 sw=4: diff --git a/oam/oamcpp/liboamcpp.h b/oam/oamcpp/liboamcpp.h index cf5e03248..b4fbd207b 100644 --- a/oam/oamcpp/liboamcpp.h +++ b/oam/oamcpp/liboamcpp.h @@ -115,7 +115,6 @@ const std::string UnassignedName = "unassigned"; const std::string configSections[] = {"SystemConfig", "SystemModuleConfig", "SystemModuleConfig", - "SystemExtDeviceConfig", "SessionManager", "VersionBuffer", "OIDManager", @@ -235,7 +234,6 @@ struct ModuleTypeConfig_s { std::string ModuleType; //!< Module Type std::string ModuleDesc; //!< Module Description - std::string RunType; //!< Run Type uint16_t ModuleCount; //!< Module Equipage Count uint16_t ModuleCPUCriticalThreshold; //!< CPU Critical Threahold % uint16_t ModuleCPUMajorThreshold; //!< CPU Major Threahold % @@ -433,4 +431,3 @@ class Oam #undef EXPORT -// vim:ts=4 sw=4: diff --git a/oamapps/columnstoreDB/columnstoreDB.cpp b/oamapps/columnstoreDB/columnstoreDB.cpp index f716f9bd8..3bdca8b64 100644 --- a/oamapps/columnstoreDB/columnstoreDB.cpp +++ b/oamapps/columnstoreDB/columnstoreDB.cpp @@ -271,7 +271,7 @@ int main(int argc, char** argv) Oam oam; BRM::DBRM dbrm; - char c; + signed char c; // Invokes member function `int operator ()(void);' while ((c = getopt(argc, argv, "c:h")) != -1) diff --git a/oamapps/columnstoreSupport/columnstoreSupport.cpp b/oamapps/columnstoreSupport/columnstoreSupport.cpp index 0f8abad74..d7bf28a9b 100644 --- a/oamapps/columnstoreSupport/columnstoreSupport.cpp +++ b/oamapps/columnstoreSupport/columnstoreSupport.cpp @@ -62,7 +62,6 @@ string rootPassword = ""; string debug_flag = "0"; string mysqlpw = " "; string tmpDir; -string ProfileFile; int runningThreads = 0; pthread_mutex_t mutex1 = PTHREAD_MUTEX_INITIALIZER; @@ -121,9 +120,8 @@ void* childReportThread(threadInfo_t* st) cout << "Get " + reportType + " report data for " + remoteModuleName + " " << endl; - string cmd = "remote_command.sh " + remoteModuleIP + " " + rootPassword + " '. " + ProfileFile + ";" + - reportType + "Report.sh " + remoteModuleName + "' " + debug_flag + " - forcetty"; - + string cmd = "remote_command.sh " + remoteModuleIP + " " + rootPassword + ";" + reportType + "Report.sh " + + remoteModuleName + "' " + debug_flag + " - forcetty"; int rtnCode = system(cmd.c_str()); if (WEXITSTATUS(rtnCode) != 0) @@ -333,7 +331,7 @@ int main(int argc, char* argv[]) } // get Local Module Name and Server Install Indicator - string singleServerInstall; + string singleServerInstall = "n"; oamModuleInfo_t st; @@ -348,15 +346,6 @@ int main(int argc, char* argv[]) exit(-1); } - try - { - oam.getSystemConfig("SingleServerInstall", singleServerInstall); - } - catch (...) - { - singleServerInstall = "y"; - } - if (argc == 1) { argv[1] = &helpArg[0]; @@ -592,17 +581,6 @@ int main(int argc, char* argv[]) exit(-1); } - // Get Profile file - try - { - ProfileFile = sysConfig->getConfig(InstallSection, "ProfileFile"); - } - catch (...) - { - cout << "ERROR: Problem getting ProfileFile" << endl; - exit(-1); - } - string ModuleSection = "SystemModuleConfig"; for (unsigned int i = 0; i < sysModuleTypeConfig.moduletypeconfig.size(); i++) diff --git a/oamapps/columnstoreSupport/mcsSupportUtil.cpp b/oamapps/columnstoreSupport/mcsSupportUtil.cpp index 4c3e7106e..586a342e5 100644 --- a/oamapps/columnstoreSupport/mcsSupportUtil.cpp +++ b/oamapps/columnstoreSupport/mcsSupportUtil.cpp @@ -160,7 +160,6 @@ void getModuleTypeConfig(FILE* pOutputFile) fprintf(pOutputFile, "ModuleType '%s' Configuration information\n", moduletype.c_str()); fprintf(pOutputFile, "ModuleDesc = %s\n", systemmoduletypeconfig.moduletypeconfig[i].ModuleDesc.c_str()); - fprintf(pOutputFile, "RunType = %s\n", systemmoduletypeconfig.moduletypeconfig[i].RunType.c_str()); fprintf(pOutputFile, "ModuleCount = %i\n", moduleCount); if (moduleCount > 0) @@ -379,23 +378,16 @@ void getStorageConfig(FILE* pOutputFile) string volumeName = oam::UnassignedName; string deviceNameID = "PMVolumeDeviceName" + oam.itoa(*pt); string deviceName = oam::UnassignedName; - string amazonDeviceNameID = "PMVolumeAmazonDeviceName" + oam.itoa(*pt); - string amazondeviceName = oam::UnassignedName; try { oam.getSystemConfig(volumeNameID, volumeName); oam.getSystemConfig(deviceNameID, deviceName); - oam.getSystemConfig(amazonDeviceNameID, amazondeviceName); } catch (...) { continue; } - - fprintf(pOutputFile, - "Amazon EC2 Volume Name/Device Name/Amazon Device Name for DBRoot%u: %s, %s, %s", *pt, - volumeName.c_str(), deviceName.c_str(), amazondeviceName.c_str()); } } catch (exception& e) @@ -412,22 +404,16 @@ void getStorageConfig(FILE* pOutputFile) string volumeName = oam::UnassignedName; string deviceNameID = "PMVolumeDeviceName" + oam.itoa(*pt1); string deviceName = oam::UnassignedName; - string amazonDeviceNameID = "PMVolumeAmazonDeviceName" + oam.itoa(*pt1); - string amazondeviceName = oam::UnassignedName; try { oam.getSystemConfig( volumeNameID, volumeName); oam.getSystemConfig( deviceNameID, deviceName); - oam.getSystemConfig( amazonDeviceNameID, amazondeviceName); } catch (...) { continue; } - - fprintf(pOutputFile,"Amazon EC2 Volume Name/Device Name/Amazon Device Name for DBRoot%u: %s, %s, - %s",*pt1,volumeName.c_str(),deviceName.c_str(),amazondeviceName.c_str()); }*/ } diff --git a/oamapps/postConfigure/mycnfUpgrade.cpp b/oamapps/postConfigure/mycnfUpgrade.cpp index d46bd773a..ae79eb1bc 100644 --- a/oamapps/postConfigure/mycnfUpgrade.cpp +++ b/oamapps/postConfigure/mycnfUpgrade.cpp @@ -26,28 +26,30 @@ * @file */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include +#include #include +#include +#include +#include +#include #include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + #include "liboamcpp.h" #include "installdir.h" #include "mcsconfig.h" @@ -178,7 +180,7 @@ int main(int argc, char* argv[]) { includeArg = line; - boost::regex icludeArgRegEx("^#*\\s*" + includeArg + "\\s*="); + std::regex icludeArgRegEx("^#*\\s*" + includeArg + "\\s*="); // see if in columnstore.cnf.rpmsave ifstream mycnfsavefile(mycnfsaveFile.c_str()); char line[200]; @@ -188,7 +190,7 @@ int main(int argc, char* argv[]) { oldbuf = line; - if (boost::regex_search(oldbuf.begin(), oldbuf.end(), icludeArgRegEx)) + if (std::regex_search(oldbuf.begin(), oldbuf.end(), icludeArgRegEx)) { // found in columnstore.cnf.rpmsave, check if this is commented out if (line[0] != '#') @@ -205,7 +207,7 @@ int main(int argc, char* argv[]) { newbuf = line1; - if (boost::regex_search(newbuf.begin(), newbuf.end(), icludeArgRegEx)) + if (std::regex_search(newbuf.begin(), newbuf.end(), icludeArgRegEx)) { newbuf = oldbuf; cout << "Updated argument: " << includeArg << endl; @@ -240,9 +242,9 @@ int main(int argc, char* argv[]) while (mycnffile.getline(line1, 200)) { newbuf = line1; - boost::regex mysqldSectionRegEx("\\[mysqld\\]"); + std::regex mysqldSectionRegEx("\\[mysqld\\]"); - if (boost::regex_search(newbuf.begin(), newbuf.end(), mysqldSectionRegEx)) + if (std::regex_search(newbuf.begin(), newbuf.end(), mysqldSectionRegEx)) { lines.push_back(newbuf); newbuf = oldbuf; @@ -284,4 +286,3 @@ int main(int argc, char* argv[]) exit(0); } -// vim:ts=4 sw=4: diff --git a/oamapps/sessionWalker/sessionwalker.cpp b/oamapps/sessionWalker/sessionwalker.cpp index 5f0960128..06cf08bd0 100644 --- a/oamapps/sessionWalker/sessionwalker.cpp +++ b/oamapps/sessionWalker/sessionwalker.cpp @@ -133,4 +133,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/primitives/blockcache/blockcacheclient.h b/primitives/blockcache/blockcacheclient.h index 307814898..e83c04c19 100644 --- a/primitives/blockcache/blockcacheclient.h +++ b/primitives/blockcache/blockcacheclient.h @@ -176,4 +176,3 @@ class blockCacheClient } // namespace dbbc -// vim:ts=4 sw=4: diff --git a/primitives/blockcache/blockrequestprocessor.h b/primitives/blockcache/blockrequestprocessor.h index c0257ef41..0970c40d5 100644 --- a/primitives/blockcache/blockrequestprocessor.h +++ b/primitives/blockcache/blockrequestprocessor.h @@ -190,4 +190,3 @@ class BlockRequestProcessor }; } // namespace dbbc -// vim:ts=4 sw=4: diff --git a/primitives/blockcache/filerequest.h b/primitives/blockcache/filerequest.h index cdca79f48..8cee46733 100644 --- a/primitives/blockcache/filerequest.h +++ b/primitives/blockcache/filerequest.h @@ -320,4 +320,3 @@ class fileRequest } // namespace dbbc -// vim:ts=4 sw=4: diff --git a/primitives/blockcache/iomanager.cpp b/primitives/blockcache/iomanager.cpp index cd0952ede..b1b727797 100644 --- a/primitives/blockcache/iomanager.cpp +++ b/primitives/blockcache/iomanager.cpp @@ -1442,4 +1442,3 @@ void ioManager::handleBlockReadError(fileRequest* fr, const string& errMsg, bool } } // namespace dbbc -// vim:ts=4 sw=4: diff --git a/primitives/blockcache/iomanager.h b/primitives/blockcache/iomanager.h index 70b4a09b5..cc11b61e7 100644 --- a/primitives/blockcache/iomanager.h +++ b/primitives/blockcache/iomanager.h @@ -147,4 +147,3 @@ void dropFDCache(); void purgeFDCache(std::vector& files); } // namespace dbbc -// vim:ts=4 sw=4: diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index 2432cc690..01be9e39d 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -1,5 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2016-2021 MariaDB Corporation + Copyright (C) 2016-2022 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -22,6 +22,7 @@ #include #include #include +#include #ifndef _MSC_VER #include #else @@ -42,6 +43,8 @@ using namespace boost; #include "simd_sse.h" #include "utils/common/columnwidth.h" +#include "exceptclasses.h" + using namespace logging; using namespace dbbc; using namespace primitives; @@ -60,6 +63,55 @@ inline uint64_t order_swap(uint64_t x) return ret; } +// Dummy template +template= sizeof(uint128_t), T>::type* = nullptr> +inline T orderSwap(T x) +{ + return x; +} + +template::type* = nullptr> +inline T orderSwap(T x) +{ + T ret = (x >> 56) | + ((x << 40) & 0x00FF000000000000ULL) | + ((x << 24) & 0x0000FF0000000000ULL) | + ((x << 8) & 0x000000FF00000000ULL) | + ((x >> 8) & 0x00000000FF000000ULL) | + ((x >> 24) & 0x0000000000FF0000ULL) | + ((x >> 40) & 0x000000000000FF00ULL) | + (x << 56); + return ret; +} + +template::type* = nullptr> +inline T orderSwap(T x) +{ + T ret = (x >> 24) | + ((x << 8) & 0x00FF0000U) | + ((x >> 8) & 0x0000FF00U) | + (x << 24); + return ret; +} + +template::type* = nullptr> +inline T orderSwap(T x) +{ + T ret = (x >> 8) | (x <<8); + return ret; +} + +template::type* = nullptr> +inline T orderSwap(T x) +{ + return x; +} + template inline int compareBlock(const void* a, const void* b) { @@ -105,8 +157,11 @@ inline bool colCompare_(const T& val1, const T& val2, uint8_t COP) } } -inline bool colCompareStr(const ColRequestHeaderDataType& type, uint8_t COP, const utils::ConstString& val1, - const utils::ConstString& val2) +inline bool colCompareStr(const ColRequestHeaderDataType &type, + uint8_t COP, + const utils::ConstString &val1, + const utils::ConstString &val2, + const bool printOut = false) { int error = 0; bool rc = primitives::StringComparator(type).op(&error, COP, val1, val2); @@ -537,16 +592,6 @@ inline bool isNullValue(const T val, const T NULL_VALUE) return val == NULL_VALUE; } -template <> -inline bool isNullValue(const int64_t val, const int64_t NULL_VALUE) -{ - //@bug 339 might be a token here - // TODO: what's up with the alternative NULL here? - constexpr const int64_t ALT_NULL_VALUE = 0xFFFFFFFFFFFFFFFELL; - - return (val == NULL_VALUE || val == ALT_NULL_VALUE); -} - // // FILTER A COLUMN VALUE // @@ -1187,7 +1232,7 @@ void scalarFiltering( #if defined(__x86_64__) template ::type* = nullptr> -inline SIMD_WRAPPER_TYPE simdDataLoadTemplate(VT& processor, const T* srcArray, const T* origSrcArray, +inline SIMD_WRAPPER_TYPE simdDataLoad(VT& processor, const T* srcArray, const T* origSrcArray, const primitives::RIDType* ridArray, const uint16_t iter) { return {processor.loadFrom(reinterpret_cast(srcArray))}; @@ -1197,7 +1242,7 @@ inline SIMD_WRAPPER_TYPE simdDataLoadTemplate(VT& processor, const T* srcArray, // TODO Move the logic into simd namespace class methods and use intrinsics template ::type* = nullptr> -inline SIMD_WRAPPER_TYPE simdDataLoadTemplate(VT& processor, const T* srcArray, const T* origSrcArray, +inline SIMD_WRAPPER_TYPE simdDataLoad(VT& processor, const T* srcArray, const T* origSrcArray, const primitives::RIDType* ridArray, const uint16_t iter) { constexpr const uint16_t WIDTH = sizeof(T); @@ -1213,6 +1258,32 @@ inline SIMD_WRAPPER_TYPE simdDataLoadTemplate(VT& processor, const T* srcArray, return {result}; } +template ::type* = nullptr> +inline SIMD_WRAPPER_TYPE simdSwapedOrderDataLoad(const ColRequestHeaderDataType &type, VT& processor, typename VT::SimdType& dataVector) +{ + return {dataVector}; +} + +template ::type* = nullptr> +inline SIMD_WRAPPER_TYPE simdSwapedOrderDataLoad(const ColRequestHeaderDataType &type, + VT& processor, typename VT::SimdType& dataVector) +{ + constexpr const uint16_t WIDTH = sizeof(T); + constexpr const uint16_t VECTOR_SIZE = VT::vecByteSize / WIDTH; + using SimdType = typename VT::SimdType; + SimdType result; + T* resultTypedPtr = reinterpret_cast(&result); + T* srcTypedPtr = reinterpret_cast(&dataVector); + for (uint32_t i = 0; i < VECTOR_SIZE; ++i) + { + utils::ConstString s{reinterpret_cast(&srcTypedPtr[i]), WIDTH}; + resultTypedPtr[i] = orderSwap(type.strnxfrm(s.rtrimZero())); + } + return {result}; +} + // This routine filters input block in a vectorized manner. // It supports all output types, all input types. // It doesn't support KIND==TEXT so upper layers filters this KIND out beforehand. @@ -1222,8 +1293,8 @@ inline SIMD_WRAPPER_TYPE simdDataLoadTemplate(VT& processor, const T* srcArray, // to glue the masks produced by actual filters. // Then it takes a vector of data, run filters and logical function using pointers. // See the corresponding dispatcher to get more details on vector processing class. -template +template void vectorizedFiltering(NewColRequestHeader* in, ColResultHeader* out, const T* srcArray, const uint32_t srcSize, primitives::RIDType* ridArray, const uint16_t ridSize, ParsedColumnFilter* parsedColumnFilter, const bool validMinMax, const T emptyValue, @@ -1233,8 +1304,12 @@ void vectorizedFiltering(NewColRequestHeader* in, ColResultHeader* out, const T* using SimdType = typename VT::SimdType; using SimdWrapperType = typename VT::SimdWrapperType; using FilterType = typename VT::FilterType; + using UT = typename std::conditional::value || datatypes::is_uint128_t::value || std::is_same::value, + FilterType, typename datatypes::make_unsigned::type>::type; VT simdProcessor; SimdType dataVec; + [[maybe_unused]] SimdType swapedOrderDataVec; + [[maybe_unused]] auto typeHolder = in->colType; SimdType emptyFilterArgVec = simdProcessor.emptyNullLoadValue(emptyValue); SimdType nullFilterArgVec = simdProcessor.emptyNullLoadValue(nullValue); MT writeMask, nonEmptyMask, nonNullMask, nonNullOrEmptyMask; @@ -1300,11 +1375,27 @@ void vectorizedFiltering(NewColRequestHeader* in, ColResultHeader* out, const T* for (uint32_t j = 0; j < filterCount; ++j) { // Preload filter argument values only once. - filterArgsVectors[j] = simdProcessor.loadValue(*((FilterType*)&filterValues[j])); + if constexpr (KIND == KIND_TEXT) + { + // Preload filter argument values only once. + // First cast filter value as the corresponding unsigned int value + UT filterValue = *((UT*)&filterValues[j]); + // Cast to ConstString to preprocess the string + utils::ConstString s{reinterpret_cast(&filterValue), sizeof(UT)}; + // Strip all 0 bytes on the right, convert byte into collation weights array + // and swap bytes order. + UT bigEndianFilterWeights = orderSwap(typeHolder.strnxfrm(s.rtrimZero())); + filterArgsVectors.push_back(simdProcessor.loadValue(bigEndianFilterWeights)); + } + else + { + FilterType filterValue = *((FilterType*)&filterValues[j]); + filterArgsVectors.push_back(simdProcessor.loadValue(filterValue)); + } switch (filterCOPs[j]) { case (COMPARE_EQ): - // Skipping extra filter pass generated by IS NULL + // Filter against NULL value if (memcmp(&filterValues[j], &nullValue, sizeof(nullValue)) == 0) copFunctorVec.push_back(std::mem_fn(&VT::nullEmptyCmpEq)); else @@ -1337,9 +1428,10 @@ void vectorizedFiltering(NewColRequestHeader* in, ColResultHeader* out, const T* { primitives::RIDType ridOffset = i * VECTOR_SIZE; assert(!HAS_INPUT_RIDS || (HAS_INPUT_RIDS && ridSize >= ridOffset)); - dataVec = simdDataLoadTemplate(simdProcessor, srcArray, - origSrcArray, ridArray, i) - .v; + dataVec = simdDataLoad(simdProcessor, srcArray, + origSrcArray, ridArray, i).v; + if constexpr(KIND==KIND_TEXT) + swapedOrderDataVec = simdSwapedOrderDataLoad(typeHolder, simdProcessor, dataVec).v; nonEmptyMask = simdProcessor.nullEmptyCmpNe(dataVec, emptyFilterArgVec); writeMask = nonEmptyMask; // NULL check @@ -1354,7 +1446,11 @@ void vectorizedFiltering(NewColRequestHeader* in, ColResultHeader* out, const T* for (uint32_t j = 0; j < filterCount; ++j) { // filter using compiled filter and preloaded filter argument - filterMask = copFunctorVec[j](simdProcessor, dataVec, filterArgsVectors[j]); + if constexpr(KIND==KIND_TEXT) + filterMask = copFunctorVec[j](simdProcessor, swapedOrderDataVec, filterArgsVectors[j]); + else + filterMask = copFunctorVec[j](simdProcessor, dataVec, filterArgsVectors[j]); + filterMask = bopFunctor(prevFilterMask, filterMask); prevFilterMask = filterMask; } @@ -1397,7 +1493,6 @@ void vectorizedFiltering(NewColRequestHeader* in, ColResultHeader* out, const T* out->Min = Min; out->Max = Max; } - // process the tail. scalarFiltering changes out contents, e.g. Min/Max, NVALS, RIDs and values array // This tail also sets out::Min/Max, out::validMinMax if validMinMax is set. uint32_t processedSoFar = rid; @@ -1534,7 +1629,8 @@ void filterColumnData(NewColRequestHeader* in, ColResultHeader* out, uint16_t* r #if defined(__x86_64__) // Don't use vectorized filtering for text based data types. - if (KIND <= KIND_FLOAT && WIDTH < 16) + if (WIDTH < 16 && + (KIND != KIND_TEXT || (KIND == KIND_TEXT && in->colType.strnxfrmIsValid()) )) { bool canUseFastFiltering = true; for (uint32_t i = 0; i < filterCount; ++i) @@ -1680,6 +1776,8 @@ template ::value || datatypes::is_uint128_t::value, T, + typename datatypes::make_unsigned::type>::type; const uint16_t ridSize = in->NVALS; uint16_t* ridArray = in->getRIDArrayPtr(W); const uint32_t itemsPerBlock = logicalBlockMode ? BLOCK_SIZE : BLOCK_SIZE / W; @@ -1690,14 +1788,12 @@ void PrimitiveProcessor::_scanAndFilterTypeDispatcher(NewColRequestHeader* in, C dataType == execplan::CalpontSystemCatalog::TEXT) && !isDictTokenScan(in)) { - filterColumnData(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter); + filterColumnData(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter); return; } if (datatypes::isUnsigned(dataType)) { - using UT = typename std::conditional::value || datatypes::is_uint128_t::value, T, - typename datatypes::make_unsigned::type>::type; filterColumnData(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter); return; } @@ -1764,4 +1860,3 @@ template void primitives::PrimitiveProcessor::columnScanAndFilter(NewC ColResultHeader*); } // namespace primitives -// vim:ts=2 sw=2: diff --git a/primitives/linux-port/dictionary.cpp b/primitives/linux-port/dictionary.cpp index 2942d4fda..40cac5014 100644 --- a/primitives/linux-port/dictionary.cpp +++ b/primitives/linux-port/dictionary.cpp @@ -31,6 +31,7 @@ using namespace std; #include "messageobj.h" #include "exceptclasses.h" #include "dataconvert.h" +#include "string_prefixes.h" #include using namespace logging; @@ -391,8 +392,13 @@ void PrimitiveProcessor::nextSig(int NVALS, const PrimToken* tokens, p_DataValue } void PrimitiveProcessor::p_Dictionary(const DictInput* in, vector* out, bool skipNulls, +#if defined(XXX_PRIMITIVES_TOKEN_RANGES_XXX) + uint32_t charsetNumber, boost::shared_ptr eqFilter, + uint8_t eqOp, uint64_t minMax[2]) +#else uint32_t charsetNumber, boost::shared_ptr eqFilter, uint8_t eqOp) +#endif { PrimToken* outToken; const DictFilterElement* filter = 0; @@ -437,6 +443,14 @@ void PrimitiveProcessor::p_Dictionary(const DictInput* in, vector* out, sigptr.len != -1; nextSig(in->NVALS, in->tokens, &sigptr, in->OutputType, (in->InputFlags ? true : false), skipNulls)) { +#if defined(XXX_PRIMITIVES_TOKEN_RANGES_XXX) + if (minMax) + { + uint64_t v = encodeStringPrefix_check_null(sigptr.data, sigptr.len, charsetNumber); + minMax[1] = minMax[1] < v ? v : minMax[1]; + minMax[0] = minMax[0] > v ? v : minMax[0]; + } +#endif // do aggregate processing if (in->OutputType & OT_AGGREGATE) { @@ -620,4 +634,3 @@ void PrimitiveProcessor::p_Dictionary(const DictInput* in, vector* out, } } // namespace primitives -// vim:ts=4 sw=4: diff --git a/primitives/linux-port/index.cpp b/primitives/linux-port/index.cpp index bbfc949ef..2d86d35e2 100644 --- a/primitives/linux-port/index.cpp +++ b/primitives/linux-port/index.cpp @@ -1130,4 +1130,3 @@ void PrimitiveProcessor::p_IdxList(const IndexListHeader* rqst, IndexListHeader* } } // namespace primitives -// vim:ts=4 sw=4: diff --git a/primitives/linux-port/primitiveprocessor.h b/primitives/linux-port/primitiveprocessor.h index bbd555b48..98bdd9fee 100644 --- a/primitives/linux-port/primitiveprocessor.h +++ b/primitives/linux-port/primitiveprocessor.h @@ -40,7 +40,7 @@ #ifdef POSIX_REGEX #include #else -#include +#include #endif #include #include @@ -54,6 +54,9 @@ class PrimTest; +// XXX: turn off dictionary range setting during scan. +#define XXX_PRIMITIVES_TOKEN_RANGES_XXX + namespace primitives { enum ColumnFilterMode @@ -423,7 +426,13 @@ class PrimitiveProcessor // void p_ColAggregate(const NewColAggRequestHeader *in, NewColAggResultHeader *out); void p_Dictionary(const DictInput* in, std::vector* out, bool skipNulls, uint32_t charsetNumber, - boost::shared_ptr eqFilter, uint8_t eqOp); +#if !defined(XXX_PRIMITIVES_TOKEN_RANGES_XXX) + boost::shared_ptr eqFilter, uint8_t eqOp +#else + boost::shared_ptr eqFilter, uint8_t eqOp, + uint64_t minMax[2] // as name suggests, [0] is min, [1] is max. +#endif + ); inline void setLogicalBlockMode(bool b) { @@ -562,4 +571,3 @@ boost::shared_ptr _parseColumnFilter( } // namespace primitives -// vim:ts=2 sw=2: diff --git a/primitives/primproc/CMakeLists.txt b/primitives/primproc/CMakeLists.txt index b8d2e3741..69d83f464 100644 --- a/primitives/primproc/CMakeLists.txt +++ b/primitives/primproc/CMakeLists.txt @@ -19,14 +19,17 @@ set(PrimProc_SRCS pseudocc.cpp rtscommand.cpp umsocketselector.cpp + serviceexemgr.cpp + sqlfrontsessionthread.cpp + rssmonfcn.cpp + activestatementcounter.cpp + femsghandler.cpp ../../utils/common/crashtrace.cpp) add_executable(PrimProc ${PrimProc_SRCS}) add_dependencies(PrimProc loggingcpp) - +target_include_directories(PrimProc PRIVATE ${Boost_INCLUDE_DIRS}) target_link_libraries(PrimProc ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} threadpool cacheutils dbbc processor) -install(TARGETS PrimProc DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) - - +install(TARGETS PrimProc DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) \ No newline at end of file diff --git a/exemgr/activestatementcounter.cpp b/primitives/primproc/activestatementcounter.cpp similarity index 98% rename from exemgr/activestatementcounter.cpp rename to primitives/primproc/activestatementcounter.cpp index 27a3df1de..b25e61549 100644 --- a/exemgr/activestatementcounter.cpp +++ b/primitives/primproc/activestatementcounter.cpp @@ -57,4 +57,3 @@ void ActiveStatementCounter::decr(bool& counted) --fStatementCount; condvar.notify_one(); } -// vim:ts=4 sw=4: diff --git a/exemgr/activestatementcounter.h b/primitives/primproc/activestatementcounter.h similarity index 98% rename from exemgr/activestatementcounter.h rename to primitives/primproc/activestatementcounter.h index 149ad23e9..87e7163aa 100644 --- a/exemgr/activestatementcounter.h +++ b/primitives/primproc/activestatementcounter.h @@ -62,4 +62,3 @@ class ActiveStatementCounter BRM::VSS fVss; }; -// vim:ts=4 sw=4: diff --git a/primitives/primproc/batchprimitiveprocessor.cpp b/primitives/primproc/batchprimitiveprocessor.cpp index afc5e6daa..99224c7ac 100644 --- a/primitives/primproc/batchprimitiveprocessor.cpp +++ b/primitives/primproc/batchprimitiveprocessor.cpp @@ -117,6 +117,7 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor() , validCPData(false) , minVal(MAX64) , maxVal(MIN64) +, cpDataFromDictScan(false) , lbidForCP(0) , hasWideColumnOut(false) , busyLoaderCount(0) @@ -138,6 +139,7 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor() , processorThreads(0) , ptMask(0) , firstInstance(false) + , valuesLBID(0) { pp.setLogicalBlockMode(true); pp.setBlockPtr((int*)blockData); @@ -167,6 +169,7 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor(ByteStream& b, double prefetch, , validCPData(false) , minVal(MAX64) , maxVal(MIN64) + , cpDataFromDictScan(false) , lbidForCP(0) , hasWideColumnOut(false) , busyLoaderCount(0) @@ -186,10 +189,10 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor(ByteStream& b, double prefetch, , sockIndex(0) , endOfJoinerRan(false) , processorThreads(_processorThreads) - , // processorThreads(32), // ptMask(processorThreads - 1), - firstInstance(true) + , firstInstance(true) + , valuesLBID(0) { // promote processorThreads to next power of 2. also need to change the name to bucketCount or similar processorThreads = nextPowOf2(processorThreads); @@ -2075,6 +2078,7 @@ void BatchPrimitiveProcessor::writeProjectionPreamble() { *serialized << (uint8_t)1; *serialized << lbidForCP; + *serialized << ((uint8_t)cpDataFromDictScan); if (UNLIKELY(hasWideColumnOut)) { // PSA width @@ -2173,6 +2177,7 @@ void BatchPrimitiveProcessor::makeResponse() { *serialized << (uint8_t)1; *serialized << lbidForCP; + *serialized << ((uint8_t)cpDataFromDictScan); if (UNLIKELY(hasWideColumnOut)) { @@ -2274,6 +2279,7 @@ int BatchPrimitiveProcessor::operator()() } validCPData = false; + cpDataFromDictScan = false; #ifdef PRIMPROC_STOPWATCH stopwatch->start("BPP() execute"); execute(stopwatch); @@ -2812,4 +2818,3 @@ void BatchPrimitiveProcessor::buildVSSCache(uint32_t loopCount) } } // namespace primitiveprocessor -// vim:ts=4 sw=4: diff --git a/primitives/primproc/batchprimitiveprocessor.h b/primitives/primproc/batchprimitiveprocessor.h index 5e53ae8f5..7a18c39d5 100644 --- a/primitives/primproc/batchprimitiveprocessor.h +++ b/primitives/primproc/batchprimitiveprocessor.h @@ -257,6 +257,7 @@ class BatchPrimitiveProcessor int128_t max128Val; int64_t maxVal; }; + bool cpDataFromDictScan; uint64_t lbidForCP; bool hasWideColumnOut; @@ -432,6 +433,7 @@ class BatchPrimitiveProcessor uint processorThreads; uint ptMask; bool firstInstance; + uint64_t valuesLBID; static const uint64_t maxResultCount = 1048576; // 2^20 diff --git a/primitives/primproc/bpp.h b/primitives/primproc/bpp.h index 2190129bb..8d9161695 100644 --- a/primitives/primproc/bpp.h +++ b/primitives/primproc/bpp.h @@ -27,7 +27,6 @@ // Copyright: See COPYING file that comes with this distribution // // -#include "primitivemsg.h" #include "bytestream.h" #include "messagequeue.h" #include "serializeable.h" diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index 16426f49d..315d9293e 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -106,8 +106,8 @@ void ColumnCommand::execute() { values = bpp->values; wide128Values = bpp->wide128Values; + bpp->valuesLBID = lbid; } - _execute(); } @@ -225,9 +225,13 @@ void ColumnCommand::issuePrimitive() loadData(); if (!suppressFilter) + { bpp->getPrimitiveProcessor().setParsedColumnFilter(parsedColumnFilter); + } else + { bpp->getPrimitiveProcessor().setParsedColumnFilter(emptyFilter); + } switch (colType.colWidth) { @@ -282,6 +286,7 @@ void ColumnCommand::updateCPDataNarrow() if (_isScan) { bpp->validCPData = (outMsg->ValidMinMax && !wasVersioned); + bpp->cpDataFromDictScan = false; bpp->lbidForCP = lbid; bpp->maxVal = static_cast(outMsg->Max); bpp->minVal = static_cast(outMsg->Min); @@ -295,6 +300,7 @@ void ColumnCommand::updateCPDataWide() if (_isScan) { bpp->validCPData = (outMsg->ValidMinMax && !wasVersioned); + bpp->cpDataFromDictScan = false; bpp->lbidForCP = lbid; if (colType.isWideDecimalType()) { @@ -1226,4 +1232,3 @@ void ColumnCommandInt128::issuePrimitive() } } // namespace primitiveprocessor -// vim:ts=4 sw=4: diff --git a/primitives/primproc/columncommand.h b/primitives/primproc/columncommand.h index 57dda7142..518374ff0 100644 --- a/primitives/primproc/columncommand.h +++ b/primitives/primproc/columncommand.h @@ -306,4 +306,3 @@ inline void ColumnCommand::fillEmptyBlock(uint } // namespace primitiveprocessor -// vim:ts=4 sw=4: diff --git a/primitives/primproc/dictstep.cpp b/primitives/primproc/dictstep.cpp index 60adfa573..69fa86ccc 100644 --- a/primitives/primproc/dictstep.cpp +++ b/primitives/primproc/dictstep.cpp @@ -46,6 +46,8 @@ extern uint32_t dictBufferSize; DictStep::DictStep() : Command(DICT_STEP), strValues(NULL), filterCount(0), bufferSize(0) { + fMinMax[0] = MAX_UBIGINT; + fMinMax[1] = MIN_UBIGINT; } DictStep::~DictStep() @@ -65,6 +67,8 @@ DictStep& DictStep::operator=(const DictStep& d) eqOp = d.eqOp; filterCount = d.filterCount; charsetNumber = d.charsetNumber; + fMinMax[0] = d.fMinMax[0]; + fMinMax[1] = d.fMinMax[1]; return *this; } @@ -113,7 +117,6 @@ void DictStep::prep(int8_t outputType, bool makeAbsRids) primMsg->ism.Interleave = 0; primMsg->ism.Flags = 0; - // primMsg->ism.Flags = PrimitiveMsg::planFlagsToPrimFlags(traceFlags); primMsg->ism.Command = DICT_SIGNATURE; primMsg->ism.Size = bufferSize; primMsg->ism.Type = 2; @@ -147,8 +150,11 @@ void DictStep::issuePrimitive(bool isFilter) bpp->physIO += blocksRead; bpp->touchedBlocks++; } - +#if !defined(XXX_PRIMITIVES_TOKEN_RANGES_XXX) bpp->pp.p_Dictionary(primMsg, &result, isFilter, charsetNumber, eqFilter, eqOp); +#else + bpp->pp.p_Dictionary(primMsg, &result, isFilter, charsetNumber, eqFilter, eqOp, fMinMax); +#endif } void DictStep::copyResultToTmpSpace(OrderedToken* ot) @@ -390,6 +396,14 @@ void DictStep::_execute() copyResultToFinalPosition(newRidList.get()); copyRidsForFilterCmd(); } + if (fMinMax[0] <= fMinMax[1] && bpp->valuesLBID != 0) + { + bpp->validCPData = true; + bpp->cpDataFromDictScan = true; + bpp->lbidForCP = bpp->valuesLBID; + bpp->maxVal = fMinMax[1]; + bpp->minVal = fMinMax[0]; + } // cout << "DS: /_execute()\n"; } diff --git a/primitives/primproc/dictstep.h b/primitives/primproc/dictstep.h index 10af6c009..577abe6f3 100644 --- a/primitives/primproc/dictstep.h +++ b/primitives/primproc/dictstep.h @@ -158,6 +158,7 @@ class DictStep : public Command bool hasEqFilter; boost::shared_ptr eqFilter; uint8_t eqOp; // COMPARE_EQ or COMPARE_NE + uint64_t fMinMax[2]; friend class RTSCommand; }; diff --git a/exemgr/femsghandler.cpp b/primitives/primproc/femsghandler.cpp similarity index 100% rename from exemgr/femsghandler.cpp rename to primitives/primproc/femsghandler.cpp diff --git a/exemgr/femsghandler.h b/primitives/primproc/femsghandler.h similarity index 100% rename from exemgr/femsghandler.h rename to primitives/primproc/femsghandler.h diff --git a/primitives/primproc/primitiveserver.cpp b/primitives/primproc/primitiveserver.cpp index 99aa7a315..fa3ec5ebd 100644 --- a/primitives/primproc/primitiveserver.cpp +++ b/primitives/primproc/primitiveserver.cpp @@ -945,7 +945,6 @@ struct AsynchLoader QueryContext ver; uint32_t txn; int compType; - uint8_t dataWidth; bool LBIDTrace; uint32_t sessionID; uint32_t* cacheCount; @@ -1420,8 +1419,8 @@ struct BPPHandler /* Uncomment these lines to verify duplicate(). == op might need updating */ // if (*bpp != *dup) - // cerr << "createBPP: duplicate mismatch at index " << i << - // endl; + // cerr << "createBPP: duplicate mismatch at index " << i + // << endl; // idbassert(*bpp == *dup); bppv->add(dup); } @@ -2440,7 +2439,7 @@ PrimitiveServer::~PrimitiveServer() { } -void PrimitiveServer::start(Service* service) +void PrimitiveServer::start(Service* service, utils::USpaceSpinLock& startupRaceLock) { // start all the server threads for (int i = 1; i <= fServerThreads; i++) @@ -2451,7 +2450,7 @@ void PrimitiveServer::start(Service* service) fServerpool.invoke(ServerThread(oss.str(), this)); } - + startupRaceLock.release(); service->NotifyServiceStarted(); fServerpool.wait(); @@ -2575,4 +2574,3 @@ bool BPPV::aborted() // end workaround } // namespace primitiveprocessor -// vim:ts=4 sw=4: diff --git a/primitives/primproc/primitiveserver.h b/primitives/primproc/primitiveserver.h index d3b37fc0b..6dde07130 100644 --- a/primitives/primproc/primitiveserver.h +++ b/primitives/primproc/primitiveserver.h @@ -126,7 +126,7 @@ class PrimitiveServer /** @brief start the primitive server * */ - void start(Service* p); + void start(Service* p, utils::USpaceSpinLock& startupRaceLock); /** @brief get a pointer the shared processor thread pool */ diff --git a/primitives/primproc/primproc.cpp b/primitives/primproc/primproc.cpp index bc901803d..11f6650c8 100644 --- a/primitives/primproc/primproc.cpp +++ b/primitives/primproc/primproc.cpp @@ -1,5 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2016 MariaDB Corporation + Copyright (C) 2016-2022 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -30,22 +30,20 @@ #endif #include #include -#ifndef _MSC_VER #include #include -#else -#include -#endif + #include #include #include +#include +#include +#include //#define NDEBUG #include using namespace std; #include -#include -#include using namespace boost; #include "configcpp.h" @@ -74,8 +72,9 @@ using namespace idbdatafile; #include "mariadb_my_sys.h" +#include "spinlock.h" #include "service.h" -#include "threadnaming.h" +#include "serviceexemgr.h" class Opt { @@ -118,6 +117,14 @@ class ServicePrimProc : public Service, public Opt { return m_fg ? Child() : RunForking(); } + std::atomic_flag& getStartupRaceFlag() + { + return startupRaceFlag_; + } + + private: + // Since C++20 flag's init value is false. + std::atomic_flag startupRaceFlag_{false}; }; namespace primitiveprocessor @@ -155,28 +162,11 @@ int toInt(const string& val) void setupSignalHandlers() { #ifndef _MSC_VER - signal(SIGHUP, SIG_IGN); - struct sigaction ign; - - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = SIG_IGN; - sigaction(SIGPIPE, &ign, 0); - - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = SIG_IGN; - sigaction(SIGUSR1, &ign, 0); - memset(&ign, 0, sizeof(ign)); ign.sa_handler = SIG_IGN; sigaction(SIGUSR2, &ign, 0); - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = fatalHandler; - sigaction(SIGSEGV, &ign, 0); - sigaction(SIGABRT, &ign, 0); - sigaction(SIGFPE, &ign, 0); - sigset_t sigset; sigemptyset(&sigset); sigaddset(&sigset, SIGPIPE); @@ -387,6 +377,18 @@ int ServicePrimProc::Child() NotifyServiceInitializationFailed(); return 2; } + utils::USpaceSpinLock startupRaceLock(getStartupRaceFlag()); + std::thread exeMgrThread( + [this, cf]() + { + exemgr::Opt opt; + exemgr::globServiceExeMgr = new exemgr::ServiceExeMgr(opt, cf); + // primitive delay to avoid 'not connected to PM' log error messages + // from EM. PrimitiveServer::start() releases SpinLock after sockets + // are available. + utils::USpaceSpinLock startupRaceLock(this->getStartupRaceFlag()); + exemgr::globServiceExeMgr->Child(); + }); int serverThreads = 1; int serverQueueSize = 10; @@ -400,7 +402,6 @@ int ServicePrimProc::Child() bool rotatingDestination = false; uint32_t deleteBlocks = 128; bool PTTrace = false; - int temp; string strTemp; int priority = -1; const string primitiveServers("PrimitiveServers"); @@ -419,7 +420,7 @@ int ServicePrimProc::Child() gDebugLevel = primitiveprocessor::NONE; - temp = toInt(cf->getConfig(primitiveServers, "ServerThreads")); + int temp = toInt(cf->getConfig(primitiveServers, "ServerThreads")); if (temp > 0) serverThreads = temp; @@ -602,7 +603,6 @@ int ServicePrimProc::Child() if (temp >= 0) maxPct = temp; - // @bug4507, configurable pm aggregation AggregationMemoryCheck // We could use this same mechanism for other growing buffers. int aggPct = 95; temp = toInt(cf->getConfig("SystemConfig", "MemoryCheckPercent")); @@ -766,7 +766,7 @@ int ServicePrimProc::Child() } #endif - server.start(this); + server.start(this, startupRaceLock); cerr << "server.start() exited!" << endl; @@ -787,4 +787,3 @@ int main(int argc, char** argv) return ServicePrimProc(opt).Run(); } -// vim:ts=4 sw=4: diff --git a/primitives/primproc/rssmonfcn.cpp b/primitives/primproc/rssmonfcn.cpp new file mode 100644 index 000000000..8a9b62d83 --- /dev/null +++ b/primitives/primproc/rssmonfcn.cpp @@ -0,0 +1,69 @@ +/* Copyright (C) 2022 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include + +#include "rssmonfcn.h" +#include "serviceexemgr.h" + +namespace exemgr +{ + void RssMonFcn::operator()() const + { + logging::Logger& msgLog = globServiceExeMgr->getLogger(); + auto* statementsRunningCount = globServiceExeMgr->getStatementsRunningCount(); + for (;;) + { + size_t rssMb = rss(); + size_t pct = rssMb * 100 / fMemTotal; + + if (pct > fMaxPct) + { + if (fMaxPct >= 95) + { + std::cerr << "Too much memory allocated!" << std::endl; + logging::Message::Args args; + args.add((int)pct); + args.add((int)fMaxPct); + msgLog.logMessage(logging::LOG_TYPE_CRITICAL, ServiceExeMgr::logRssTooBig, args, logging::LoggingID(16)); + exit(1); + } + + if (statementsRunningCount->cur() == 0) + { + std::cerr << "Too much memory allocated!" << std::endl; + logging::Message::Args args; + args.add((int)pct); + args.add((int)fMaxPct); + msgLog.logMessage(logging::LOG_TYPE_WARNING, ServiceExeMgr::logRssTooBig, args, logging::LoggingID(16)); + exit(1); + } + + std::cerr << "Too much memory allocated, but stmts running" << std::endl; + } + + // Update sessionMemMap entries lower than current mem % use + globServiceExeMgr->updateSessionMap(pct); + + pause_(); + } + } + void startRssMon(size_t maxPct, int pauseSeconds) + { + new std::thread(RssMonFcn(maxPct, pauseSeconds)); + } +} // namespace \ No newline at end of file diff --git a/primitives/primproc/rssmonfcn.h b/primitives/primproc/rssmonfcn.h new file mode 100644 index 000000000..341038e91 --- /dev/null +++ b/primitives/primproc/rssmonfcn.h @@ -0,0 +1,36 @@ +/* Copyright (C) 2022 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#pragma once + +#include + +#include "MonitorProcMem.h" + +namespace exemgr +{ +class RssMonFcn : public utils::MonitorProcMem +{ + public: + RssMonFcn(size_t maxPct, int pauseSeconds) : MonitorProcMem(maxPct, 0, 21, pauseSeconds) + { + } + + void operator()() const; +}; + +} // namespace \ No newline at end of file diff --git a/primitives/primproc/serviceexemgr.cpp b/primitives/primproc/serviceexemgr.cpp new file mode 100644 index 000000000..8746b68f7 --- /dev/null +++ b/primitives/primproc/serviceexemgr.cpp @@ -0,0 +1,457 @@ +/* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2019-22 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +/********************************************************************** + * $Id: main.cpp 1000 2013-07-24 21:05:51Z pleblanc $ + * + * + ***********************************************************************/ +/** + * @brief execution plan manager main program + * + * This is the ?main? program for dealing with execution plans and + * result sets. It sits in a loop waiting for CalpontExecutionPlan + * from the FEP. It then passes the CalpontExecutionPlan to the + * JobListFactory from which a JobList is obtained. This is passed + * to to the Query Manager running in the real-time portion of the + * EC. The ExecutionPlanManager waits until the Query Manager + * returns a result set for the job list. These results are passed + * into the CalpontResultFactory, which outputs a CalpontResultSet. + * The ExecutionPlanManager passes the CalpontResultSet into the + * VendorResultFactory which produces a result set tailored to the + * specific DBMS front end in use. The ExecutionPlanManager then + * sends the VendorResultSet back to the Calpont Database Connector + * on the Front-End Processor where it is returned to the DBMS + * front-end. + */ +#include +#include +#include +#include + +#undef root_name +#include + +#include "calpontselectexecutionplan.h" +#include "mcsanalyzetableexecutionplan.h" +#include "activestatementcounter.h" +#include "distributedenginecomm.h" +#include "resourcemanager.h" +#include "configcpp.h" +#include "queryteleserverparms.h" +#include "iosocket.h" +#include "joblist.h" +#include "joblistfactory.h" +#include "oamcache.h" +#include "simplecolumn.h" +#include "bytestream.h" +#include "telestats.h" +#include "messageobj.h" +#include "messagelog.h" +#include "sqllogger.h" +#include "femsghandler.h" +#include "idberrorinfo.h" +#include "MonitorProcMem.h" +#include "liboamcpp.h" +#include "crashtrace.h" +#include "service.h" + +#include +#include +#include + +#include "dbrm.h" + +#include "mariadb_my_sys.h" +#include "statistics.h" +#include "serviceexemgr.h" +#include "sqlfrontsessionthread.h" + +namespace exemgr +{ +ServiceExeMgr* globServiceExeMgr = nullptr; +void startRssMon(size_t maxPct, int pauseSeconds); + +void added_a_pm(int) +{ + logging::LoggingID logid(21, 0, 0); + logging::Message::Args args1; + logging::Message msg(1); + args1.add("exeMgr caught SIGHUP. Resetting connections"); + msg.format(args1); + std::cout << msg.msg().c_str() << std::endl; + logging::Logger logger(logid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_DEBUG, msg, logid); + + auto* dec = exemgr::globServiceExeMgr->getDec(); + if (dec) + { + oam::OamCache* oamCache = oam::OamCache::makeOamCache(); + oamCache->forceReload(); + dec->Setup(); + } +} + +void printTotalUmMemory(int sig) +{ + int64_t num = globServiceExeMgr->getRm().availableMemory(); + std::cout << "Total UM memory available: " << num << std::endl; +} + +void ServiceExeMgr::setupSignalHandlers() +{ + struct sigaction ign; + + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = SIG_IGN; + + sigaction(SIGPIPE, &ign, 0); + + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = exemgr::added_a_pm; + sigaction(SIGHUP, &ign, 0); + ign.sa_handler = exemgr::printTotalUmMemory; + sigaction(SIGUSR1, &ign, 0); + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); +} + +void cleanTempDir() +{ + using TempDirPurpose = config::Config::TempDirPurpose; + struct Dirs + { + std::string section; + std::string allowed; + TempDirPurpose purpose; + }; + std::vector dirs{{"HashJoin", "AllowDiskBasedJoin", TempDirPurpose::Joins}, + {"RowAggregation", "AllowDiskBasedAggregation", TempDirPurpose::Aggregates}}; + const auto config = config::Config::makeConfig(); + + for (const auto& dir : dirs) + { + std::string allowStr = config->getConfig(dir.section, dir.allowed); + bool allow = (allowStr == "Y" || allowStr == "y"); + + std::string tmpPrefix = config->getTempFileDir(dir.purpose); + + if (allow && tmpPrefix.empty()) + { + std::cerr << "Empty tmp directory name for " << dir.section << std::endl; + logging::LoggingID logid(16, 0, 0); + logging::Message::Args args; + logging::Message message(8); + args.add("Empty tmp directory name for:"); + args.add(dir.section); + message.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_CRITICAL, message, logid); + } + + tmpPrefix += "/"; + + idbassert(tmpPrefix != "/"); + + /* This is quite scary as ExeMgr usually runs as root */ + try + { + if (allow) + { + boost::filesystem::remove_all(tmpPrefix); + } + boost::filesystem::create_directories(tmpPrefix); + } + catch (const std::exception& ex) + { + std::cerr << ex.what() << std::endl; + logging::LoggingID logid(16, 0, 0); + logging::Message::Args args; + logging::Message message(8); + args.add("Exception whilst cleaning tmpdir: "); + args.add(ex.what()); + message.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_WARNING, message, logid); + } + catch (...) + { + std::cerr << "Caught unknown exception during tmpdir cleanup" << std::endl; + logging::LoggingID logid(16, 0, 0); + logging::Message::Args args; + logging::Message message(8); + args.add("Unknown exception whilst cleaning tmpdir"); + message.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_WARNING, message, logid); + } + } +} + +const std::string ServiceExeMgr::prettyPrintMiniInfo(const std::string& in) +{ + // 1. take the std::string and tok it by '\n' + // 2. for each part in each line calc the longest part + // 3. padding to each longest value, output a header and the lines + using CharTraits = std::basic_string::traits_type; + using CharSeparator = boost::char_separator; + using Tokeniser = boost::tokenizer>; + CharSeparator sep1("\n"); + Tokeniser tok1(in, sep1); + std::vector lines; + std::string header = "Desc Mode Table TableOID ReferencedColumns PIO LIO PBE Elapsed Rows"; + const int header_parts = 10; + lines.push_back(header); + + for (auto iter1 = tok1.begin(); iter1 != tok1.end(); ++iter1) + { + if (!iter1->empty()) + lines.push_back(*iter1); + } + + std::vector lens; + + for (int i = 0; i < header_parts; i++) + lens.push_back(0); + + std::vector> lineparts; + std::vector::iterator iter2; + int j; + + for (iter2 = lines.begin(), j = 0; iter2 != lines.end(); ++iter2, j++) + { + CharSeparator sep2(" "); + Tokeniser tok2(*iter2, sep2); + int i; + std::vector parts; + Tokeniser::iterator iter3; + + for (iter3 = tok2.begin(), i = 0; iter3 != tok2.end(); ++iter3, i++) + { + if (i >= header_parts) + break; + + std::string part(*iter3); + + if (j != 0 && i == 8) + part.resize(part.size() - 3); + + assert(i < header_parts); + + if (part.size() > lens[i]) + lens[i] = part.size(); + + parts.push_back(part); + } + + assert(i == header_parts); + lineparts.push_back(parts); + } + + std::ostringstream oss; + + std::vector>::iterator iter1 = lineparts.begin(); + std::vector>::iterator end1 = lineparts.end(); + + oss << "\n"; + + while (iter1 != end1) + { + std::vector::iterator iter2 = iter1->begin(); + std::vector::iterator end2 = iter1->end(); + assert(distance(iter2, end2) == header_parts); + int i = 0; + + while (iter2 != end2) + { + assert(i < header_parts); + oss << std::setw(lens[i]) << std::left << *iter2 << " "; + ++iter2; + i++; + } + + oss << "\n"; + ++iter1; + } + + return oss.str(); +} + +int ServiceExeMgr::Child() +{ + // Make sure CSC thinks it's on a UM or else bucket reuse stuff below will stall + if (!m_e) + setenv("CALPONT_CSC_IDENT", "um", 1); + + setupSignalHandlers(); + int err = 0; + if (!m_debug) + err = setupResources(); + std::string errMsg; + + switch (err) + { + case -1: + case -3: errMsg = "Error getting file limits, please see non-root install documentation"; break; + + case -2: errMsg = "Error setting file limits, please see non-root install documentation"; break; + + case -4: + errMsg = "Could not install file limits to required value, please see non-root install documentation"; + break; + + default: errMsg = "Couldn't change working directory or unknown error"; break; + } + + cleanTempDir(); + + logging::MsgMap msgMap; + msgMap[logDefaultMsg] = logging::Message(logDefaultMsg); + msgMap[logDbProfStartStatement] = logging::Message(logDbProfStartStatement); + msgMap[logDbProfEndStatement] = logging::Message(logDbProfEndStatement); + msgMap[logStartSql] = logging::Message(logStartSql); + msgMap[logEndSql] = logging::Message(logEndSql); + msgMap[logRssTooBig] = logging::Message(logRssTooBig); + msgMap[logDbProfQueryStats] = logging::Message(logDbProfQueryStats); + msgMap[logExeMgrExcpt] = logging::Message(logExeMgrExcpt); + msgLog_.msgMap(msgMap); + + dec_ = joblist::DistributedEngineComm::instance(rm_, true); + dec_->Open(); + + bool tellUser = true; + + messageqcpp::MessageQueueServer* mqs; + + statementsRunningCount_ = new ActiveStatementCounter(rm_->getEmExecQueueSize()); + const std::string ExeMgr = "ExeMgr1"; + for (;;) + { + try + { + mqs = new messageqcpp::MessageQueueServer(ExeMgr, rm_->getConfig(), messageqcpp::ByteStream::BlockSize, + 64); + break; + } + catch (std::runtime_error& re) + { + std::string what = re.what(); + + if (what.find("Address already in use") != std::string::npos) + { + if (tellUser) + { + std::cerr << "Address already in use, retrying..." << std::endl; + tellUser = false; + } + + sleep(5); + } + else + { + throw; + } + } + } + + // class jobstepThreadPool is used by other processes. We can't call + // resourcemanaager (rm) functions during the static creation of threadpool + // because rm has a "isExeMgr" flag that is set upon creation (rm is a singleton). + // From the pools perspective, it has no idea if it is ExeMgr doing the + // creation, so it has no idea which way to set the flag. So we set the max here. + joblist::JobStep::jobstepThreadPool.setMaxThreads(rm_->getJLThreadPoolSize()); + joblist::JobStep::jobstepThreadPool.setName("ExeMgrJobList"); + + if (rm_->getJlThreadPoolDebug() == "Y" || rm_->getJlThreadPoolDebug() == "y") + { + joblist::JobStep::jobstepThreadPool.setDebug(true); + joblist::JobStep::jobstepThreadPool.invoke( + threadpool::ThreadPoolMonitor(&joblist::JobStep::jobstepThreadPool)); + } + + int serverThreads = rm_->getEmServerThreads(); + int maxPct = rm_->getEmMaxPct(); + int pauseSeconds = rm_->getEmSecondsBetweenMemChecks(); + int priority = rm_->getEmPriority(); + + FEMsgHandler::threadPool.setMaxThreads(serverThreads); + FEMsgHandler::threadPool.setName("FEMsgHandler"); + + if (maxPct > 0) + { + // Defined in rssmonfcn.cpp + exemgr::startRssMon(maxPct, pauseSeconds); + } + + setpriority(PRIO_PROCESS, 0, priority); + + std::string teleServerHost(rm_->getConfig()->getConfig("QueryTele", "Host")); + + if (!teleServerHost.empty()) + { + int teleServerPort = toInt(rm_->getConfig()->getConfig("QueryTele", "Port")); + + if (teleServerPort > 0) + { + teleServerParms_ = querytele::QueryTeleServerParms(teleServerHost, teleServerPort); + } + } + + std::cout << "Starting ExeMgr: st = " << serverThreads << ", qs = " << rm_->getEmExecQueueSize() + << ", mx = " << maxPct << ", cf = " << rm_->getConfig()->configFile() << std::endl; + + { + BRM::DBRM* dbrm = new BRM::DBRM(); + dbrm->setSystemQueryReady(true); + delete dbrm; + } + + threadpool::ThreadPool exeMgrThreadPool(serverThreads, 0); + exeMgrThreadPool.setName("ExeMgrServer"); + + if (rm_->getExeMgrThreadPoolDebug() == "Y" || rm_->getExeMgrThreadPoolDebug() == "y") + { + exeMgrThreadPool.setDebug(true); + exeMgrThreadPool.invoke(threadpool::ThreadPoolMonitor(&exeMgrThreadPool)); + } + + // Load statistics. + try + { + statistics::StatisticsManager::instance()->loadFromFile(); + } + catch (...) + { + std::cerr << "Cannot load statistics from file " << std::endl; + } + + for (;;) + { + messageqcpp::IOSocket ios; + ios = mqs->accept(); + exeMgrThreadPool.invoke(exemgr::SQLFrontSessionThread(ios, dec_, rm_)); + } + + exeMgrThreadPool.wait(); + + return 0; +} +} // namespace exemgr diff --git a/primitives/primproc/serviceexemgr.h b/primitives/primproc/serviceexemgr.h new file mode 100644 index 000000000..606ae48db --- /dev/null +++ b/primitives/primproc/serviceexemgr.h @@ -0,0 +1,348 @@ +/* Copyright (C) 2022 Mariadb Corporation. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#pragma once + +#include +#include +#include +#include + +#undef root_name +#include + +#include "calpontselectexecutionplan.h" +#include "mcsanalyzetableexecutionplan.h" +#include "activestatementcounter.h" +#include "distributedenginecomm.h" +#include "resourcemanager.h" +#include "configcpp.h" +#include "queryteleserverparms.h" +#include "iosocket.h" +#include "joblist.h" +#include "joblistfactory.h" +#include "oamcache.h" +#include "simplecolumn.h" +#include "bytestream.h" +#include "telestats.h" +#include "messageobj.h" +#include "messagelog.h" +#include "sqllogger.h" +#include "femsghandler.h" +#include "idberrorinfo.h" +#include "MonitorProcMem.h" +#include "liboamcpp.h" +#include "crashtrace.h" +#include "service.h" + +#include +#include +#include + +#include "dbrm.h" + +#include "mariadb_my_sys.h" +#include "statistics.h" + +namespace exemgr +{ + class Opt + { + public: + int m_debug; + bool m_e; + bool m_fg; + Opt() : m_debug(0), m_e(false), m_fg(false) {}; + Opt(int argc, char* argv[]) : m_debug(0), m_e(false), m_fg(false) + { + int c; + while ((c = getopt(argc, argv, "edf")) != EOF) + { + switch (c) + { + case 'd': m_debug++; break; + + case 'e': m_e = true; break; + + case 'f': m_fg = true; break; + + case '?': + default: break; + } + } + } + int getDebugLevel() const + { + return m_debug; + } + }; + + class ServiceExeMgr : public Service, public Opt + { + using SessionMemMap_t = std::map; + using ThreadCntPerSessionMap_t = std::map; + protected: + void log(logging::LOG_TYPE type, const std::string& str) + { + logging::LoggingID logid(16); + logging::Message::Args args; + logging::Message message(8); + args.add(strerror(errno)); + message.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(type, message, logid); + } + + public: + ServiceExeMgr(const Opt& opt) : Service("ExeMgr"), Opt(opt), msgLog_(logging::Logger(16)) + { + bool runningWithExeMgr = true; + rm_ = joblist::ResourceManager::instance(runningWithExeMgr); + } + ServiceExeMgr(const Opt& opt, config::Config* aConfig) : Service("ExeMgr"), Opt(opt), msgLog_(logging::Logger(16)) + { + bool runningWithExeMgr = true; + rm_ = joblist::ResourceManager::instance(runningWithExeMgr, aConfig); + } + void LogErrno() override + { + log(logging::LOG_TYPE_CRITICAL, std::string(strerror(errno))); + } + void ParentLogChildMessage(const std::string& str) override + { + log(logging::LOG_TYPE_INFO, str); + } + int Child() override; + int Run() + { + return m_fg ? Child() : RunForking(); + } + static const constexpr unsigned logDefaultMsg = logging::M0000; + static const constexpr unsigned logDbProfStartStatement = logging::M0028; + static const constexpr unsigned logDbProfEndStatement = logging::M0029; + static const constexpr unsigned logStartSql = logging::M0041; + static const constexpr unsigned logEndSql = logging::M0042; + static const constexpr unsigned logRssTooBig = logging::M0044; + static const constexpr unsigned logDbProfQueryStats = logging::M0047; + static const constexpr unsigned logExeMgrExcpt = logging::M0055; + // If any flags other than the table mode flags are set, produce output to screeen + static const constexpr uint32_t flagsWantOutput = (0xffffffff & ~execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_AUTOSWITCH & + ~execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF); + logging::Logger& getLogger() + { + return msgLog_; + } + void updateSessionMap(const size_t pct) + { + std::lock_guard lk(sessionMemMapMutex_); + + for (auto mapIter = sessionMemMap_.begin(); mapIter != sessionMemMap_.end(); ++mapIter) + { + if (pct > mapIter->second) + { + mapIter->second = pct; + } + } + } + ThreadCntPerSessionMap_t& getThreadCntPerSessionMap() + { + return threadCntPerSessionMap_; + } + std::mutex& getThreadCntPerSessionMapMutex() + { + return threadCntPerSessionMapMutex_; + } + void initMaxMemPct(uint32_t sessionId) + { + // WIP + if (sessionId < 0x80000000) + { + std::lock_guard lk(sessionMemMapMutex_); + auto mapIter = sessionMemMap_.find(sessionId); + + if (mapIter == sessionMemMap_.end()) + { + sessionMemMap_[sessionId] = 0; + } + else + { + mapIter->second = 0; + } + } + } + uint64_t getMaxMemPct(const uint32_t sessionId) + { + uint64_t maxMemoryPct = 0; + // WIP + if (sessionId < 0x80000000) + { + std::lock_guard lk(sessionMemMapMutex_); + auto mapIter = sessionMemMap_.find(sessionId); + + if (mapIter != sessionMemMap_.end()) + { + maxMemoryPct = (uint64_t)mapIter->second; + } + } + + return maxMemoryPct; + } + void deleteMaxMemPct(uint32_t sessionId) + { + if (sessionId < 0x80000000) + { + std::lock_guard lk(sessionMemMapMutex_); + auto mapIter = sessionMemMap_.find(sessionId); + + if (mapIter != sessionMemMap_.end()) + { + sessionMemMap_.erase(sessionId); + } + } + } + //...Increment the number of threads using the specified sessionId + void incThreadCntPerSession(uint32_t sessionId) + { + std::lock_guard lk(threadCntPerSessionMapMutex_); + auto mapIter = threadCntPerSessionMap_.find(sessionId); + + if (mapIter == threadCntPerSessionMap_.end()) + threadCntPerSessionMap_.insert(ThreadCntPerSessionMap_t::value_type(sessionId, 1)); + else + mapIter->second++; + } + //...Decrement the number of threads using the specified sessionId. + //...When the thread count for a sessionId reaches 0, the corresponding + //...CalpontSystemCatalog objects are deleted. + //...The user query and its associated catalog query have a different + //...session Id where the highest bit is flipped. + //...The object with id(sessionId | 0x80000000) cannot be removed before + //...user query session completes because the sysdata may be used for + //...debugging/stats purpose, such as result graph, etc. + void decThreadCntPerSession(uint32_t sessionId) + { + std::lock_guard lk(threadCntPerSessionMapMutex_); + auto mapIter = threadCntPerSessionMap_.find(sessionId); + + if (mapIter != threadCntPerSessionMap_.end()) + { + if (--mapIter->second == 0) + { + threadCntPerSessionMap_.erase(mapIter); + execplan::CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId); + execplan::CalpontSystemCatalog::removeCalpontSystemCatalog((sessionId ^ 0x80000000)); + } + } + } + ActiveStatementCounter* getStatementsRunningCount() + { + return statementsRunningCount_; + } + joblist::DistributedEngineComm* getDec() + { + return dec_; + } + int toInt(const std::string& val) + { + if (val.length() == 0) + return -1; + + return static_cast(config::Config::fromText(val)); + } + const std::string prettyPrintMiniInfo(const std::string& in); + + const std::string timeNow() + { + time_t outputTime = time(0); + struct tm ltm; + char buf[32]; // ctime(3) says at least 26 + size_t len = 0; + asctime_r(localtime_r(&outputTime, <m), buf); + len = strlen(buf); + + if (len > 0) + --len; + + if (buf[len] == '\n') + buf[len] = 0; + + return buf; + } + querytele::QueryTeleServerParms& getTeleServerParms() + { + return teleServerParms_; + } + joblist::ResourceManager& getRm() + { + return *rm_; + } + private: + void setupSignalHandlers(); + int8_t setupCwd() + { + std::string workdir = rm_->getScWorkingDir(); + int8_t rc = chdir(workdir.c_str()); + + if (rc < 0 || access(".", W_OK) != 0) + rc = chdir("/tmp"); + + return (rc < 0) ? -5 : rc; + } + int setupResources() + { + struct rlimit rlim; + + if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) + { + return -1; + } + rlim.rlim_cur = rlim.rlim_max = 65536; + + if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) + { + return -2; + } + if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) + { + return -3; + } + if (rlim.rlim_cur != 65536) + { + return -4; + } + return 0; + } + + logging::Logger msgLog_; + SessionMemMap_t sessionMemMap_; // track memory% usage during a query + std::mutex sessionMemMapMutex_; + //...The FrontEnd may establish more than 1 connection (which results in + // more than 1 ExeMgr thread) per session. These threads will share + // the same CalpontSystemCatalog object for that session. Here, we + // define a std::map to track how many threads are sharing each session, so + // that we know when we can safely delete a CalpontSystemCatalog object + // shared by multiple threads per session. + ThreadCntPerSessionMap_t threadCntPerSessionMap_; + std::mutex threadCntPerSessionMapMutex_; + ActiveStatementCounter* statementsRunningCount_; + joblist::DistributedEngineComm* dec_; + joblist::ResourceManager* rm_; + // Its attributes are set in Child() + querytele::QueryTeleServerParms teleServerParms_; + }; + extern ServiceExeMgr* globServiceExeMgr; +} \ No newline at end of file diff --git a/primitives/primproc/sqlfrontsessionthread.cpp b/primitives/primproc/sqlfrontsessionthread.cpp new file mode 100644 index 000000000..3b5094e7a --- /dev/null +++ b/primitives/primproc/sqlfrontsessionthread.cpp @@ -0,0 +1,985 @@ +/* Copyright (C) 2022 Mariadb Corporation. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include "sqlfrontsessionthread.h" + +namespace exemgr +{ + uint64_t SQLFrontSessionThread::getMaxMemPct(uint32_t sessionId) + { + return globServiceExeMgr->getMaxMemPct(sessionId); + } + void SQLFrontSessionThread::deleteMaxMemPct(uint32_t sessionId) + { + return globServiceExeMgr->deleteMaxMemPct(sessionId); + } + void SQLFrontSessionThread::incThreadCntPerSession(uint32_t sessionId) + { + return globServiceExeMgr->incThreadCntPerSession(sessionId); + } + void SQLFrontSessionThread::decThreadCntPerSession(uint32_t sessionId) + { + return globServiceExeMgr->decThreadCntPerSession(sessionId); + } + + void SQLFrontSessionThread::initMaxMemPct(uint32_t sessionId) + { + return globServiceExeMgr->initMaxMemPct(sessionId); + } + //...Get and log query stats to specified output stream + const std::string SQLFrontSessionThread::formatQueryStats( + joblist::SJLP& jl, // joblist associated with query + const std::string& label, // header label to print in front of log output + bool includeNewLine, // include line breaks in query stats std::string + bool vtableModeOn, bool wantExtendedStats, uint64_t rowsReturned) + { + std::ostringstream os; + + // Get stats if not already acquired for current query + if (!fStatsRetrieved) + { + if (wantExtendedStats) + { + // wait for the ei data to be written by another thread (brain-dead) + struct timespec req = {0, 250000}; // 250 usec + nanosleep(&req, 0); + } + + // Get % memory usage during current query for sessionId + jl->querySummary(wantExtendedStats); + fStats = jl->queryStats(); + fStats.fMaxMemPct = getMaxMemPct(fStats.fSessionID); + fStats.fRows = rowsReturned; + fStatsRetrieved = true; + } + + std::string queryMode; + queryMode = (vtableModeOn ? "Distributed" : "Standard"); + + // Log stats to specified output stream + os << label << ": MaxMemPct-" << fStats.fMaxMemPct << "; NumTempFiles-" << fStats.fNumFiles + << "; TempFileSpace-" << roundBytes(fStats.fFileBytes) << "; ApproxPhyI/O-" << fStats.fPhyIO + << "; CacheI/O-" << fStats.fCacheIO << "; BlocksTouched-" << fStats.fMsgRcvCnt; + + if (includeNewLine) + os << std::endl << " "; // insert line break + else + os << "; "; // continue without line break + + os << "PartitionBlocksEliminated-" << fStats.fCPBlocksSkipped << "; MsgBytesIn-" + << roundBytes(fStats.fMsgBytesIn) << "; MsgBytesOut-" << roundBytes(fStats.fMsgBytesOut) << "; Mode-" + << queryMode; + + return os.str(); + } + + //... Round off to human readable format (KB, MB, or GB). + const std::string SQLFrontSessionThread::roundBytes(uint64_t value) const + { + const char* units[] = {"B", "KB", "MB", "GB", "TB"}; + uint64_t i = 0, up = 0; + uint64_t roundedValue = value; + + while (roundedValue > 1024 && i < 4) + { + up = (roundedValue & 512); + roundedValue /= 1024; + i++; + } + + if (up) + roundedValue++; + + std::ostringstream oss; + oss << roundedValue << units[i]; + return oss.str(); + } + + //...Round off to nearest (1024*1024) MB + uint64_t SQLFrontSessionThread::roundMB(uint64_t value) const + { + uint64_t roundedValue = value >> 20; + + if (value & 524288) + roundedValue++; + + return roundedValue; + } + + void SQLFrontSessionThread::setRMParms(const execplan::CalpontSelectExecutionPlan::RMParmVec& parms) + { + for (execplan::CalpontSelectExecutionPlan::RMParmVec::const_iterator it = parms.begin(); + it != parms.end(); ++it) + { + switch (it->id) + { + case execplan::PMSMALLSIDEMEMORY: + { + globServiceExeMgr->getRm().addHJPmMaxSmallSideMap(it->sessionId, it->value); + break; + } + + case execplan::UMSMALLSIDEMEMORY: + { + globServiceExeMgr->getRm().addHJUmMaxSmallSideMap(it->sessionId, it->value); + break; + } + + default:; + } + } + } + + void SQLFrontSessionThread::buildSysCache(const execplan::CalpontSelectExecutionPlan& csep, + boost::shared_ptr csc) + { + const execplan::CalpontSelectExecutionPlan::ColumnMap& colMap = csep.columnMap(); + std::string schemaName; + + for (auto it = colMap.begin(); it != colMap.end(); ++it) + { + const auto sc = dynamic_cast((it->second).get()); + + if (sc) + { + schemaName = sc->schemaName(); + + // only the first time a schema is got will actually query + // system catalog. System catalog keeps a schema name std::map. + // if a schema exists, the call getSchemaInfo returns without + // doing anything. + if (!schemaName.empty()) + csc->getSchemaInfo(schemaName); + } + } + + execplan::CalpontSelectExecutionPlan::SelectList::const_iterator subIt; + + for (subIt = csep.derivedTableList().begin(); subIt != csep.derivedTableList().end(); ++subIt) + { + buildSysCache(*(dynamic_cast(subIt->get())), csc); + } + } + + void SQLFrontSessionThread::writeCodeAndError(messageqcpp::ByteStream::quadbyte code, const std::string emsg) + { + messageqcpp::ByteStream emsgBs; + messageqcpp::ByteStream tbs; + tbs << code; + fIos.write(tbs); + emsgBs << emsg; + fIos.write(emsgBs); + } + + void SQLFrontSessionThread::analyzeTableExecute(messageqcpp::ByteStream& bs, joblist::SJLP& jl, bool& stmtCounted) + { + auto* statementsRunningCount = globServiceExeMgr->getStatementsRunningCount(); + messageqcpp::ByteStream::quadbyte qb; + execplan::MCSAnalyzeTableExecutionPlan caep; + + bs = fIos.read(); + caep.unserialize(bs); + + statementsRunningCount->incr(stmtCounted); + jl = joblist::JobListFactory::makeJobList(&caep, fRm, false, true); + + // Joblist is empty. + if (jl->status() == logging::statisticsJobListEmpty) + { + if (caep.traceOn()) + std::cout << "JobList is empty " << std::endl; + + jl.reset(); + bs.restart(); + qb = ANALYZE_TABLE_SUCCESS; + bs << qb; + fIos.write(bs); + bs.reset(); + statementsRunningCount->decr(stmtCounted); + return; + } + + if (UNLIKELY(fEc->getNumConnections() != fEc->connectedPmServers())) + { + std::cout << "fEc setup " << std::endl; + fEc->Setup(); + } + + if (jl->status() == 0) + { + std::string emsg; + + if (jl->putEngineComm(fEc) != 0) + throw std::runtime_error(jl->errMsg()); + } + else + { + throw std::runtime_error("ExeMgr: could not build a JobList!"); + } + + // Execute a joblist. + jl->doQuery(); + + FEMsgHandler msgHandler(jl, &fIos); + + msgHandler.start(); + auto rowCount = jl->projectTable(100, bs); + msgHandler.stop(); + + auto outRG = (static_cast(jl.get()))->getOutputRowGroup(); + + if (caep.traceOn()) + std::cout << "Row count " << rowCount << std::endl; + + // Process `RowGroup`, increase an epoch and save statistics to the file. + auto* statisticsManager = statistics::StatisticsManager::instance(); + statisticsManager->analyzeColumnKeyTypes(outRG, caep.traceOn()); + statisticsManager->incEpoch(); + statisticsManager->saveToFile(); + + // Distribute statistics across all ExeMgr clients if possible. + statistics::StatisticsDistributor::instance()->distributeStatistics(); + + // Send the signal back to front-end. + bs.restart(); + qb = ANALYZE_TABLE_SUCCESS; + bs << qb; + fIos.write(bs); + bs.reset(); + statementsRunningCount->decr(stmtCounted); + } + + void SQLFrontSessionThread::analyzeTableHandleStats(messageqcpp::ByteStream& bs) + { + messageqcpp::ByteStream::quadbyte qb; +#ifdef DEBUG_STATISTICS + std::cout << "Get distributed statistics on ExeMgr(Client) from ExeMgr(Server) " << std::endl; +#endif + bs = fIos.read(); +#ifdef DEBUG_STATISTICS + std::cout << "Read the hash from statistics on ExeMgr(Client) from ExeMgr(Server) " << std::endl; +#endif + uint64_t dataHashRec; + bs >> dataHashRec; + + uint64_t dataHash = statistics::StatisticsManager::instance()->computeHashFromStats(); + // The stats are the same. + if (dataHash == dataHashRec) + { +#ifdef DEBUG_STATISTICS + std::cout << "The hash is the same as rec hash on ExeMgr(Client) from ExeMgr(Server) " << std::endl; +#endif + qb = ANALYZE_TABLE_SUCCESS; + bs << qb; + fIos.write(bs); + bs.reset(); + return; + } + + bs.restart(); + qb = ANALYZE_TABLE_NEED_STATS; + bs << qb; + fIos.write(bs); + + bs.restart(); + bs = fIos.read(); +#ifdef DEBUG_STATISTICS + std::cout << "Read statistics on ExeMgr(Client) from ExeMgr(Server) " << std::endl; +#endif + statistics::StatisticsManager::instance()->unserialize(bs); + statistics::StatisticsManager::instance()->saveToFile(); + +#ifdef DEBUG_STATISTICS + std::cout << "Write flag on ExeMgr(Client) to ExeMgr(Server)" << std::endl; +#endif + qb = ANALYZE_TABLE_SUCCESS; + bs << qb; + fIos.write(bs); + bs.reset(); + } + + void SQLFrontSessionThread::operator()() + { + messageqcpp::ByteStream bs, inbs; + execplan::CalpontSelectExecutionPlan csep; + csep.sessionID(0); + joblist::SJLP jl; + bool incSQLFrontSessionThreadCnt = true; + std::mutex jlMutex; + std::condition_variable jlCleanupDone; + int destructing = 0; + int gDebug = globServiceExeMgr->getDebugLevel(); + logging::Logger& msgLog = globServiceExeMgr->getLogger(); + + bool selfJoin = false; + bool tryTuples = false; + bool usingTuples = false; + bool stmtCounted = false; + auto* statementsRunningCount = globServiceExeMgr->getStatementsRunningCount(); + + try + { + for (;;) + { + selfJoin = false; + tryTuples = false; + usingTuples = false; + + if (jl) + { + // puts the real destruction in another thread to avoid + // making the whole session wait. It can take several seconds. + std::unique_lock scoped(jlMutex); + destructing++; + std::thread bgdtor( + [jl, &jlMutex, &jlCleanupDone, &destructing] + { + std::unique_lock scoped(jlMutex); + const_cast(jl).reset(); // this happens second; does real destruction + if (--destructing == 0) + jlCleanupDone.notify_one(); + }); + jl.reset(); // this runs first + bgdtor.detach(); + } + + bs = fIos.read(); + + if (bs.length() == 0) + { + if (gDebug > 1 || (gDebug && !csep.isInternal())) + std::cout << "### Got a close(1) for session id " << csep.sessionID() << std::endl; + + // connection closed by client + fIos.close(); + break; + } + else if (bs.length() < 4) // Not a CalpontSelectExecutionPlan + { + if (gDebug) + std::cout << "### Got a not-a-plan for session id " << csep.sessionID() << " with length " + << bs.length() << std::endl; + + fIos.close(); + break; + } + else if (bs.length() == 4) // possible tuple flag + { + messageqcpp::ByteStream::quadbyte qb; + bs >> qb; + + if (qb == 4) // UM wants new tuple i/f + { + if (gDebug) + std::cout << "### UM wants tuples" << std::endl; + + tryTuples = true; + // now wait for the CSEP... + bs = fIos.read(); + } + else if (qb == 5) // somebody wants stats + { + bs.restart(); + qb = statementsRunningCount->cur(); + bs << qb; + qb = statementsRunningCount->waiting(); + bs << qb; + fIos.write(bs); + fIos.close(); + break; + } + else if (qb == ANALYZE_TABLE_EXECUTE) + { + analyzeTableExecute(bs, jl, stmtCounted); + continue; + } + else if (qb == ANALYZE_TABLE_REC_STATS) + { + analyzeTableHandleStats(bs); + continue; + } + else + { + if (gDebug) + std::cout << "### Got a not-a-plan value " << qb << std::endl; + + fIos.close(); + break; + } + } + + new_plan: + try + { + csep.unserialize(bs); + } + catch (logging::IDBExcept& ex) + { + // We can get here on illegal function parameter data type, e.g. + // SELECT blob_column|1 FROM t1; + statementsRunningCount->decr(stmtCounted); + writeCodeAndError(ex.errorCode(), std::string(ex.what())); + continue; + } + + querytele::QueryTeleStats qts; + + if (!csep.isInternal() && (csep.queryType() == "SELECT" || csep.queryType() == "INSERT_SELECT")) + { + qts.query_uuid = csep.uuid(); + qts.msg_type = querytele::QueryTeleStats::QT_START; + qts.start_time = querytele::QueryTeleClient::timeNowms(); + qts.query = csep.data(); + qts.session_id = csep.sessionID(); + qts.query_type = csep.queryType(); + qts.system_name = fOamCachePtr->getSystemName(); + qts.module_name = fOamCachePtr->getModuleName(); + qts.local_query = csep.localQuery(); + qts.schema_name = csep.schemaName(); + fTeleClient.postQueryTele(qts); + } + + if (gDebug > 1 || (gDebug && !csep.isInternal())) + std::cout << "### For session id " << csep.sessionID() << ", got a CSEP" << std::endl; + + setRMParms(csep.rmParms()); + // Re-establish lost PP connections. + if (UNLIKELY(fEc->getNumConnections() != fEc->connectedPmServers())) + { + fEc->Setup(); + } + // @bug 1021. try to get schema cache for a come in query. + // skip system catalog queries. + if (!csep.isInternal()) + { + boost::shared_ptr csc = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(csep.sessionID()); + buildSysCache(csep, csc); + } + + // As soon as we have a session id for this thread, update the + // thread count per session; only do this once per thread. + // Mask 0x80000000 is for associate user query and csc query + if (incSQLFrontSessionThreadCnt) + { + // WIP + incThreadCntPerSession(csep.sessionID() | 0x80000000); + incSQLFrontSessionThreadCnt = false; + } + + bool needDbProfEndStatementMsg = false; + logging::Message::Args args; + std::string sqlText = csep.data(); + logging::LoggingID li(16, csep.sessionID(), csep.txnID()); + + // Initialize stats for this query, including + // init sessionMemMap entry for this session to 0 memory %. + // We will need this later for traceOn() or if we receive a + // table request with qb=3 (see below). This is also recorded + // as query start time. + initStats(csep.sessionID(), sqlText); + fStats.fQueryType = csep.queryType(); + + // Log start and end statement if tracing is enabled. Keep in + // mind the trace flag won't be set for system catalog queries. + if (csep.traceOn()) + { + args.reset(); + args.add((int)csep.statementID()); + args.add((int)csep.verID().currentScn); + args.add(sqlText); + msgLog.logMessage(logging::LOG_TYPE_DEBUG, ServiceExeMgr::logDbProfStartStatement, args, li); + needDbProfEndStatementMsg = true; + } + + // Don't log subsequent self joins after first. + if (selfJoin) + sqlText = ""; + + std::ostringstream oss; + oss << sqlText << "; |" << csep.schemaName() << "|"; + logging::SQLLogger sqlLog(oss.str(), li); + + statementsRunningCount->incr(stmtCounted); + + if (tryTuples) + { + try // @bug2244: try/catch around fIos.write() calls responding to makeTupleList + { + jl = joblist::JobListFactory::makeJobList(&csep, fRm, true, true); + // assign query stats + jl->queryStats(fStats); + + if ((jl->status()) == 0 && (jl->putEngineComm(fEc) == 0)) + { + usingTuples = true; + + // Tell the FE that we're sending tuples back, not TableBands + writeCodeAndError(0, "NOERROR"); + auto tjlp = dynamic_cast(jl.get()); + assert(tjlp); + messageqcpp::ByteStream tbs; + tbs << tjlp->getOutputRowGroup(); + fIos.write(tbs); + } + else + { + const std::string emsg = jl->errMsg(); + statementsRunningCount->decr(stmtCounted); + writeCodeAndError(jl->status(), emsg); + std::cerr << "ExeMgr: could not build a tuple joblist: " << emsg << std::endl; + continue; + } + } + catch (std::exception& ex) + { + std::ostringstream errMsg; + errMsg << "ExeMgr: error writing makeJoblist " + "response; " + << ex.what(); + throw std::runtime_error(errMsg.str()); + } + catch (...) + { + std::ostringstream errMsg; + errMsg << "ExeMgr: unknown error writing makeJoblist " + "response; "; + throw std::runtime_error(errMsg.str()); + } + + if (!usingTuples) + { + if (gDebug) + std::cout << "### UM wanted tuples but it didn't work out :-(" << std::endl; + } + else + { + if (gDebug) + std::cout << "### UM wanted tuples and we'll do our best;-)" << std::endl; + } + } + else + { + usingTuples = false; + jl = joblist::JobListFactory::makeJobList(&csep, fRm, false, true); + + if (jl->status() == 0) + { + std::string emsg; + + if (jl->putEngineComm(fEc) != 0) + throw std::runtime_error(jl->errMsg()); + } + else + { + throw std::runtime_error("ExeMgr: could not build a JobList!"); + } + } + + jl->doQuery(); + + execplan::CalpontSystemCatalog::OID tableOID; + bool swallowRows = false; + joblist::DeliveredTableMap tm; + uint64_t totalBytesSent = 0; + uint64_t totalRowCount = 0; + + // Project each table as the FE asks for it + for (;;) + { + bs = fIos.read(); + + if (bs.length() == 0) + { + if (gDebug > 1 || (gDebug && !csep.isInternal())) + std::cout << "### Got a close(2) for session id " << csep.sessionID() << std::endl; + + break; + } + + if (gDebug && bs.length() > 4) + std::cout << "### For session id " << csep.sessionID() << ", got too many bytes = " << bs.length() + << std::endl; + + // TODO: Holy crud! Can this be right? + //@bug 1444 Yes, if there is a self-join + if (bs.length() > 4) + { + selfJoin = true; + statementsRunningCount->decr(stmtCounted); + goto new_plan; + } + + assert(bs.length() == 4); + + messageqcpp::ByteStream::quadbyte qb; + + try // @bug2244: try/catch around fIos.write() calls responding to qb command + { + bs >> qb; + + if (gDebug > 1 || (gDebug && !csep.isInternal())) + std::cout << "### For session id " << csep.sessionID() << ", got a command = " << qb + << std::endl; + + if (qb == 0) + { + // No more tables, query is done + break; + } + else if (qb == 1) + { + // super-secret flag indicating that the UM is going to scarf down all the rows in the + // query. + swallowRows = true; + tm = jl->deliveredTables(); + continue; + } + else if (qb == 2) + { + // UM just wants any table + assert(swallowRows); + auto iter = tm.begin(); + + if (iter == tm.end()) + { + if (gDebug > 1 || (gDebug && !csep.isInternal())) + std::cout << "### For session id " << csep.sessionID() << ", returning end flag" + << std::endl; + + bs.restart(); + bs << (messageqcpp::ByteStream::byte)1; + fIos.write(bs); + continue; + } + + tableOID = iter->first; + } + else if (qb == 3) // special option-UM wants job stats std::string + { + std::string statsString; + + // Log stats std::string to be sent back to front end + statsString = formatQueryStats( + jl, "Query Stats", false, + !(csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF), + (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG), totalRowCount); + + bs.restart(); + bs << statsString; + + if ((csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG) != 0) + { + bs << jl->extendedInfo(); + bs << globServiceExeMgr->prettyPrintMiniInfo(jl->miniInfo()); + } + else + { + std::string empty; + bs << empty; + bs << empty; + } + + // send stats to connector for inserting to the querystats table + fStats.serialize(bs); + fIos.write(bs); + continue; + } + // for table mode handling + else if (qb == 4) + { + statementsRunningCount->decr(stmtCounted); + bs = fIos.read(); + goto new_plan; + } + else // (qb > 3) + { + // Return table bands for the requested tableOID + tableOID = static_cast(qb); + } + } + catch (std::exception& ex) + { + std::ostringstream errMsg; + errMsg << "ExeMgr: error writing qb response " + "for qb cmd " + << qb << "; " << ex.what(); + throw std::runtime_error(errMsg.str()); + } + catch (...) + { + std::ostringstream errMsg; + errMsg << "ExeMgr: unknown error writing qb response " + "for qb cmd " + << qb; + throw std::runtime_error(errMsg.str()); + } + + if (swallowRows) + tm.erase(tableOID); + + FEMsgHandler msgHandler(jl, &fIos); + + if (tableOID == 100) + msgHandler.start(); + + //...Loop serializing table bands projected for the tableOID + for (;;) + { + uint32_t rowCount; + + rowCount = jl->projectTable(tableOID, bs); + + msgHandler.stop(); + + if (jl->status()) + { + const auto errInfo = logging::IDBErrorInfo::instance(); + + if (jl->errMsg().length() != 0) + bs << jl->errMsg(); + else + bs << errInfo->errorMsg(jl->status()); + } + + try // @bug2244: try/catch around fIos.write() calls projecting rows + { + if (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS3) + { + // Skip the write to the front end until the last empty band. Used to time queries + // through without any front end waiting. + if (tableOID < 3000 || rowCount == 0) + fIos.write(bs); + } + else + { + fIos.write(bs); + } + } + catch (std::exception& ex) + { + msgHandler.stop(); + std::ostringstream errMsg; + errMsg << "ExeMgr: error projecting rows " + "for tableOID: " + << tableOID << "; rowCnt: " << rowCount << "; prevTotRowCnt: " << totalRowCount << "; " + << ex.what(); + jl->abort(); + + while (rowCount) + rowCount = jl->projectTable(tableOID, bs); + + if (tableOID == 100 && msgHandler.aborted()) + { + /* TODO: modularize the cleanup code, as well as + * the rest of this fcn */ + + decThreadCntPerSession(csep.sessionID() | 0x80000000); + statementsRunningCount->decr(stmtCounted); + fIos.close(); + return; + } + + // std::cout << "connection drop\n"; + throw std::runtime_error(errMsg.str()); + } + catch (...) + { + std::ostringstream errMsg; + msgHandler.stop(); + errMsg << "ExeMgr: unknown error projecting rows " + "for tableOID: " + << tableOID << "; rowCnt: " << rowCount << "; prevTotRowCnt: " << totalRowCount; + jl->abort(); + + while (rowCount) + rowCount = jl->projectTable(tableOID, bs); + + throw std::runtime_error(errMsg.str()); + } + + totalRowCount += rowCount; + totalBytesSent += bs.length(); + + if (rowCount == 0) + { + msgHandler.stop(); + // No more bands, table is done + bs.reset(); + + // @bug 2083 decr active statement count here for table mode. + if (!usingTuples) + statementsRunningCount->decr(stmtCounted); + + break; + } + else + { + bs.restart(); + } + } // End of loop to project and serialize table bands for a table + } // End of loop to process tables + + // @bug 828 + if (csep.traceOn()) + jl->graph(csep.sessionID()); + + if (needDbProfEndStatementMsg) + { + std::string ss; + std::ostringstream prefix; + prefix << "ses:" << csep.sessionID() << " Query Totals"; + + // Log stats std::string to standard out + ss = formatQueryStats(jl, prefix.str(), true, + !(csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF), + (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG), + totalRowCount); + //@Bug 1306. Added timing info for real time tracking. + std::cout << ss << " at " << globServiceExeMgr->timeNow() << std::endl; + + // log query stats to debug log file + args.reset(); + args.add((int)csep.statementID()); + args.add(fStats.fMaxMemPct); + args.add(fStats.fNumFiles); + args.add(fStats.fFileBytes); // log raw byte count instead of MB + args.add(fStats.fPhyIO); + args.add(fStats.fCacheIO); + args.add(fStats.fMsgRcvCnt); + args.add(fStats.fMsgBytesIn); + args.add(fStats.fMsgBytesOut); + args.add(fStats.fCPBlocksSkipped); + msgLog.logMessage(logging::LOG_TYPE_DEBUG, ServiceExeMgr::logDbProfQueryStats, args, li); + //@bug 1327 + deleteMaxMemPct(csep.sessionID()); + // Calling reset here, will cause joblist destructor to be + // called, which "joins" the threads. We need to do that + // here to make sure all syslogging from all the threads + // are complete; and that our logDbProfEndStatement will + // appear "last" in the syslog for this SQL statement. + // puts the real destruction in another thread to avoid + // making the whole session wait. It can take several seconds. + int stmtID = csep.statementID(); + std::unique_lock scoped(jlMutex); + destructing++; + std::thread bgdtor( + [jl, &jlMutex, &jlCleanupDone, stmtID, &li, &destructing, &msgLog] + { + std::unique_lock scoped(jlMutex); + const_cast(jl).reset(); // this happens second; does real destruction + logging::Message::Args args; + args.add(stmtID); + msgLog.logMessage(logging::LOG_TYPE_DEBUG, ServiceExeMgr::logDbProfEndStatement, args, li); + if (--destructing == 0) + jlCleanupDone.notify_one(); + }); + jl.reset(); // this happens first + bgdtor.detach(); + } + else + // delete sessionMemMap entry for this session's memory % use + deleteMaxMemPct(csep.sessionID()); + + std::string endtime(globServiceExeMgr->timeNow()); + + if ((csep.traceFlags() & globServiceExeMgr->flagsWantOutput) && (csep.sessionID() < 0x80000000)) + { + std::cout << "For session " << csep.sessionID() << ": " << totalBytesSent << " bytes sent back at " + << endtime << std::endl; + + // @bug 663 - Implemented caltraceon(16) to replace the + // $FIFO_SINK compiler definition in pColStep. + // This option consumes rows in the project steps. + if (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS4) + { + std::cout << std::endl; + std::cout << "**** No data returned to DM. Rows consumed " + "in ProjectSteps - caltrace(16) is on (FIFO_SINK)." + " ****" + << std::endl; + std::cout << std::endl; + } + else if (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS3) + { + std::cout << std::endl; + std::cout << "**** No data returned to DM - caltrace(8) is " + "on (SWALLOW_ROWS_EXEMGR). ****" + << std::endl; + std::cout << std::endl; + } + } + + statementsRunningCount->decr(stmtCounted); + + if (!csep.isInternal() && (csep.queryType() == "SELECT" || csep.queryType() == "INSERT_SELECT")) + { + qts.msg_type = querytele::QueryTeleStats::QT_SUMMARY; + qts.max_mem_pct = fStats.fMaxMemPct; + qts.num_files = fStats.fNumFiles; + qts.phy_io = fStats.fPhyIO; + qts.cache_io = fStats.fCacheIO; + qts.msg_rcv_cnt = fStats.fMsgRcvCnt; + qts.cp_blocks_skipped = fStats.fCPBlocksSkipped; + qts.msg_bytes_in = fStats.fMsgBytesIn; + qts.msg_bytes_out = fStats.fMsgBytesOut; + qts.rows = totalRowCount; + qts.end_time = querytele::QueryTeleClient::timeNowms(); + qts.session_id = csep.sessionID(); + qts.query_type = csep.queryType(); + qts.query = csep.data(); + qts.system_name = fOamCachePtr->getSystemName(); + qts.module_name = fOamCachePtr->getModuleName(); + qts.local_query = csep.localQuery(); + fTeleClient.postQueryTele(qts); + } + } + + // Release CSC object (for sessionID) that was added by makeJobList() + // Mask 0x80000000 is for associate user query and csc query. + // (actual joblist destruction happens at the top of this loop) + decThreadCntPerSession(csep.sessionID() | 0x80000000); + } + catch (std::exception& ex) + { + decThreadCntPerSession(csep.sessionID() | 0x80000000); + statementsRunningCount->decr(stmtCounted); + std::cerr << "### ExeMgr ses:" << csep.sessionID() << " caught: " << ex.what() << std::endl; + logging::Message::Args args; + logging::LoggingID li(16, csep.sessionID(), csep.txnID()); + args.add(ex.what()); + msgLog.logMessage(logging::LOG_TYPE_CRITICAL, ServiceExeMgr::logExeMgrExcpt, args, li); + fIos.close(); + } + catch (...) + { + decThreadCntPerSession(csep.sessionID() | 0x80000000); + statementsRunningCount->decr(stmtCounted); + std::cerr << "### Exception caught!" << std::endl; + logging::Message::Args args; + logging::LoggingID li(16, csep.sessionID(), csep.txnID()); + args.add("ExeMgr caught unknown exception"); + msgLog.logMessage(logging::LOG_TYPE_CRITICAL, ServiceExeMgr::logExeMgrExcpt, args, li); + fIos.close(); + } + + // make sure we don't leave scope while joblists are being destroyed + std::unique_lock scoped(jlMutex); + while (destructing > 0) + jlCleanupDone.wait(scoped); +} +}; // namespace exemgr \ No newline at end of file diff --git a/primitives/primproc/sqlfrontsessionthread.h b/primitives/primproc/sqlfrontsessionthread.h new file mode 100644 index 000000000..ca605ea79 --- /dev/null +++ b/primitives/primproc/sqlfrontsessionthread.h @@ -0,0 +1,131 @@ +/* Copyright (C) 2022 Mariadb Corporation. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#pragma once + +#include +#include +#include +#include + +#undef root_name +#include + +#include "calpontselectexecutionplan.h" +#include "mcsanalyzetableexecutionplan.h" +#include "activestatementcounter.h" +#include "distributedenginecomm.h" +#include "resourcemanager.h" +#include "configcpp.h" +#include "queryteleserverparms.h" +#include "iosocket.h" +#include "joblist.h" +#include "joblistfactory.h" +#include "oamcache.h" +#include "simplecolumn.h" +#include "bytestream.h" +#include "telestats.h" +#include "messageobj.h" +#include "messagelog.h" +#include "sqllogger.h" +#include "femsghandler.h" +#include "idberrorinfo.h" +#include "MonitorProcMem.h" +#include "liboamcpp.h" +#include "crashtrace.h" +#include "service.h" + +#include +#include +#include + +#include "dbrm.h" + +#include "mariadb_my_sys.h" +#include "statistics.h" +#include "serviceexemgr.h" + +namespace exemgr +{ + class SQLFrontSessionThread + { + public: + SQLFrontSessionThread(const messageqcpp::IOSocket& ios, joblist::DistributedEngineComm* ec, + joblist::ResourceManager* rm) + : fIos(ios) + , fEc(ec) + , fRm(rm) + , fStatsRetrieved(false) + , fTeleClient(globServiceExeMgr->getTeleServerParms()) + , fOamCachePtr(oam::OamCache::makeOamCache()) + { + } + + private: + messageqcpp::IOSocket fIos; + joblist::DistributedEngineComm* fEc; + joblist::ResourceManager* fRm; + querystats::QueryStats fStats; + + // Variables used to store return stats + bool fStatsRetrieved; + + querytele::QueryTeleClient fTeleClient; + + oam::OamCache* fOamCachePtr; // this ptr is copyable... + + //...Reinitialize stats for start of a new query + void initStats(uint32_t sessionId, std::string& sqlText) + { + initMaxMemPct(sessionId); + + fStats.reset(); + fStats.setStartTime(); + fStats.fSessionID = sessionId; + fStats.fQuery = sqlText; + fStatsRetrieved = false; + } + //...Get % memory usage during latest query for sesssionId. + //...SessionId >= 0x80000000 is system catalog query we can ignore. + static uint64_t getMaxMemPct(uint32_t sessionId); + //...Delete sessionMemMap entry for the specified session's memory % use. + //...SessionId >= 0x80000000 is system catalog query we can ignore. + static void deleteMaxMemPct(uint32_t sessionId); + //...Get and log query stats to specified output stream + const std::string formatQueryStats( + joblist::SJLP& jl, // joblist associated with query + const std::string& label, // header label to print in front of log output + bool includeNewLine, // include line breaks in query stats std::string + bool vtableModeOn, bool wantExtendedStats, uint64_t rowsReturned); + static void incThreadCntPerSession(uint32_t sessionId); + static void decThreadCntPerSession(uint32_t sessionId); + //...Init sessionMemMap entry for specified session to 0 memory %. + //...SessionId >= 0x80000000 is system catalog query we can ignore. + static void initMaxMemPct(uint32_t sessionId); + //... Round off to human readable format (KB, MB, or GB). + const std::string roundBytes(uint64_t value) const; + void setRMParms(const execplan::CalpontSelectExecutionPlan::RMParmVec& parms); + void buildSysCache(const execplan::CalpontSelectExecutionPlan& csep, + boost::shared_ptr csc); + void writeCodeAndError(messageqcpp::ByteStream::quadbyte code, const std::string emsg); + void analyzeTableExecute(messageqcpp::ByteStream& bs, joblist::SJLP& jl, bool& stmtCounted); + void analyzeTableHandleStats(messageqcpp::ByteStream& bs); + uint64_t roundMB(uint64_t value) const; + public: + void operator()(); + }; +} \ No newline at end of file diff --git a/primitives/primproc/udf.cpp b/primitives/primproc/udf.cpp index 7a836590d..188e5128a 100644 --- a/primitives/primproc/udf.cpp +++ b/primitives/primproc/udf.cpp @@ -69,4 +69,3 @@ void loadUDFs() } } // namespace primitiveprocessor -// vim:ts=4 sw=4: diff --git a/primitives/primproc/umsocketselector.cpp b/primitives/primproc/umsocketselector.cpp index 7b58fa73a..5cdd5f383 100644 --- a/primitives/primproc/umsocketselector.cpp +++ b/primitives/primproc/umsocketselector.cpp @@ -119,7 +119,6 @@ void UmSocketSelector::loadUMModuleInfo() std::cout << "ModuleConfig for type: " << UM_MODTYPE << std::endl; std::cout << "ModuleDesc = " << moduleTypeConfig.ModuleDesc << std::endl; std::cout << "ModuleCount = " << moduleCount << std::endl; - std::cout << "RunType = " << moduleTypeConfig.RunType << std::endl; #endif if (moduleCount > 0) diff --git a/storage-manager/CMakeLists.txt b/storage-manager/CMakeLists.txt index cd71417b8..695d02d03 100755 --- a/storage-manager/CMakeLists.txt +++ b/storage-manager/CMakeLists.txt @@ -3,7 +3,7 @@ project(storagemanager) include_directories(include ${CMAKE_BINARY_DIR}/include ${ENGINE_UTILS_COMMON_INCLUDE} ${S3API_DIR}) -set(storagemanager_SRCS +set(storagemanager_SRCS src/AppendTask.cpp src/ClientRequestProcessor.cpp src/ListDirectoryTask.cpp @@ -62,12 +62,12 @@ set(CMAKE_INSTALL_RPATH $ORIGIN $ORIGIN/../lib) add_library(storagemanager SHARED ${storagemanager_SRCS}) add_dependencies(storagemanager marias3) target_compile_definitions(storagemanager PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) -target_link_libraries(storagemanager boost_chrono boost_system boost_thread boost_filesystem boost_regex pthread ${S3API_DEPS}) -set_property(TARGET storagemanager PROPERTY CXX_STANDARD 11) +target_link_libraries(storagemanager Boost::chrono Boost::system Boost::thread Boost::filesystem Boost::regex pthread ${S3API_DEPS}) +set_property(TARGET storagemanager PROPERTY CXX_STANDARD 20) add_executable(StorageManager src/main.cpp) target_link_libraries(StorageManager storagemanager) -set_property(TARGET StorageManager PROPERTY CXX_STANDARD 11) +set_property(TARGET StorageManager PROPERTY CXX_STANDARD 20) set(TMPDIR ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) @@ -75,7 +75,7 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) add_executable(unit_tests src/unit_tests.cpp) target_compile_definitions(unit_tests PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) target_link_libraries(unit_tests storagemanager) -set_property(TARGET unit_tests PROPERTY CXX_STANDARD 11) +set_property(TARGET unit_tests PROPERTY CXX_STANDARD 20) add_executable(testS3Connection src/testS3Connection.cpp) target_compile_definitions(testS3Connection PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) @@ -96,10 +96,10 @@ add_custom_command( ${CMAKE_CURRENT_BINARY_DIR}/test_data # COMMAND ${CMAKE_COMMAND} -E copy # ../bin/unit_tests # what is putting our bins in ../bin? -# ${CMAKE_CURRENT_BINARY_DIR} +# ${CMAKE_CURRENT_BINARY_DIR} ) -# The includes and lib linkages required to link against cloudio ... +# The includes and lib linkages required to link against cloudio ... # pretty crazy. When lib dependencies are eventually config'd right, # change this to only include and link against cloudio. include_directories(${ENGINE_SRC_DIR}/utils/cloudio ${ENGINE_COMMON_INCLUDES}) @@ -141,12 +141,12 @@ install(TARGETS StorageManager smcat smput smls smrm testS3Connection COMPONENT columnstore-engine ) -install(FILES storagemanager.cnf +install(FILES storagemanager.cnf DESTINATION ${ENGINE_SYSCONFDIR}/columnstore COMPONENT columnstore-engine ) -install(FILES storagemanager.cnf +install(FILES storagemanager.cnf RENAME storagemanager.cnf.example DESTINATION ${ENGINE_SYSCONFDIR}/columnstore COMPONENT columnstore-engine diff --git a/storage-manager/src/Config.cpp b/storage-manager/src/Config.cpp index f634ececb..fa9873484 100644 --- a/storage-manager/src/Config.cpp +++ b/storage-manager/src/Config.cpp @@ -23,7 +23,6 @@ #include #include #include -#include #include #include @@ -31,6 +30,7 @@ #include #include #include +#include #include "SMLogging.h" @@ -165,13 +165,13 @@ bool Config::reload() return rtn; } -string use_envvar(const boost::smatch& envvar) +string use_envvar(const std::smatch& envvar) { char* env = getenv(envvar[1].str().c_str()); return (env ? env : ""); } -string expand_numbers(const boost::smatch& match) +string expand_numbers(const std::smatch& match) { long long num = stol(match[1].str()); char suffix = (char)::tolower(match[2].str()[0]); @@ -187,6 +187,20 @@ string expand_numbers(const boost::smatch& match) return ::to_string(num); } +std::string regex_replace_with_format(const std::string& input, + const std::regex& regex, + std::function format) +{ + + std::ostringstream output; + std::sregex_iterator begin(input.begin(), input.end(), regex), end; + for(; begin != end; begin++){ + output << begin->prefix() << format(*begin); + } + output << input.substr(input.size() - begin->position()); + return output.str(); +} + string Config::getValue(const string& section, const string& key) const { // if we care, move this envvar substition stuff to where the file is loaded @@ -202,15 +216,15 @@ string Config::getValue(const string& section, const string& key) const } s.unlock(); - boost::regex re("\\$\\{(.+)\\}"); + std::regex re("\\$\\{(.+)\\}"); - ret = boost::regex_replace(ret, re, use_envvar); + ret = regex_replace_with_format(ret, re, use_envvar); // do the numeric substitutions. ex, the suffixes m, k, g // ehhhhh. going to end up turning a string to a number, to a string, and then to a number again // don't like that. OTOH who cares. - boost::regex num_re("^([[:digit:]]+)([mMkKgG])$", boost::regex::extended); - ret = boost::regex_replace(ret, num_re, expand_numbers); + std::regex num_re("^([[:digit:]]+)([mMkKgG])$", std::regex::extended); + ret = regex_replace_with_format(ret, num_re, expand_numbers); return ret; } diff --git a/storage-manager/src/Downloader.cpp b/storage-manager/src/Downloader.cpp index 7d21b7b4f..1a0897f9f 100644 --- a/storage-manager/src/Downloader.cpp +++ b/storage-manager/src/Downloader.cpp @@ -214,7 +214,11 @@ void Downloader::configListener() // Downloader threads string stmp = Config::get()->getValue("ObjectStorage", "max_concurrent_downloads"); if (maxDownloads == 0) + { maxDownloads = 20; + workers.setMaxThreads(maxDownloads); + logger->log(LOG_INFO, "max_concurrent_downloads = %u",maxDownloads); + } if (stmp.empty()) { logger->log(LOG_CRIT, "max_concurrent_downloads is not set. Using current value = %u", maxDownloads); diff --git a/storage-manager/src/IOCoordinator.cpp b/storage-manager/src/IOCoordinator.cpp index b17d502b2..52c1567f6 100644 --- a/storage-manager/src/IOCoordinator.cpp +++ b/storage-manager/src/IOCoordinator.cpp @@ -25,8 +25,6 @@ #include #include #include -#define BOOST_SPIRIT_THREADSAFE -#include #include #include "checks.h" #include "vlarray.h" @@ -1266,9 +1264,10 @@ boost::shared_array IOCoordinator::mergeJournal(const char* object, con boost::shared_array headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead); stringstream ss; ss << headertxt.get(); - boost::property_tree::ptree header; - boost::property_tree::json_parser::read_json(ss, header); - assert(header.get("version") == 1); + + nlohmann::json header = nlohmann::json::parse(ss); + + assert(header["version"] == 1); // start processing the entries while (1) @@ -1353,9 +1352,9 @@ int IOCoordinator::mergeJournalInMem(boost::shared_array& objData, size boost::shared_array headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead); stringstream ss; ss << headertxt.get(); - boost::property_tree::ptree header; - boost::property_tree::json_parser::read_json(ss, header); - assert(header.get("version") == 1); + + nlohmann::json header = nlohmann::json::parse(ss); + assert(header["version"] == 1); // read the journal file into memory size_t journalBytes = ::lseek(journalFD, 0, SEEK_END) - l_bytesRead; @@ -1433,9 +1432,9 @@ int IOCoordinator::mergeJournalInMem_bigJ(boost::shared_array& objData, boost::shared_array headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead); stringstream ss; ss << headertxt.get(); - boost::property_tree::ptree header; - boost::property_tree::json_parser::read_json(ss, header); - assert(header.get("version") == 1); + + nlohmann::json header = nlohmann::json::parse(ss); + assert(header["version"] == 1); // start processing the entries while (1) diff --git a/storage-manager/src/MetadataFile.cpp b/storage-manager/src/MetadataFile.cpp index e45c95173..5386beae5 100644 --- a/storage-manager/src/MetadataFile.cpp +++ b/storage-manager/src/MetadataFile.cpp @@ -20,20 +20,17 @@ */ #include "MetadataFile.h" #include -#define BOOST_SPIRIT_THREADSAFE -#include -#include -#include #include #include #include +#include #include +#include #define max(x, y) (x > y ? x : y) #define min(x, y) (x < y ? x : y) using namespace std; -namespace bpt = boost::property_tree; namespace bf = boost::filesystem; namespace @@ -120,12 +117,13 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename) { if (boost::filesystem::exists(mFilename)) { - jsontree.reset(new bpt::ptree()); - boost::property_tree::read_json(mFilename.string(), *jsontree); + std::ifstream i(mFilename.string()); + jsontree.reset(new nlohmann::json); + i >> *jsontree; jsonCache.put(mFilename, jsontree); s.unlock(); mVersion = 1; - mRevision = jsontree->get("revision"); + mRevision = (*jsontree)["revision"]; } else { @@ -140,7 +138,7 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename) { s.unlock(); mVersion = 1; - mRevision = jsontree->get("revision"); + mRevision = (*jsontree)["revision"];; } ++metadataFilesAccessed; } @@ -162,12 +160,13 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename, no_create_t, if (boost::filesystem::exists(mFilename)) { _exists = true; - jsontree.reset(new bpt::ptree()); - boost::property_tree::read_json(mFilename.string(), *jsontree); + jsontree.reset(new nlohmann::json); + std::ifstream i(mFilename.string()); + i >> *jsontree; jsonCache.put(mFilename, jsontree); s.unlock(); mVersion = 1; - mRevision = jsontree->get("revision"); + mRevision = (*jsontree)["revision"]; } else { @@ -182,7 +181,7 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename, no_create_t, s.unlock(); _exists = true; mVersion = 1; - mRevision = jsontree->get("revision"); + mRevision = (*jsontree)["revision"]; } ++metadataFilesAccessed; } @@ -193,11 +192,10 @@ MetadataFile::~MetadataFile() void MetadataFile::makeEmptyJsonTree() { - jsontree.reset(new bpt::ptree()); - boost::property_tree::ptree objs; - jsontree->put("version", mVersion); - jsontree->put("revision", mRevision); - jsontree->add_child("objects", objs); + jsontree.reset(new nlohmann::json); + (*jsontree)["version"] = mVersion; + (*jsontree)["revision"] = mRevision; + (*jsontree)["objects"] = nlohmann::json::array(); } void MetadataFile::printKPIs() @@ -219,11 +217,11 @@ size_t MetadataFile::getLength() const { size_t totalSize = 0; - auto& objects = jsontree->get_child("objects"); + auto &objects = (*jsontree)["objects"]; if (!objects.empty()) { - auto& lastObject = objects.back().second; - totalSize = lastObject.get("offset") + lastObject.get("length"); + auto& lastObject = objects.back(); + totalSize = lastObject["offset"].get() + lastObject["length"].get(); } return totalSize; } @@ -243,10 +241,9 @@ vector MetadataFile::metadataRead(off_t offset, size_t length) c rather than write a new alg. */ set mObjects; - BOOST_FOREACH (const boost::property_tree::ptree::value_type& v, jsontree->get_child("objects")) + for(const auto &v : (*jsontree)["objects"]) { - mObjects.insert(metadataObject(v.second.get("offset"), v.second.get("length"), - v.second.get("key"))); + mObjects.insert(metadataObject(v["offset"], v["length"], v["key"])); } if (mObjects.size() == 0) @@ -288,20 +285,20 @@ metadataObject MetadataFile::addMetadataObject(const boost::filesystem::path& fi // metadataObject addObject; - auto& objects = jsontree->get_child("objects"); + auto& objects = (*jsontree)["objects"]; if (!objects.empty()) { - auto& lastObject = objects.back().second; - addObject.offset = lastObject.get("offset") + mpConfig->mObjectSize; + auto& lastObject = objects.back(); + addObject.offset = lastObject["offset"].get() + mpConfig->mObjectSize; } addObject.length = length; addObject.key = getNewKey(filename.string(), addObject.offset, addObject.length); - boost::property_tree::ptree object; - object.put("offset", addObject.offset); - object.put("length", addObject.length); - object.put("key", addObject.key); - objects.push_back(make_pair("", object)); + nlohmann::json object = nlohmann::json::object(); + object["offset"] = addObject.offset; + object["length"] = addObject.length; + object["key"] = addObject.key; + objects.push_back(object); return addObject; } @@ -312,7 +309,8 @@ int MetadataFile::writeMetadata() if (!boost::filesystem::exists(mFilename.parent_path())) boost::filesystem::create_directories(mFilename.parent_path()); - write_json(mFilename.string(), *jsontree); + std::ofstream o(mFilename.c_str()); + o << *jsontree; _exists = true; boost::unique_lock s(jsonCache.getMutex()); @@ -324,13 +322,13 @@ int MetadataFile::writeMetadata() bool MetadataFile::getEntry(off_t offset, metadataObject* out) const { metadataObject addObject; - BOOST_FOREACH (const boost::property_tree::ptree::value_type& v, jsontree->get_child("objects")) + for(auto &v: (*jsontree)["objects"]) { - if (v.second.get("offset") == offset) + if (v["offset"].get() == offset) { out->offset = offset; - out->length = v.second.get("length"); - out->key = v.second.get("key"); + out->length = v["length"].get(); + out->key = v["key"]; return true; } } @@ -339,10 +337,10 @@ bool MetadataFile::getEntry(off_t offset, metadataObject* out) const void MetadataFile::removeEntry(off_t offset) { - bpt::ptree& objects = jsontree->get_child("objects"); - for (bpt::ptree::iterator it = objects.begin(); it != objects.end(); ++it) + auto& objects = (*jsontree)["objects"]; + for (auto it = objects.begin(); it != objects.end(); ++it) { - if (it->second.get("offset") == offset) + if ((*it)["offset"].get() == offset) { objects.erase(it); break; @@ -352,7 +350,7 @@ void MetadataFile::removeEntry(off_t offset) void MetadataFile::removeAllEntries() { - jsontree->get_child("objects").clear(); + (*jsontree)["objects"] = nlohmann::json::array(); } void MetadataFile::deletedMeta(const bf::path& p) @@ -456,21 +454,21 @@ void MetadataFile::setLengthInKey(string& key, size_t newLength) void MetadataFile::printObjects() const { - BOOST_FOREACH (const boost::property_tree::ptree::value_type& v, jsontree->get_child("objects")) + for (auto& v : (*jsontree)["objects"]) { - printf("Name: %s Length: %zu Offset: %lld\n", v.second.get("key").c_str(), - v.second.get("length"), (long long)v.second.get("offset")); + printf("Name: %s Length: %zu Offset: %lld\n", v["key"].get().c_str(), + v["length"].get(), (long long)v["offset"].get()); } } void MetadataFile::updateEntry(off_t offset, const string& newName, size_t newLength) { - for (auto& v : jsontree->get_child("objects")) + for (auto& v : (*jsontree)["objects"]) { - if (v.second.get("offset") == offset) + if (v["offset"].get() == offset) { - v.second.put("key", newName); - v.second.put("length", newLength); + v["key"] = newName; + v["length"] = newLength; return; } } @@ -482,11 +480,11 @@ void MetadataFile::updateEntry(off_t offset, const string& newName, size_t newLe void MetadataFile::updateEntryLength(off_t offset, size_t newLength) { - for (auto& v : jsontree->get_child("objects")) + for (auto& v : (*jsontree)["objects"]) { - if (v.second.get("offset") == offset) + if (v["offset"].get() == offset) { - v.second.put("length", newLength); + v["length"] = newLength; return; } } @@ -498,11 +496,12 @@ void MetadataFile::updateEntryLength(off_t offset, size_t newLength) off_t MetadataFile::getMetadataNewObjectOffset() { - auto& objects = jsontree->get_child("objects"); + auto& objects = (*jsontree)["objects"]; if (objects.empty()) return 0; - auto& lastObject = jsontree->get_child("objects").back().second; - return lastObject.get("offset") + lastObject.get("length"); + + auto& lastObject = objects.back(); + return lastObject["offset"].get() + lastObject["length"].get(); } metadataObject::metadataObject() : offset(0), length(0) diff --git a/storage-manager/src/MetadataFile.h b/storage-manager/src/MetadataFile.h index 5c302c618..94d65c404 100644 --- a/storage-manager/src/MetadataFile.h +++ b/storage-manager/src/MetadataFile.h @@ -28,6 +28,8 @@ #include #include +#include + namespace storagemanager { struct metadataObject @@ -110,7 +112,7 @@ class MetadataFile static void printKPIs(); - typedef boost::shared_ptr Jsontree_t; + typedef boost::shared_ptr Jsontree_t; private: MetadataConfig* mpConfig; diff --git a/storage-manager/src/Replicator.cpp b/storage-manager/src/Replicator.cpp index 10fc07002..2a9241306 100644 --- a/storage-manager/src/Replicator.cpp +++ b/storage-manager/src/Replicator.cpp @@ -27,12 +27,12 @@ #include #include #include -#define BOOST_SPIRIT_THREADSAFE -#include #include #include #include +#include + using namespace std; namespace @@ -279,12 +279,14 @@ int Replicator::addJournalEntry(const boost::filesystem::path& filename, const u stringstream ss; ss << headertxt.get(); headerRollback = headertxt.get(); - boost::property_tree::ptree header; + nlohmann::json header; + try { - boost::property_tree::json_parser::read_json(ss, header); + header = nlohmann::json::parse(ss); } - catch (const boost::property_tree::json_parser::json_parser_error& e) + + catch (const nlohmann::json::exception& e) { mpLogger->log(LOG_CRIT, "%s", e.what()); errno = EIO; @@ -296,8 +298,8 @@ int Replicator::addJournalEntry(const boost::filesystem::path& filename, const u errno = EIO; return -1; } - assert(header.get("version") == 1); - uint64_t currentMaxOffset = header.get("max_offset"); + assert(header["version"] == 1); + uint64_t currentMaxOffset = header["max_offset"]; if (thisEntryMaxOffset > currentMaxOffset) { bHeaderChanged = true; diff --git a/storage-manager/src/S3Storage.cpp b/storage-manager/src/S3Storage.cpp index 758d9b2e6..9f14e1562 100644 --- a/storage-manager/src/S3Storage.cpp +++ b/storage-manager/src/S3Storage.cpp @@ -26,9 +26,9 @@ #include #include #include -#define BOOST_SPIRIT_THREADSAFE -#include -#include + +#include "utils/json/json.hpp" + #include "Utilities.h" using namespace std; @@ -258,12 +258,12 @@ bool S3Storage::getCredentialsFromMetadataEC2() logger->log(LOG_ERR, "CURL fail %u", curl_res); return false; } - stringstream credentials(readBuffer); - boost::property_tree::ptree pt; - boost::property_tree::read_json(credentials, pt); - key = pt.get("AccessKeyId"); - secret = pt.get("SecretAccessKey"); - token = pt.get("Token"); + + nlohmann::json pt = nlohmann::json::parse(readBuffer); + key = pt["AccessKeyId"]; + secret = pt["SecretAccessKey"]; + token = pt["Token"]; + // logger->log(LOG_INFO, "S3Storage: key = %s secret = %s token = // %s",key.c_str(),secret.c_str(),token.c_str()); @@ -626,7 +626,7 @@ int S3Storage::copyObject(const string& _sourceKey, const string& _destKey) #if 0 // no s3-s3 copy yet. get & put for now. - + int err; boost::shared_array data; size_t len; diff --git a/storage-manager/src/Synchronizer.cpp b/storage-manager/src/Synchronizer.cpp index ed02f5eb1..1c48f0887 100644 --- a/storage-manager/src/Synchronizer.cpp +++ b/storage-manager/src/Synchronizer.cpp @@ -884,7 +884,11 @@ void Synchronizer::configListener() // Uploader threads string stmp = Config::get()->getValue("ObjectStorage", "max_concurrent_uploads"); if (maxUploads == 0) + { maxUploads = 20; + threadPool->setMaxThreads(maxUploads); + logger->log(LOG_INFO, "max_concurrent_uploads = %u",maxUploads); + } if (stmp.empty()) { logger->log(LOG_CRIT, "max_concurrent_uploads is not set. Using current value = %u", maxUploads); diff --git a/storage-manager/src/smcat.cpp b/storage-manager/src/smcat.cpp index bc1b1e1a9..101ab41a7 100644 --- a/storage-manager/src/smcat.cpp +++ b/storage-manager/src/smcat.cpp @@ -93,7 +93,7 @@ void catFileOffline(const char* filename, int prefixlen) void catFileOnline(const char* filename, int prefixlen) { uint8_t data[8192]; - off_t offset = 0; + [[maybe_unused]] off_t offset = 0; int read_err, write_err, count; idbdatafile::SMDataFile df(filename, O_RDONLY, 0); diff --git a/storage-manager/src/unit_tests.cpp b/storage-manager/src/unit_tests.cpp index 1864939f4..0a9dd8d2a 100644 --- a/storage-manager/src/unit_tests.cpp +++ b/storage-manager/src/unit_tests.cpp @@ -1056,10 +1056,10 @@ bool copytask(bool connectionTest = false) copy_cmd* cmd = (copy_cmd*)buf; cmd->opcode = COPY; cmd->file1.flen = strlen(source); - strncpy(cmd->file1.filename, source, cmd->file1.flen); + memcpy(cmd->file1.filename, source, cmd->file1.flen); f_name* file2 = (f_name*)&cmd->file1.filename[cmd->file1.flen]; file2->flen = strlen(dest); - strncpy(file2->filename, dest, file2->flen); + memcpy(file2->filename, dest, file2->flen); uint len = (uint64_t)&file2->filename[file2->flen] - (uint64_t)buf; diff --git a/storage-manager/storagemanager.cnf.in b/storage-manager/storagemanager.cnf.in index a1b086d4e..1b95faf5a 100644 --- a/storage-manager/storagemanager.cnf.in +++ b/storage-manager/storagemanager.cnf.in @@ -55,9 +55,6 @@ journal_path = @ENGINE_DATADIR@/storagemanager/journal # max_concurrent_downloads is what is sounds like, per node. # This is not a global setting. -# -# The default was changed from 20 to 21 as a temporary workaround -# for a bug. It can be anything but 20. max_concurrent_downloads = 21 # max_concurrent_uploads is what is sounds like, per node. @@ -66,9 +63,6 @@ max_concurrent_downloads = 21 # has low upstream bandwidth, consider lowering this value to the minimum # necessary to saturate your network. This will reduce the latency of certain # operations and improve your experience. -# -# The default was changed from 20 to 21 as a temporary workaround -# for a bug. It can be anything but 20. max_concurrent_uploads = 21 # common_prefix_depth is the depth of the common prefix that all files diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index abbba30e4..a8af32368 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,35 +1,56 @@ -include_directories( ${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE}) +include_directories( ${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE} ) if (WITH_UNITTESTS) - enable_testing() + set(EXTERNAL_INSTALL_LOCATION ${CMAKE_BINARY_DIR}/external) + ExternalProject_Add(googletest + GIT_REPOSITORY https://github.com/google/googletest + GIT_TAG release-1.11.0 + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EXTERNAL_INSTALL_LOCATION} -DBUILD_SHARED_LIBS=ON + ) + + include_directories(${EXTERNAL_INSTALL_LOCATION}/include) + # lib64 for RPM-based distros + link_directories(${EXTERNAL_INSTALL_LOCATION}/lib ${EXTERNAL_INSTALL_LOCATION}/lib64) + set(GTEST_LIBRARIES gtest gtest_main pthread) include(GoogleTest) - find_package(GTest REQUIRED) #GoogleTest tests add_executable(rowgroup_tests rowgroup-tests.cpp) + add_dependencies(rowgroup_tests googletest) target_link_libraries(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) gtest_discover_tests(rowgroup_tests TEST_PREFIX columnstore:) add_executable(mcs_decimal_tests mcs_decimal-tests.cpp) + add_dependencies(mcs_decimal_tests googletest) target_link_libraries(mcs_decimal_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) gtest_discover_tests(mcs_decimal_tests TEST_PREFIX columnstore:) add_executable(dataconvert_tests dataconvert-tests.cpp) + add_dependencies(dataconvert_tests googletest) target_link_libraries(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) gtest_discover_tests(dataconvert_tests TEST_PREFIX columnstore:) add_executable(rebuild_em_tests rebuild-em-tests.cpp) + add_dependencies(rebuild_em_tests googletest) target_link_libraries(rebuild_em_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS}) gtest_discover_tests(rebuild_em_tests TEST_PREFIX columnstore:) add_executable(compression_tests compression-tests.cpp) + add_dependencies(compression_tests googletest) target_link_libraries(compression_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS}) gtest_discover_tests(compression_tests TEST_PREFIX columnstore:) add_executable(column_scan_filter_tests primitives_column_scan_and_filter.cpp) + target_compile_options(column_scan_filter_tests PRIVATE -Wno-error -Wno-sign-compare) + add_dependencies(column_scan_filter_tests googletest) target_link_libraries(column_scan_filter_tests ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc) gtest_discover_tests(column_scan_filter_tests TEST_PREFIX columnstore:) + add_executable(simd_processors simd_processors.cpp) + add_dependencies(simd_processors googletest) + target_link_libraries(simd_processors ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc) + gtest_discover_tests(simd_processors TEST_PREFIX columnstore:) + # CPPUNIT TESTS add_executable(we_shared_components_tests shared_components_tests.cpp) add_dependencies(we_shared_components_tests loggingcpp) @@ -41,6 +62,11 @@ if (WITH_UNITTESTS) target_link_libraries(comparators_tests ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) add_test(NAME columnstore:comparators_tests, COMMAND comparators_tests) + # standalone EM routines test + # add_executable(brm_em_standalone brm-em-standalone.cpp) + # target_link_libraries(brm_em_standalone ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) + # install(TARGETS brm_em_standalone DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) + endif() # Saving this as the example of the microbench diff --git a/tests/brm-em-standalone.cpp b/tests/brm-em-standalone.cpp new file mode 100644 index 000000000..92b410107 --- /dev/null +++ b/tests/brm-em-standalone.cpp @@ -0,0 +1,1914 @@ +/* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2016-2021 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +/***************************************************************************** + * $Id: tdriver-dbrm2.cpp 1823 2013-01-21 14:13:09Z rdempsey $ + * + ****************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "brm.h" +#include "extentmap.h" +#include "IDBPolicy.h" + +//#define BRM_VERBOSE 1 + +using namespace BRM; +using namespace std; + +int threadStop; +int oid = 1; +uint64_t opCount = 0; +LBID_t lbidCounter = 0; +VER_t nextTxnID = 1; +u_int64_t vbOffset = 0; +pthread_mutex_t pthreadMutex; +const std::vector colWidthsAvailable = {1, 2, 4, 8, 16}; +const DBRootT dbroot = 1; +const uint32_t KibiBlocks = 1024; + +struct Range +{ + LBID_t start, end, nextBlock; + VER_t txnID; + Range() + { + start = end = nextBlock = 0; + txnID = 0; + } +}; + +struct EMEntries +{ + LBID_t LBIDstart; + uint32_t size; + OID_t OID; + uint32_t FBO; + uint32_t HWM; + uint32_t secondHWM; + int32_t txnID; + DBRootT dbroot; + PartitionNumberT partNum; + SegmentT segNum; + struct EMEntries* next; + EMEntries() : HWM(0), secondHWM(0), txnID(0), next(nullptr) + { } + EMEntries(const uint32_t aSize, const OID_t aOid, const uint32_t aFbo, + const LBID_t aLBIDStart, EMEntries* aNext) + : LBIDstart(aLBIDStart), size(aSize), OID(aOid), FBO(aFbo), HWM(0), + secondHWM(0), txnID(0), next(aNext) + { } + EMEntries(const uint32_t aSize, const OID_t aOid, const uint32_t aFbo, + const LBID_t aLBIDStart, EMEntries* aNext, const DBRootT aDbroot, + const PartitionNumberT aPartNum, const SegmentT aSegNum) + : LBIDstart(aLBIDStart), size(aSize), OID(aOid), FBO(aFbo), HWM(0), + secondHWM(0), txnID(0), dbroot(aDbroot), partNum(aPartNum), + segNum(aSegNum), next(aNext) + { } +}; +/* +static void* BRMRunner_2(void* arg) +{ + + vector copyList, copiedList, committedList; + vector::iterator rit; + vector writtenList; + vector::iterator lit; + + pthread_mutex_t listMutex; + int op; + uint32_t randstate; + DBRM* brm; + struct timeval tv; + VER_t txnID; + + pthread_mutex_init(&listMutex, NULL); + gettimeofday(&tv, NULL); + randstate = static_cast(tv.tv_usec); + brm = new DBRM(); + + while (!threadStop) + { + op = rand_r(&randstate) % 9; + + switch (op) + { + case 0: // beginVBCopy + { + int blockCount, size, err; + Range newEntry; + VBRange_v vbRanges; + VBRange_v::iterator vit; + LBIDRange_v ranges; + LBIDRange range; + + size = rand_r(&randstate) % 10000; + + pthread_mutex_lock(&pthreadMutex); + newEntry.start = lbidCounter; + lbidCounter += size; + txnID = nextTxnID++; + pthread_mutex_unlock(&pthreadMutex); + + newEntry.nextBlock = newEntry.start; + newEntry.end = newEntry.start + size; + range.start = newEntry.start; + range.size = size; + + err = brm->beginVBCopy(txnID, dbroot, ranges, vbRanges); + CPPUNIT_ASSERT(err == 0); + + for (blockCount = 0, vit = vbRanges.begin(); vit != vbRanges.end(); vit++) + blockCount += (*vit).size; + + CPPUNIT_ASSERT(blockCount == size); + + pthread_mutex_lock(&listMutex); + copyList.push_back(newEntry); + pthread_mutex_unlock(&listMutex); + + err = brm->beginVBCopy(txnID, dbroot, ranges, vbRanges); + CPPUNIT_ASSERT(err == -1); + break; + } + + case 1: // writeVBEntry + { + int randIndex; + Range* entry; + + pthread_mutex_lock(&listMutex); + + if (copyList.size() == 0) + break; + + randIndex = rand_r(&randstate) % copyList.size(); + entry = &(copyList[randIndex]); + entry->nextBlock++; + txnID = entry->txnID; + break; + } + + default: + cerr << "not finished yet" << endl; + } + } + + return NULL; +} +*/ + +/* +static void* BRMRunner_1(void* arg) +{ + + // keep track of LBID ranges allocated here and + // randomly allocate, lookup, delete, get/set HWM, and + // destroy the EM object. + +#ifdef BRM_VERBOSE + int threadNum = reinterpret_cast(arg); +#endif + int op, listSize = 0, i; + uint32_t randstate; + struct EMEntries* head = NULL, *tmp; + struct timeval tv; + DBRM* brm; + ExtentMap em; + vector lbids; + LBID_t lbid; + uint32_t colWidth; + PartitionNumberT partNum; + SegmentT segmentNum; + execplan::CalpontSystemCatalog::ColDataType colDataType; + +#ifdef BRM_VERBOSE + cerr << "thread number " << threadNum << " started." << endl; +#endif + + gettimeofday(&tv, NULL); + randstate = static_cast(tv.tv_usec); + brm = new DBRM(); + + + while (!threadStop) + { + auto randNumber = rand_r(&randstate); + op = randNumber % 10; + colWidth = colWidthsAvailable[randNumber % colWidthsAvailable.size()]; + partNum = randNumber % std::numeric_limits::max(); + segmentNum = randNumber % std::numeric_limits::max(); + colDataType = (execplan::CalpontSystemCatalog::ColDataType) (randNumber % (int)execplan::CalpontSystemCatalog::ColDataType::TIMESTAMP); +#ifdef BRM_VERBOSE + cerr << "next op is " << op << endl; +#endif + + switch (op) + { + case 0: //allocate space for a new file + { + struct EMEntries* newEm; + size_t size = rand_r(&randstate) % 102399 + 1; + int entries, OID, allocdSize, err; + uint32_t startBlockOffset; + + + pthread_mutex_lock(&pthreadMutex); + OID = oid++; + opCount++; + pthread_mutex_unlock(&pthreadMutex); + lbids.clear(); + for (size_t i = 0; i < size; ++i) + { + err = brm->createColumnExtent_DBroot(OID, colWidth, dbroot, + partNum, segmentNum, colDataType, lbid, allocdSize, startBlockOffset); + CPPUNIT_ASSERT(err == 0); + lbids.push_back(lbid); + } + + entries = size / brm->getExtentSize(); + + if ((size % brm->getExtentSize()) != 0) + entries++; + + if ((uint32_t)entries != lbids.size()) + cerr << "entries = " << entries << " lbids.size = " << lbids.size() << endl; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0 ; i < entries; i++) + { + + newEm = new EMEntries(brm->getExtentSize(), OID, brm->getExtentSize(), lbids[i], + head, dbroot, partNum, segmentNum); + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created new space for OID " << newEm->OID << endl; +#endif + em.checkConsistency(); + break; + } + + case 1: //allocate space for an existing file + { + if (listSize == 0) + break; + + struct EMEntries* newEm, *tmp; + int size = rand_r(&randstate) % 102399 + 1; + int fileRand = rand_r(&randstate) % listSize; + int i, lastExtent, blockEnd, oid; + int tmpHWM, entries, allocdSize, err; + uint32_t startBlockOffset; + vector lbids; + LBID_t lbid; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) + { + if (tmp->OID != oid) + continue; + + tmpHWM = tmp->HWM; + blockEnd = tmp->FBO + tmp->size; + + if (lastExtent < blockEnd) + lastExtent = blockEnd; + } + + err = brm->createColumnExtentExactFile(oid, colWidth, dbroot, + partNum, segmentNum, colDataType, lbid, allocdSize, startBlockOffset); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); + + entries = size / brm->getExtentSize(); + + if ((size % brm->getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + + newEm = new EMEntries((i != entries) ? brm->getExtentSize() : size % brm->getExtentSize(), + oid, lastExtent + (i * brm->getExtentSize()), + lbids[i], head, dbroot, partNum, segmentNum); + newEm->HWM = tmpHWM; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created another extent for OID " << newEm->OID << endl; +#endif + em.checkConsistency(); + break; + } + + case 2: //delete an OID + { + if (listSize == 0) + break; + + struct EMEntries* tmp, *prev; + int fileRand = rand_r(&randstate) % listSize; + int i, oid, err; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + err = brm->deleteOID(oid); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); + + for (tmp = head; tmp != NULL;) + { + if (tmp->OID == oid) + { + if (tmp == head) + { + head = head->next; + delete tmp; + tmp = head; + } + else + { + prev->next = tmp->next; + delete tmp; + tmp = prev->next; + } + + listSize--; + } + else + { + prev = tmp; + tmp = tmp->next; + } + } + +#ifdef BRM_VERBOSE + cerr << "deleted OID " << oid << endl; +#endif + em.checkConsistency(); + break; + } + + case 3: //lookup by LBID + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset, oid; + struct EMEntries* tmp; + LBID_t target; + uint32_t fbo; + DBRootT localDbroot; + PartitionNumberT localPartNum; + SegmentT localSegmentNum; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + + target = tmp->LBIDstart + offset; + err = brm->lookupLocal(target, 0, false, oid, localDbroot, localPartNum, + localSegmentNum, fbo); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); +#ifdef BRM_VERBOSE + cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; + cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == tmp->OID); + CPPUNIT_ASSERT(fbo == offset + tmp->FBO); + em.checkConsistency(); + break; + } + + case 4: //lookup by OID, FBO + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, oid, err, offset; + struct EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + oid = tmp->OID; + + err = brm->lookupLocal(oid, partNum, segmentNum, offset + tmp->FBO, lbid); + + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); +#ifdef BRM_VERBOSE + cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << + " got lbid " << lbid << endl; + cerr << " lbid should be " << tmp->LBIDstart + offset << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbid == static_cast(tmp->LBIDstart + offset)); + em.checkConsistency(); + break; + } + + case 5: //getHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, status; + struct EMEntries* tmp; + uint32_t hwm; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + err = brm->getLocalHWM(tmp->OID, partNum, segmentNum, hwm, status); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); +#ifdef BRM_VERBOSE + cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM + << " BRM says it's " << hwm << endl; +#endif + CPPUNIT_ASSERT(hwm == tmp->HWM); + em.checkConsistency(); + break; + } + + case 6: //setHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, hwm, oid, err; + struct EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + hwm = rand_r(&randstate) % (tmp->FBO + brm->getExtentSize()); + err = brm->setLocalHWM(oid, tmp->partNum, tmp->segNum, hwm); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); + + for (tmp = head; tmp != NULL; tmp = tmp->next) + if (tmp->OID == oid) + tmp->HWM = hwm; + +#ifdef BRM_VERBOSE + cerr << "setHWM of OID " << oid << " to " << hwm << endl; +#endif + em.checkConsistency(); + break; + } + + case 7: // renew this EM object + { + delete brm; + brm = new DBRM(); +#ifdef BRM_VERBOSE + cerr << "got a new BRM instance" << endl; +#endif + em.checkConsistency(); + break; + } +#if 0 + case 8: //getBulkInsertVars + { + if (listSize == 0) + break; + + HWM_t hwm; + VER_t txnID; + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + lbid = tmp->LBIDstart + offset; + err = brm->getBulkInsertVars(lbid, hwm, txnID); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == tmp->secondHWM); + CPPUNIT_ASSERT(txnID == tmp->txnID); + break; + } + + case 9: //setBulkInsertVars + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + tmp->secondHWM = rand_r(&randstate) % MAXINT; + tmp->txnID = rand_r(&randstate) % MAXINT; + err = brm->setBulkInsertVars(tmp->LBIDstart + offset, + tmp->secondHWM, tmp->txnID); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); + break; + } +#endif + default: + break; + } + } + + delete brm; + + while (head != NULL) + { + tmp = head->next; + delete head; + head = tmp; + } + +#ifdef BRM_VERBOSE + cerr << "thread " << threadNum << " exiting" << endl; +#endif + return NULL; +} +*/ +DBRM brm_si; +/* +static void* BRMRunner_si(void* arg) +{ + + // keep track of LBID ranges allocated here and + // randomly allocate, lookup, delete, get/set HWM, and + // destroy the EM object. + +#ifdef BRM_VERBOSE + int threadNum = reinterpret_cast(arg); +#endif + int op, listSize = 0, i; + uint32_t randstate; + struct EMEntries* head = NULL, *tmp; + struct timeval tv; + ExtentMap em; + vector lbids; + +#ifdef BRM_VERBOSE + cerr << "thread number " << threadNum << " started." << endl; +#endif + + gettimeofday(&tv, NULL); + randstate = static_cast(tv.tv_usec); + + while (!threadStop) + { + op = rand_r(&randstate) % 10; +#ifdef BRM_VERBOSE + cerr << "next op is " << op << endl; +#endif + + switch (op) + { + case 0: //allocate space for a new file + { + struct EMEntries* newEm; + int size = rand_r(&randstate) % 102399 + 1; + int entries, OID, allocdSize, err; + + pthread_mutex_lock(&pthreadMutex); + OID = oid++; + opCount++; + pthread_mutex_unlock(&pthreadMutex); + + err = brm_si.createExtent(size, OID, lbids, allocdSize); + CPPUNIT_ASSERT(err == 0); + + entries = size / brm_si.getExtentSize(); + + if ((size % brm_si.getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0 ; i < entries; i++) + { + + newEm = new EMEntries(); + newEm->size = brm_si.getExtentSize(); + newEm->OID = OID; + newEm->FBO = i * brm_si.getExtentSize(); + newEm->LBIDstart = lbids[i]; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created new space for OID " << newEm->OID << endl; +#endif + em.checkConsistency(); + break; + } + + case 1: //allocate space for an existing file + { + if (listSize == 0) + break; + + struct EMEntries* newEm, *tmp; + int size = rand_r(&randstate) % 102399 + 1; + int fileRand = rand_r(&randstate) % listSize; + int i, lastExtent, blockEnd, oid; + int tmpHWM, entries, allocdSize, err; + vector lbids; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) + { + if (tmp->OID != oid) + continue; + + tmpHWM = tmp->HWM; + blockEnd = tmp->FBO + tmp->size; + + if (lastExtent < blockEnd) + lastExtent = blockEnd; + } + + err = brm_si.createExtent(size, oid, lbids, allocdSize); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); + + entries = size / brm_si.getExtentSize(); + + if ((size % brm_si.getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + + newEm = new EMEntries(); + + if (i != entries) + newEm->size = brm_si.getExtentSize(); + else + newEm->size = size % brm_si.getExtentSize(); + + newEm->OID = oid; + newEm->FBO = lastExtent + (i * brm_si.getExtentSize()); + newEm->LBIDstart = lbids[i]; + newEm->HWM = tmpHWM; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created another extent for OID " << newEm->OID << endl; +#endif + em.checkConsistency(); + break; + } + + case 2: //delete an OID + { + if (listSize == 0) + break; + + struct EMEntries* tmp, *prev; + int fileRand = rand_r(&randstate) % listSize; + int i, oid, err; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + err = brm_si.deleteOID(oid); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); + + for (tmp = head; tmp != NULL;) + { + if (tmp->OID == oid) + { + if (tmp == head) + { + head = head->next; + delete tmp; + tmp = head; + } + else + { + prev->next = tmp->next; + delete tmp; + tmp = prev->next; + } + + listSize--; + } + else + { + prev = tmp; + tmp = tmp->next; + } + } + +#ifdef BRM_VERBOSE + cerr << "deleted OID " << oid << endl; +#endif + em.checkConsistency(); + break; + } + + case 3: //lookup by LBID + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset, oid; + struct EMEntries* tmp; + LBID_t target; + uint32_t fbo; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + + target = tmp->LBIDstart + offset; + err = brm_si.lookup(target, 0, false, oid, fbo); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); +#ifdef BRM_VERBOSE + cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; + cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == tmp->OID); + CPPUNIT_ASSERT(fbo == offset + tmp->FBO); + em.checkConsistency(); + break; + } + + case 4: //lookup by OID, FBO + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, oid, err, offset; + struct EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + oid = tmp->OID; + + err = brm_si.lookup(oid, offset + tmp->FBO, lbid); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); +#ifdef BRM_VERBOSE + cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << + " got lbid " << lbid << endl; + cerr << " lbid should be " << tmp->LBIDstart + offset << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbid == static_cast(tmp->LBIDstart + offset)); + em.checkConsistency(); + break; + } + + case 5: //getHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err; + struct EMEntries* tmp; + uint32_t hwm; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + err = brm_si.getHWM(tmp->OID, hwm); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); +#ifdef BRM_VERBOSE + cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM + << " BRM says it's " << hwm << endl; +#endif + CPPUNIT_ASSERT(hwm == tmp->HWM); + em.checkConsistency(); + break; + } + + case 6: //setHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, hwm, oid, err; + struct EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + hwm = rand_r(&randstate) % (tmp->FBO + brm_si.getExtentSize()); + err = brm_si.setHWM(oid, hwm); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); + + for (tmp = head; tmp != NULL; tmp = tmp->next) + if (tmp->OID == oid) + tmp->HWM = hwm; + +#ifdef BRM_VERBOSE + cerr << "setHWM of OID " << oid << " to " << hwm << endl; +#endif + em.checkConsistency(); + break; + } + + case 7: //getBulkInsertVars + { + if (listSize == 0) + break; + + HWM_t hwm; + VER_t txnID; + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + lbid = tmp->LBIDstart + offset; + err = brm_si.getBulkInsertVars(lbid, hwm, txnID); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == tmp->secondHWM); + CPPUNIT_ASSERT(txnID == tmp->txnID); + break; + } + + case 8: //setBulkInsertVars + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + tmp->secondHWM = rand_r(&randstate) % MAXINT; + tmp->txnID = rand_r(&randstate) % MAXINT; + err = brm_si.setBulkInsertVars(tmp->LBIDstart + offset, + tmp->secondHWM, tmp->txnID); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); + break; + } + + default: + break; + } + } + + while (head != NULL) + { + tmp = head->next; + delete head; + head = tmp; + } + +#ifdef BRM_VERBOSE + cerr << "thread " << threadNum << " exiting" << endl; +#endif + return NULL; +} +*/ + +static void* EMRunner(void* arg) +{ + // keep track of LBID ranges allocated here and + // randomly allocate, lookup, delete, get/set HWM, and + // destroy the EM object. + +#ifdef BRM_VERBOSE + uint64_t threadNum = (uint64_t)arg; +#endif + int op, listSize = 0; + uint32_t randstate; + struct EMEntries* head = NULL, *tmp; + struct timeval tv; + ExtentMap* em; + LBID_t lbid; + uint32_t colWidth; + PartitionNumberT partNum; + SegmentT segmentNum; + execplan::CalpontSystemCatalog::ColDataType colDataType; + +#ifdef BRM_VERBOSE + cerr << "thread number " << threadNum << " started." << endl; +#endif + + gettimeofday(&tv, NULL); + randstate = static_cast(tv.tv_usec); + //pthread_mutex_lock(&pthreadMutex); + em = new ExtentMap(); + //pthread_mutex_unlock(&pthreadMutex); + + while (!threadStop) + { + auto randNumber = rand_r(&randstate); + op = randNumber % 10; + + colWidth = colWidthsAvailable[randNumber % colWidthsAvailable.size()]; + partNum = randNumber % std::numeric_limits::max(); + segmentNum = randNumber % std::numeric_limits::max(); + colDataType = (execplan::CalpontSystemCatalog::ColDataType) (randNumber % (int)execplan::CalpontSystemCatalog::ColDataType::TIMESTAMP); +#ifdef BRM_VERBOSE + cerr << "next op is " << op << endl; +#endif + + switch (op) + { + case 0: //allocate space for a new file + { + vector emEntriesVec; + struct EMEntries* newEm; + size_t numberOfExtents = randNumber % 4 + 1; + int OID; + uint32_t startBlockOffset; + + pthread_mutex_lock(&pthreadMutex); + OID = oid++; + pthread_mutex_unlock(&pthreadMutex); + + em->getExtents(OID, emEntriesVec, false, false, true); + size_t extentsNumberBefore = emEntriesVec.size(); + int allocdsize; + for (size_t i = 0; i < numberOfExtents; ++i) + { + em->createColumnExtent_DBroot(OID, colWidth, dbroot, colDataType, + partNum, segmentNum, lbid, allocdsize, startBlockOffset); + em->confirmChanges(); + + newEm = new EMEntries(allocdsize, OID, startBlockOffset, lbid, + head, dbroot, partNum, segmentNum); + head = newEm; + listSize++; + } + + emEntriesVec.clear(); + em->getExtents(OID, emEntriesVec, false, false, true); + size_t extentsNumberAfter = emEntriesVec.size(); + + CPPUNIT_ASSERT(extentsNumberBefore + numberOfExtents == extentsNumberAfter); + +#ifdef BRM_VERBOSE + cerr << "created new space for OID " << newEm->OID << endl; +#endif + //em->checkConsistency(); + break; + } +/* + case 1: //allocate space for an existing file + { + if (listSize == 0) + break; + + struct EMEntries* newEm, *tmp; + size_t size = rand_r(&randstate) % 10; + int fileRand = rand_r(&randstate) % listSize; + int i, lastExtent, blockEnd, oid; + int tmpHWM, entries, allocdSize; + uint32_t startBlockOffset; + lbids.clear(); + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) + { + if (tmp->OID != oid) + continue; + + tmpHWM = tmp->HWM; + blockEnd = tmp->FBO + tmp->size; + + if (lastExtent < blockEnd) + lastExtent = blockEnd; + } + + for (size_t i = 0; i < size; ++i) + { + em->createColumnExtent_DBroot(oid, colWidth, dbroot, colDataType, + partNum, segmentNum, lbid, allocdSize, startBlockOffset); + em->confirmChanges(); + lbids.push_back(lbid); + } + + //em->createExtent(size, oid, lbids, allocdSize); + //em->confirmChanges(); + + entries = size / em->getExtentSize(); + + if ((size % em->getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + newEm = new EMEntries((i != entries) ? em->getExtentSize() : size % em->getExtentSize(), + oid, lastExtent + (i * em->getExtentSize()), + lbids[i], head, dbroot, partNum, segmentNum); + newEm->HWM = tmpHWM; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created another extent for OID " << newEm->OID << endl; +#endif + em->checkConsistency(); + break; + } +*/ + + case 2: //delete an OID + { + if (listSize == 0) + break; + + struct EMEntries* tmp, *prev; + int fileRand = rand_r(&randstate) % listSize; + int i, oid; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + em->deleteOID(oid); + em->confirmChanges(); + + vector emEntriesVec; + em->getExtents(oid, emEntriesVec, false, false, true); + CPPUNIT_ASSERT(emEntriesVec.empty()); + + for (tmp = head; tmp != NULL;) + { + if (tmp->OID == oid) + { + if (tmp == head) + { + head = head->next; + delete tmp; + tmp = head; + } + else + { + prev->next = tmp->next; + delete tmp; + tmp = prev->next; + } + + listSize--; + } + else + { + prev = tmp; + tmp = tmp->next; + } + } + +#ifdef BRM_VERBOSE + cerr << "deleted OID " << oid << endl; +#endif + //em->checkConsistency(); + break; + } + + case 3: //lookup by LBID + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset, oid; + struct EMEntries* tmp; + LBID_t target; + uint32_t fbo; + DBRootT localDbroot; + PartitionNumberT localPartNum; + SegmentT localSegmentNum; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % (tmp->size - 1); + + target = tmp->LBIDstart + offset; + err = em->lookupLocal(target, oid, localDbroot, localPartNum, localSegmentNum, fbo); +#ifdef BRM_VERBOSE + cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; + cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; + cerr << "op 3 fbo " << fbo << " offset + tmp->FBO " << offset + tmp->FBO << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == tmp->OID); + CPPUNIT_ASSERT(fbo == offset + tmp->FBO); + //em->checkConsistency(); + break; + } + + case 4: //lookup by OID, FBO + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, oid, err, offset; + struct EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % (tmp->size - 1); + oid = tmp->OID; + + err = em->lookupLocal(oid, tmp->partNum, tmp->segNum, offset + tmp->FBO, lbid); +#ifdef BRM_VERBOSE + cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << + " got lbid " << lbid << endl; + cerr << " lbid should be " << tmp->LBIDstart + offset << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbid == tmp->LBIDstart + offset); + //em->checkConsistency(); + break; + } + + case 5: //getHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, status; + struct EMEntries* tmp; + uint32_t hwm; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + hwm = em->getLocalHWM(tmp->OID, tmp->partNum, tmp->segNum, status); +#ifdef BRM_VERBOSE_I + cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM + << " BRM says it's " << hwm << endl; +#endif + CPPUNIT_ASSERT(hwm == tmp->HWM); + //em->checkConsistency(); + break; + } + + case 6: //setHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, hwm, oid; + struct EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + hwm = rand_r(&randstate) % (tmp->size - 1); + bool firstNode = true; + em->setLocalHWM(oid, tmp->partNum, tmp->segNum, hwm, firstNode); + + em->confirmChanges(); + + tmp->HWM = hwm; + +#ifdef BRM_VERBOSE + cerr << "setHWM of OID " << oid << " to " << hwm << endl; +#endif + //em->checkConsistency(); + break; + } + +/* + case 7: // renew this EM object + { + delete em; + em = new ExtentMap(); +#ifdef BRM_VERBOSE + cerr << "got a new EM instance" << endl; +#endif + em->checkConsistency(); + break; + } + case 8: //getBulkInsertVars + { + if (listSize == 0) + break; + + HWM_t hwm; + VER_t txnID; + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + lbid = tmp->LBIDstart + offset; + err = em->getBulkInsertVars(lbid, hwm, txnID); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == tmp->secondHWM); + CPPUNIT_ASSERT(txnID == tmp->txnID); + break; + } + + case 9: //setBulkInsertVars + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + tmp->secondHWM = rand_r(&randstate) % MAXINT; + tmp->txnID = rand_r(&randstate) % MAXINT; + err = em->setBulkInsertVars(tmp->LBIDstart + offset, + tmp->secondHWM, tmp->txnID); + em->confirmChanges(); + CPPUNIT_ASSERT(err == 0); + break; + } +*/ + default: + break; + } + } + + delete em; + + while (head != NULL) + { + tmp = head->next; + delete head; + head = tmp; + } + +#ifdef BRM_VERBOSE + cerr << "thread " << threadNum << " exiting" << endl; +#endif + return NULL; +} + +/* +ExtentMap em_si; +static void* EMRunner_si(void* arg) +{ + + // keep track of LBID ranges allocated here and + // randomly allocate, lookup, delete, get/set HWM, and + // destroy the EM object. + +#ifdef BRM_VERBOSE + int threadNum = reinterpret_cast(arg); +#endif + int op, listSize = 0, i; + uint32_t randstate; + struct EMEntries* head = NULL, *tmp; + struct timeval tv; + vector lbids; + +#ifdef BRM_VERBOSE + cerr << "thread number " << threadNum << " started." << endl; +#endif + + gettimeofday(&tv, NULL); + randstate = static_cast(tv.tv_usec); + + while (!threadStop) + { + op = rand_r(&randstate) % 9; +#ifdef BRM_VERBOSE + cerr << "next op is " << op << endl; +#endif + + switch (op) + { + case 0: //allocate space for a new file + { + struct EMEntries* newEm; + int size = rand_r(&randstate) % 102399 + 1; + int entries, OID, allocdSize; + + pthread_mutex_lock(&pthreadMutex); + OID = oid++; + pthread_mutex_unlock(&pthreadMutex); + + em_si.createExtent(size, OID, lbids, allocdSize); + em_si.confirmChanges(); + + entries = size / em_si.getExtentSize(); + + if ((size % em_si.getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0 ; i < entries; i++) + { + + newEm = new EMEntries(); + newEm->size = em_si.getExtentSize(); + newEm->OID = OID; + newEm->FBO = i * em_si.getExtentSize(); + newEm->LBIDstart = lbids[i]; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created new space for OID " << newEm->OID << endl; +#endif + em_si.checkConsistency(); + break; + } + + case 1: //allocate space for an existing file + { + if (listSize == 0) + break; + + struct EMEntries* newEm, *tmp; + int size = rand_r(&randstate) % 102399 + 1; + int fileRand = rand_r(&randstate) % listSize; + int i, lastExtent, blockEnd, oid; + int tmpHWM, entries, allocdSize; + vector lbids; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) + { + if (tmp->OID != oid) + continue; + + tmpHWM = tmp->HWM; + blockEnd = tmp->FBO + tmp->size; + + if (lastExtent < blockEnd) + lastExtent = blockEnd; + } + + em_si.createExtent(size, oid, lbids, allocdSize); + em_si.confirmChanges(); + + entries = size / em_si.getExtentSize(); + + if ((size % em_si.getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + + newEm = new EMEntries(); + + if (i != entries) + newEm->size = em_si.getExtentSize(); + else + newEm->size = size % em_si.getExtentSize(); + + newEm->OID = oid; + newEm->FBO = lastExtent + (i * em_si.getExtentSize()); + newEm->LBIDstart = lbids[i]; + newEm->HWM = tmpHWM; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created another extent for OID " << newEm->OID << endl; +#endif + em_si.checkConsistency(); + break; + } + + case 2: //delete an OID + { + if (listSize == 0) + break; + + struct EMEntries* tmp, *prev; + int fileRand = rand_r(&randstate) % listSize; + int i, oid; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + em_si.deleteOID(oid); + em_si.confirmChanges(); + + for (tmp = head; tmp != NULL;) + { + if (tmp->OID == oid) + { + if (tmp == head) + { + head = head->next; + delete tmp; + tmp = head; + } + else + { + prev->next = tmp->next; + delete tmp; + tmp = prev->next; + } + + listSize--; + } + else + { + prev = tmp; + tmp = tmp->next; + } + } + +#ifdef BRM_VERBOSE + cerr << "deleted OID " << oid << endl; +#endif + em_si.checkConsistency(); + break; + } + + case 3: //lookup by LBID + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset, oid; + struct EMEntries* tmp; + LBID_t target; + uint32_t fbo; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + + target = tmp->LBIDstart + offset; + err = em_si.lookup(target, oid, fbo); +#ifdef BRM_VERBOSE + cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; + cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == tmp->OID); + CPPUNIT_ASSERT(fbo == offset + tmp->FBO); + em_si.checkConsistency(); + break; + } + + case 4: //lookup by OID, FBO + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, oid, err, offset; + struct EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + oid = tmp->OID; + + err = em_si.lookup(oid, offset + tmp->FBO, lbid); +#ifdef BRM_VERBOSE + cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << + " got lbid " << lbid << endl; + cerr << " lbid should be " << tmp->LBIDstart + offset << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbid == tmp->LBIDstart + offset); + em_si.checkConsistency(); + break; + } + + case 5: //getHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i; + struct EMEntries* tmp; + uint32_t hwm; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + hwm = em_si.getHWM(tmp->OID); +#ifdef BRM_VERBOSE + cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM + << " BRM says it's " << hwm << endl; +#endif + CPPUNIT_ASSERT(hwm == tmp->HWM); + em_si.checkConsistency(); + break; + } + + case 6: //setHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, hwm, oid; + struct EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + hwm = rand_r(&randstate) % (tmp->FBO + em_si.getExtentSize()); + em_si.setHWM(oid, hwm); + em_si.confirmChanges(); + + for (tmp = head; tmp != NULL; tmp = tmp->next) + if (tmp->OID == oid) + tmp->HWM = hwm; + +#ifdef BRM_VERBOSE + cerr << "setHWM of OID " << oid << " to " << hwm << endl; +#endif + em_si.checkConsistency(); + break; + } + + case 7: //getBulkInsertVars + { + if (listSize == 0) + break; + + HWM_t hwm; + VER_t txnID; + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + lbid = tmp->LBIDstart + offset; + err = em_si.getBulkInsertVars(lbid, hwm, txnID); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == tmp->secondHWM); + CPPUNIT_ASSERT(txnID == tmp->txnID); + break; + } + + case 8: //setBulkInsertVars + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + tmp->secondHWM = rand_r(&randstate) % MAXINT; + tmp->txnID = rand_r(&randstate) % MAXINT; + err = em_si.setBulkInsertVars(tmp->LBIDstart + offset, + tmp->secondHWM, tmp->txnID); + em_si.confirmChanges(); + CPPUNIT_ASSERT(err == 0); + break; + } + + default: + break; + } + } + + while (head != NULL) + { + tmp = head->next; + delete head; + head = tmp; + } + +#ifdef BRM_VERBOSE + cerr << "thread " << threadNum << " exiting" << endl; +#endif + return NULL; +} +*/ + + +class LongBRMTests : public CppUnit::TestFixture +{ + + CPPUNIT_TEST_SUITE(LongBRMTests); + + CPPUNIT_TEST(longEMTest_1); +// CPPUNIT_TEST(longEMTest_2); +// CPPUNIT_TEST(longBRMTest_1); +// CPPUNIT_TEST(longBRMTest_2); + + CPPUNIT_TEST_SUITE_END(); + +private: +public: + void longEMTest_1() + { + const int threadCount = 10; + int i; + pthread_t threads[threadCount]; + + cerr << endl << "Multithreaded, multiple instance ExtentMap test. " + "This runs for 5 minutes." << endl; + + threadStop = 0; + pthread_mutex_init(&pthreadMutex, nullptr); + + for (i = 0; i < threadCount; i++) + { + if (pthread_create(&threads[i], NULL, EMRunner, + reinterpret_cast(i + 1)) < 0) + throw logic_error("Error creating threads for the ExtentMap test"); + + usleep(1000); + } + + sleep(300); + threadStop = 1; + + for (i = 0; i < threadCount; i++) + { + cerr << "Waiting for thread #" << i << endl; + pthread_join(threads[i], nullptr); + } + } + +/* + void longEMTest_2() + { + const int threadCount = 10; + int i; + pthread_t threads[threadCount]; + + cerr << endl << "Multithreaded, single instance ExtentMap test. " + "This runs for 5 minutes." << endl; + + threadStop = 0; + pthread_mutex_init(&pthreadMutex, NULL); + + for (i = 0; i < threadCount; i++) + { + if (pthread_create(&threads[i], NULL, EMRunner_si, + reinterpret_cast(i + 1)) < 0) + throw logic_error("Error creating threads for the ExtentMap test"); + + usleep(1000); + } + + sleep(60); + threadStop = 1; + + for (i = 0; i < threadCount; i++) + { + cerr << "Waiting for thread #" << i << endl; + pthread_join(threads[i], NULL); + } + } + void longBRMTest_1() + { + const int threadCount = 10; + int i; + pthread_t threads[threadCount]; + + cerr << endl << "Multithreaded, multiple instance DBRM test. " + "This runs for 5 minutes." << endl; + + threadStop = 0; + pthread_mutex_init(&pthreadMutex, NULL); + opCount = 0; + + for (i = 0; i < threadCount; i++) + { + if (pthread_create(&threads[i], NULL, BRMRunner_1, + reinterpret_cast(i + 1)) < 0) + throw logic_error("Error creating threads for the DBRM test"); + + usleep(1000); + } + + sleep(300); + threadStop = 1; + + for (i = 0; i < threadCount; i++) + { + cerr << "Waiting for thread #" << i << endl; + pthread_join(threads[i], NULL); + } + + cerr << "opCount = " << opCount << endl; + } + void longBRMTest_2() + { + const int threadCount = 10; + int i; + pthread_t threads[threadCount]; + + cerr << endl << "Multithreaded, single instance DBRM test. " + "This runs for 5 minutes." << endl; + + threadStop = 0; + pthread_mutex_init(&pthreadMutex, NULL); + opCount = 0; + + for (i = 0; i < threadCount; i++) + { + if (pthread_create(&threads[i], NULL, BRMRunner_si, + reinterpret_cast(i + 1)) < 0) + throw logic_error("Error creating threads for the DBRM test"); + + usleep(1000); + } + + sleep(300); + threadStop = 1; + + for (i = 0; i < threadCount; i++) + { + cerr << "Waiting for thread #" << i << endl; + pthread_join(threads[i], NULL); + } + + cerr << "opCount = " << opCount << endl; + } +*/ + +}; +CPPUNIT_TEST_SUITE_REGISTRATION( LongBRMTests ); + +#include +#include + +int main( int argc, char** argv) +{ + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest( registry.makeTest() ); + idbdatafile::IDBPolicy::configIDBPolicy(); + bool wasSuccessful = runner.run( "", false ); + return (wasSuccessful ? 0 : 1); +} diff --git a/tests/primitives_column_scan_and_filter.cpp b/tests/primitives_column_scan_and_filter.cpp index 70f921b38..f017892d0 100644 --- a/tests/primitives_column_scan_and_filter.cpp +++ b/tests/primitives_column_scan_and_filter.cpp @@ -15,9 +15,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ +#include #include #include +#include "mcs_basic_types.h" #include "utils/common/columnwidth.h" #include "datatypes/mcs_datatype.h" #include "datatypes/mcs_int128.h" @@ -175,8 +177,10 @@ TEST_F(ColumnScanFilterTest, ColumnScan1ByteVectorized) for (i = 8034; i < 8160; ++i) EXPECT_EQ(results[i], i % 255 + 1); - EXPECT_EQ(out->Max, __col1block_cdf_umax); - EXPECT_EQ(out->Min, __col1block_cdf_umin); + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + EXPECT_EQ(expectedMax.getValue(), __col1block_cdf_umax); + EXPECT_EQ(expectedMin.getValue(), __col1block_cdf_umin); } TEST_F(ColumnScanFilterTest, ColumnScan2Bytes) @@ -200,8 +204,10 @@ TEST_F(ColumnScanFilterTest, ColumnScan2Bytes) for (i = 0; i < out->NVALS; i++) EXPECT_EQ(results[i], i); - EXPECT_EQ(out->Max, __col2block_cdf_umax); - EXPECT_EQ(out->Min, __col2block_cdf_umin); + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + EXPECT_EQ(expectedMax.getValue(), __col2block_cdf_umax); + EXPECT_EQ(expectedMin.getValue(), __col2block_cdf_umin); } TEST_F(ColumnScanFilterTest, ColumnScan4Bytes) @@ -224,8 +230,12 @@ TEST_F(ColumnScanFilterTest, ColumnScan4Bytes) for (i = 0; i < out->NVALS; i++) EXPECT_EQ(results[i], (uint32_t)i); - EXPECT_EQ(out->Max, __col4block_cdf_umax); - EXPECT_EQ(out->Min, __col4block_cdf_umin); + + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + + EXPECT_EQ(expectedMax.getValue(), __col4block_cdf_umax); + EXPECT_EQ(expectedMin.getValue(), __col4block_cdf_umin); } TEST_F(ColumnScanFilterTest, ColumnScan8Bytes) @@ -248,8 +258,11 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes) for (i = 0; i < out->NVALS; i++) ASSERT_EQ(results[i], (uint32_t)i); - EXPECT_EQ(out->Max, __col8block_cdf_umax); - EXPECT_EQ(out->Min, __col8block_cdf_umin); + + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + EXPECT_EQ(expectedMax.getValue(), __col8block_cdf_umax); + EXPECT_EQ(expectedMin.getValue(), __col8block_cdf_umin); } TEST_F(ColumnScanFilterTest, ColumnScan2Bytes1EqFilter) @@ -280,8 +293,11 @@ TEST_F(ColumnScanFilterTest, ColumnScan2Bytes1EqFilter) ASSERT_EQ(out->NVALS, 51); for (i = 0; i < out->NVALS; i++) ASSERT_EQ(results[i], i); - EXPECT_EQ(out->Max, __col2block_cdf_umax); - EXPECT_EQ(out->Min, __col2block_cdf_umin); + + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + EXPECT_EQ(expectedMax.getValue(), __col2block_cdf_umax); + EXPECT_EQ(expectedMin.getValue(), __col2block_cdf_umin); } TEST_F(ColumnScanFilterTest, ColumnScan1ByteUsingRID) @@ -432,8 +448,11 @@ TEST_F(ColumnScanFilterTest, ColumnScan4Bytes2Filters) ASSERT_EQ(results[i], 11 + (uint32_t)i); } - EXPECT_EQ(out->Max, __col4block_cdf_umax); - EXPECT_EQ(out->Min, __col4block_cdf_umin); + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + + EXPECT_EQ(expectedMax.getValue(), __col4block_cdf_umax); + EXPECT_EQ(expectedMin.getValue(), __col4block_cdf_umin); } TEST_F(ColumnScanFilterTest, ColumnScan8Bytes1EqFilter) @@ -465,8 +484,11 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes1EqFilter) for (i = 0; i < out->NVALS; i++) ASSERT_EQ(results[i], i); - EXPECT_EQ(out->Max, __col8block_cdf_umax); - EXPECT_EQ(out->Min, __col8block_cdf_umin); + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + + EXPECT_EQ(expectedMax.getValue(), __col8block_cdf_umax); + EXPECT_EQ(expectedMin.getValue(), __col8block_cdf_umin); } TEST_F(ColumnScanFilterTest, ColumnScan8BytesUsingMultipleRIDs) @@ -533,8 +555,11 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2CompFilters) for (i = 0; i < out->NVALS; i++) ASSERT_EQ(results[i], (uint32_t)(i < 10 ? i : i - 10 + 1001)); - EXPECT_EQ(out->Max, __col8block_cdf_umax); - EXPECT_EQ(out->Min, __col8block_cdf_umin); + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + + EXPECT_EQ(expectedMax.getValue(), __col8block_cdf_umax); + EXPECT_EQ(expectedMin.getValue(), __col8block_cdf_umin); } TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2EqFilters) @@ -569,8 +594,10 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2EqFilters) ASSERT_EQ(out->NVALS, 2); ASSERT_EQ(results[0], 10); ASSERT_EQ(results[1], 1000); - ASSERT_EQ(out->Max, __col8block_cdf_umax); - ASSERT_EQ(out->Min, __col8block_cdf_umin); + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + ASSERT_EQ(expectedMax.getValue(), __col8block_cdf_umax); + ASSERT_EQ(expectedMin.getValue(), __col8block_cdf_umin); } TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2EqFiltersRID) @@ -642,8 +669,11 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2FiltersRIDOutputRid) for (i = 0; i < out->NVALS; i++) ASSERT_EQ(results[i], (i < 10 ? i : i - 10 + 1001)); - ASSERT_EQ(out->Max, __col8block_cdf_umax); - ASSERT_EQ(out->Min, __col8block_cdf_umin); + + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + ASSERT_EQ(expectedMax.getValue(), __col8block_cdf_umax); + ASSERT_EQ(expectedMin.getValue(), __col8block_cdf_umin); } TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2EqFiltersRIDOutputBoth) @@ -680,8 +710,12 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2EqFiltersRIDOutputBoth) ASSERT_EQ(resultRid[i], (i < 10 ? i : i - 10 + 1001)); ASSERT_EQ(resultVal[i], (i < 10 ? i : i - 10 + 1001)); } - ASSERT_EQ(out->Max, __col8block_cdf_umax); - ASSERT_EQ(out->Min, __col8block_cdf_umin); + + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + + ASSERT_EQ(expectedMax.getValue(), __col8block_cdf_umax); + ASSERT_EQ(expectedMin.getValue(), __col8block_cdf_umin); } // void p_Col_12() @@ -926,8 +960,12 @@ TEST_F(ColumnScanFilterTest, ColumnScan16Bytes) ASSERT_EQ(results[0], NullValue); for (i = 1; i < out->NVALS; ++i) ASSERT_EQ(results[i], i + 1); - EXPECT_EQ(out->Max, __col16block_cdf_umax); - EXPECT_EQ(out->Min, __col16block_cdf_umin); + + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + + EXPECT_EQ(expectedMax.getValue(), __col16block_cdf_umax); + EXPECT_EQ(expectedMin.getValue(), __col16block_cdf_umin); } TEST_F(ColumnScanFilterTest, ColumnScan16Bytes2CompFilters) @@ -962,7 +1000,9 @@ TEST_F(ColumnScanFilterTest, ColumnScan16Bytes2CompFilters) ASSERT_EQ(results[0], 10); ASSERT_EQ(results[1], 510); - EXPECT_EQ(out->Max, __col16block_cdf_umax); - EXPECT_EQ(out->Min, __col16block_cdf_umin); + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + + EXPECT_EQ(expectedMax.getValue(), __col16block_cdf_umax); + EXPECT_EQ(expectedMin.getValue(), __col16block_cdf_umin); } -// vim:ts=2 sw=2: diff --git a/tests/primitives_scan_bench.cpp b/tests/primitives_scan_bench.cpp index 2d6b77c9a..c2431b8d9 100644 --- a/tests/primitives_scan_bench.cpp +++ b/tests/primitives_scan_bench.cpp @@ -427,4 +427,3 @@ BENCHMARK_DEFINE_F(FilterBenchFixture, BM_ColumnScan8Byte1FilterVectorizedCode)( BENCHMARK_REGISTER_F(FilterBenchFixture, BM_ColumnScan8Byte1FilterVectorizedCode); BENCHMARK_MAIN(); -// vim:ts=2 sw=2: diff --git a/tests/rowgroup-tests.cpp b/tests/rowgroup-tests.cpp index b1c6d5fa5..5073702ae 100644 --- a/tests/rowgroup-tests.cpp +++ b/tests/rowgroup-tests.cpp @@ -27,6 +27,7 @@ #define INITIAL_ROW_OFFSET 2 using CSCDataType = execplan::CalpontSystemCatalog::ColDataType; +using datatypes::TSInt128; class RowDecimalTest : public ::testing::Test { @@ -196,15 +197,15 @@ TEST_F(RowDecimalTest, InitToNullAndIsNullValueCheck) TEST_F(RowDecimalTest, GetBinaryFieldCheck) { rg.getRow(0, &r); - int128_t* a128Value; - int128_t* s128Value; + TSInt128 a128Value; + TSInt128 s128Value; for (size_t i = 0; i < sValueVector.size(); i++) { - s128Value = r.getBinaryField(0); - EXPECT_EQ(sValueVector[i], *s128Value); - a128Value = r.getBinaryField(1); - EXPECT_EQ(anotherValueVector[i], *a128Value); + s128Value = r.getTSInt128Field(0); + EXPECT_EQ(sValueVector[i], s128Value.getValue()); + a128Value = r.getTSInt128Field(1); + EXPECT_EQ(anotherValueVector[i], a128Value.getValue()); // EXPECT_EQ(s64ValueVector[i], r.getIntField(2)); // EXPECT_EQ(s32ValueVector[i],r.getIntField(3)); // EXPECT_EQ(r.getIntField(4),s16ValueVector[i]); @@ -245,22 +246,24 @@ TEST_F(RowDecimalTest, ApplyMappingCheck) int mapping[] = {0, 1, -1, -1, -1, -1}; rg.getRow(1, &r); rg.getRow(2, &rOutMappingCheck); - int128_t* s128Value = rOutMappingCheck.getBinaryField(0); - int128_t* a128Value = rOutMappingCheck.getBinaryField(1); - EXPECT_NE(sValueVector[1], *s128Value); - EXPECT_NE(anotherValueVector[1], *a128Value); + + auto s128Value = rOutMappingCheck.getTSInt128Field(0); + auto a128Value = rOutMappingCheck.getTSInt128Field(1); + EXPECT_NE(s128Value.getValue(), sValueVector[1]); + EXPECT_NE(a128Value.getValue(), anotherValueVector[1]); + applyMapping(mapping, r, &rOutMappingCheck); - s128Value = rOutMappingCheck.getBinaryField(0); - a128Value = rOutMappingCheck.getBinaryField(1); - EXPECT_EQ(sValueVector[1], *s128Value); - EXPECT_EQ(anotherValueVector[1], *a128Value); + s128Value = rOutMappingCheck.getTSInt128Field(0); + a128Value = rOutMappingCheck.getTSInt128Field(1); + EXPECT_EQ(sValueVector[1], s128Value.getValue()); + EXPECT_EQ(anotherValueVector[1], a128Value.getValue()); } TEST_F(RowDecimalTest, CopyBinaryFieldCheck) { int128_t constVal = 1; - int128_t *col1Out, *col1In; - int128_t *col2Out, *col2In; + TSInt128 col1Out, col1In; + TSInt128 col2Out, col2In; rgOut.getRow(0, &rOut); for (size_t i = 0; i < sValueVector.size(); i++) @@ -276,18 +279,18 @@ TEST_F(RowDecimalTest, CopyBinaryFieldCheck) for (size_t i = 0; i < sValueVector.size(); i++) { - col1In = r.getBinaryField(0); - col1Out = rOut.getBinaryField(0); - col2In = r.getBinaryField(1); - col2Out = rOut.getBinaryField(1); - EXPECT_NE(*col1In, *col1Out); - EXPECT_NE(*col2In, *col2Out); + col1In = r.getTSInt128Field(0); + col1Out = rOut.getTSInt128Field(0); + col2In = r.getTSInt128Field(1); + col2Out = rOut.getTSInt128Field(1); + EXPECT_NE(col1In, col1Out); + EXPECT_NE(col2In, col2Out); r.copyBinaryField(rOut, 0, 0); r.copyBinaryField(rOut, 1, 1); - col1Out = rOut.getBinaryField(0); - col2Out = rOut.getBinaryField(1); - EXPECT_EQ(*col1In, *col1Out); - EXPECT_EQ(*col2In, *col2Out); + col1Out = rOut.getTSInt128Field(0); + col2Out = rOut.getTSInt128Field(1); + EXPECT_EQ(col1In.getValue(), col1Out.getValue()); + EXPECT_EQ(col2In.getValue(), col2Out.getValue()); r.nextRow(rowSize); rOut.nextRow(rowSize); } diff --git a/tests/simd_processors.cpp b/tests/simd_processors.cpp new file mode 100644 index 000000000..8883a8743 --- /dev/null +++ b/tests/simd_processors.cpp @@ -0,0 +1,66 @@ +/* Copyright (C) 2022 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#if defined(__x86_64__) +#include +#include + +#include "simd_sse.h" +#include "datatypes/mcs_datatype.h" +#include "datatypes/mcs_int128.h" + +using namespace std; + +template +class SimdProcessorTypedTest : public testing::Test { + using IntegralType = T; + public: + + + void SetUp() override + { + } +}; + +using SimdProcessor128TypedTestTypes = ::testing::Types; +TYPED_TEST_SUITE(SimdProcessorTypedTest, SimdProcessor128TypedTestTypes); + +TYPED_TEST(SimdProcessorTypedTest, SimdFilterProcessor_simd128) +{ + using Proc = typename simd::SimdFilterProcessor; + using SimdType = typename Proc::SimdType; + constexpr static simd::MT allTrue = 0xFFFF; + constexpr static simd::MT allFalse = 0x0; + Proc proc; + SimdType lhs = proc.loadValue((TypeParam)-2); + SimdType rhs = proc.loadValue((TypeParam)-3); + EXPECT_GT((uint64_t)-2LL, (uint64_t)-3LL); + EXPECT_EQ(proc.cmpGe(lhs, rhs), allTrue); + EXPECT_EQ(proc.cmpGt(lhs, rhs), allTrue); + EXPECT_EQ(proc.cmpGe(rhs, lhs), allFalse); + EXPECT_EQ(proc.cmpGt(rhs, lhs), allFalse); + EXPECT_EQ(proc.cmpLe(rhs, lhs), allTrue); + EXPECT_EQ(proc.cmpLt(rhs, lhs), allTrue); + EXPECT_EQ(proc.cmpLe(lhs, rhs), allFalse); + EXPECT_EQ(proc.cmpLt(lhs, rhs), allFalse); + EXPECT_EQ(proc.cmpEq(rhs, lhs), allFalse); + EXPECT_EQ(proc.cmpNe(rhs, lhs), allTrue); + lhs = proc.loadValue((TypeParam)-3); + EXPECT_EQ(proc.cmpEq(lhs, rhs), allTrue); + EXPECT_EQ(proc.cmpNe(rhs, lhs), allFalse); +} +#endif \ No newline at end of file diff --git a/tools/bincvt/li2bin.cpp b/tools/bincvt/li2bin.cpp index 51b1d8b64..91bf9c4c0 100644 --- a/tools/bincvt/li2bin.cpp +++ b/tools/bincvt/li2bin.cpp @@ -278,4 +278,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/tools/clearShm/main.cpp b/tools/clearShm/main.cpp index 31e07b9a8..d31f9a75e 100644 --- a/tools/clearShm/main.cpp +++ b/tools/clearShm/main.cpp @@ -60,7 +60,7 @@ void shmDoit(key_t shm_key, const string& label) bi::offset_t memSize = 0; memObj.get_size(memSize); std::lock_guard lk(coutMutex); - cout << label << ": shm_key: " << shm_key << "; key_name: " << key_name << "; size: " << memSize + cout << label << ": shm|sem_key: " << shm_key << "; key_name: " << key_name << "; size: " << memSize << endl; } catch (...) @@ -74,6 +74,11 @@ void shmDoit(key_t shm_key, const string& label) } } +void semDoit(key_t sem_key, const string& label) +{ + shmDoit(sem_key, label); +} + void shmDoitRange(key_t shm_key, const string& label) { if (shm_key == 0) @@ -87,32 +92,6 @@ void shmDoitRange(key_t shm_key, const string& label) } } -void semDoit(key_t sem_key, const string& label) -{ - string key_name = ShmKeys::keyToName(sem_key); - - if (vFlg) - { - try - { - bi::shared_memory_object memObj(bi::open_only, key_name.c_str(), bi::read_only); - bi::offset_t memSize = 0; - memObj.get_size(memSize); - std::lock_guard lk(coutMutex); - cout << label << ": sem_key: " << sem_key << "; key_name: " << key_name << "; size: " << memSize - << endl; - } - catch (...) - { - } - } - - if (!nFlg) - { - bi::shared_memory_object::remove(key_name.c_str()); - } -} - void usage() { cout << "usage: clearShm [-cvnh]" << endl; @@ -206,6 +185,8 @@ int main(int argc, char** argv) tg.add_thread(tp); tp = new boost::thread(ThdFunc(BrmKeys.KEYRANGE_VSS_BASE, "VSS ")); tg.add_thread(tp); + tp = new boost::thread(ThdFunc(BrmKeys.KEYRANGE_EXTENTMAP_INDEX_BASE, "EXTMAP_INDX")); + tg.add_thread(tp); tg.join_all(); shmDoit(BrmKeys.MST_SYSVKEY, "MST "); @@ -226,6 +207,7 @@ int main(int argc, char** argv) semDoit(BrmKeys.KEYRANGE_EMFREELIST_BASE, "EXTMAP_FREE"); semDoit(BrmKeys.KEYRANGE_VBBM_BASE, "VBBM "); semDoit(BrmKeys.KEYRANGE_VSS_BASE, "VSS "); + semDoit(BrmKeys.KEYRANGE_EXTENTMAP_INDEX_BASE, "EXTMAP_INDX"); semDoit(BrmKeys.MST_SYSVKEY, "MST "); if (!cFlg) @@ -239,4 +221,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/tools/dbbuilder/dbbuilder.cpp b/tools/dbbuilder/dbbuilder.cpp index 38b9ffd54..79d2a4beb 100644 --- a/tools/dbbuilder/dbbuilder.cpp +++ b/tools/dbbuilder/dbbuilder.cpp @@ -324,4 +324,3 @@ int main(int argc, char* argv[]) return 1; } -// vim:ts=4 sw=4: diff --git a/tools/ddlcleanup/ddlcleanup.cpp b/tools/ddlcleanup/ddlcleanup.cpp index bbbbac437..6d1ca008a 100644 --- a/tools/ddlcleanup/ddlcleanup.cpp +++ b/tools/ddlcleanup/ddlcleanup.cpp @@ -80,4 +80,3 @@ int main(int argc, char** argv) return ddlcleanuputil::ddl_cleanup(); } -// vim:ts=4 sw=4: diff --git a/tools/editem/editem.cpp b/tools/editem/editem.cpp index 2dcb6ce81..bce63ed0e 100644 --- a/tools/editem/editem.cpp +++ b/tools/editem/editem.cpp @@ -1040,4 +1040,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/tools/passwd/CMakeLists.txt b/tools/passwd/CMakeLists.txt index cc9e7e330..b4a72c057 100644 --- a/tools/passwd/CMakeLists.txt +++ b/tools/passwd/CMakeLists.txt @@ -1,16 +1,17 @@ -include_directories( ${ENGINE_COMMON_INCLUDES} ) +include_directories( ${ENGINE_COMMON_INCLUDES} ${ENGINE_UTILS_COMMON_INCLUDE} ) ########### next target ############### - set(cspasswd_SRCS cspasswd.cpp secrets.cpp) set(cskeys_SRCS cskeys.cpp secrets.cpp) add_executable(cspasswd ${cspasswd_SRCS}) add_executable(cskeys ${cskeys_SRCS}) -target_link_libraries(cspasswd ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) -target_link_libraries(cskeys ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) +target_include_directories(cspasswd BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR}) +target_include_directories(cskeys BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR}) +target_link_libraries(cspasswd ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES}) +target_link_libraries(cskeys ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES}) install(TARGETS cspasswd DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) install(TARGETS cskeys DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) diff --git a/tools/passwd/cskeys.cpp b/tools/passwd/cskeys.cpp index 181503605..f0d61959c 100644 --- a/tools/passwd/cskeys.cpp +++ b/tools/passwd/cskeys.cpp @@ -18,10 +18,17 @@ #include #include #include +#include #include "secrets.h" #include "mcsconfig.h" +#ifdef OPENSSL_VERSION_PREREQ +#if OPENSSL_VERSION_PREREQ(3,0) + #define EVP_CIPHER_key_length EVP_CIPHER_get_key_length +#endif +#endif + using std::string; using ByteVec = std::vector; diff --git a/tools/passwd/secrets.cpp b/tools/passwd/secrets.cpp index a7635e880..179d38025 100644 --- a/tools/passwd/secrets.cpp +++ b/tools/passwd/secrets.cpp @@ -21,10 +21,10 @@ #include #include #include +#include + +#include "utils/json/json.hpp" -#define BOOST_SPIRIT_THREADSAFE -#include -#include #include "idberrorinfo.h" #include "logger.h" #include "mcsconfig.h" @@ -34,6 +34,15 @@ using std::string; + +#ifdef OPENSSL_VERSION_PREREQ +#if OPENSSL_VERSION_PREREQ(3,0) + #define EVP_CIPHER_key_length EVP_CIPHER_get_key_length + #define EVP_CIPHER_iv_length EVP_CIPHER_get_iv_length + #define EVP_CIPHER_blocksize EVP_CIPHER_get_blocksize +#endif +#endif + const char* const SECRETS_FILENAME = ".secrets"; namespace @@ -383,16 +392,16 @@ ReadKeyResult secrets_readkeys(const string& filepath) { // File contents should be json. // json_error_t err; - boost::property_tree::ptree jsontree; + nlohmann::json jsontree; try { - boost::property_tree::read_json(filepath, jsontree); + std::ifstream i(filepath); + jsontree = nlohmann::json::parse(i); } - catch (boost::property_tree::json_parser::json_parser_error& je) + catch (const nlohmann::json::exception& je) { - std::cout << "Error reading JSON from secrets file: " << je.filename() << " on line: " << je.line() - << std::endl; - std::cout << je.message() << std::endl; + std::cout << "Error reading JSON from secrets file: " << filepath << std::endl; + std::cout << je.what() << std::endl; } catch (...) { @@ -400,8 +409,8 @@ ReadKeyResult secrets_readkeys(const string& filepath) strerror(errno)); } // json_t* obj = json_load_file(filepathc, 0, &err); - string enc_cipher = jsontree.get(field_cipher); - string enc_key = jsontree.get(field_key); + string enc_cipher = jsontree[field_cipher]; + string enc_key = jsontree[field_key]; // const char* enc_cipher = json_string_value(json_object_get(obj, field_cipher)); // const char* enc_key = json_string_value(json_object_get(obj, field_key)); bool cipher_ok = !enc_cipher.empty() && (enc_cipher == CIPHER_NAME); @@ -469,6 +478,8 @@ string decrypt_password(const string& input) * @param input Encrypted password in hex form * @return Decrypted password or empty on error */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" string decrypt_password_old(const ByteVec& key, const ByteVec& iv, const std::string& input) { string rval; @@ -501,6 +512,7 @@ string decrypt_password_old(const ByteVec& key, const ByteVec& iv, const std::st } return rval; } +#pragma GCC diagnostic pop string decrypt_password(const ByteVec& key, const std::string& input) { @@ -635,23 +647,24 @@ bool secrets_write_keys(const ByteVec& key, const string& filepath, const string utils::VLArray key_hex(2 * keylen + 1); bin2hex(key.data(), keylen, key_hex.data()); - boost::property_tree::ptree jsontree; - jsontree.put(field_desc, desc); - jsontree.put(field_version, columnstore_version.c_str()); - jsontree.put(field_cipher, CIPHER_NAME); - jsontree.put(field_key, (const char*)key_hex.data()); + nlohmann::json jsontree; + jsontree[field_desc] = desc; + jsontree[field_version] = columnstore_version; + jsontree[field_cipher] = CIPHER_NAME; + jsontree[field_key] = (const char*)key_hex.data(); auto filepathc = filepath.c_str(); bool write_ok = false; errno = 0; try { - write_json(filepathc, jsontree); + std::ofstream o(filepath); + o << jsontree; } - catch (boost::property_tree::json_parser::json_parser_error& je) + catch (const nlohmann::json::exception& je) { - std::cout << "Write to secrets file: " << je.filename() << " on line: " << je.line() << std::endl; - std::cout << je.message() << std::endl; + std::cout << "Write to secrets file: " << filepath << std::endl; + std::cout << je.what() << std::endl; } catch (...) { diff --git a/tools/qfe/server.cpp b/tools/qfe/server.cpp index 018cf470a..7b3276ded 100644 --- a/tools/qfe/server.cpp +++ b/tools/qfe/server.cpp @@ -513,4 +513,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/tools/rebuildEM/CMakeLists.txt b/tools/rebuildEM/CMakeLists.txt index 54bd14e6b..5d27c2f00 100644 --- a/tools/rebuildEM/CMakeLists.txt +++ b/tools/rebuildEM/CMakeLists.txt @@ -2,5 +2,5 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(rebuildEM_SRCS main.cpp rebuildEM.cpp) add_executable(mcsRebuildEM ${rebuildEM_SRCS}) -target_link_libraries(mcsRebuildEM ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${MARIADB_CLIENT_LIBS} boost_filesystem boost_system) +target_link_libraries(mcsRebuildEM ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${MARIADB_CLIENT_LIBS} Boost::system Boost::filesystem) install(TARGETS mcsRebuildEM DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) diff --git a/tools/sendPlan/sendplan.cpp b/tools/sendPlan/sendplan.cpp index 5374d8676..bd8f4bb4e 100644 --- a/tools/sendPlan/sendplan.cpp +++ b/tools/sendPlan/sendplan.cpp @@ -390,4 +390,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/tools/setConfig/main.cpp b/tools/setConfig/main.cpp index 1c536167f..7d7fe396e 100644 --- a/tools/setConfig/main.cpp +++ b/tools/setConfig/main.cpp @@ -148,4 +148,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/utils/cacheutils/cacheutils.cpp b/utils/cacheutils/cacheutils.cpp index ead2483ad..a0042e155 100644 --- a/utils/cacheutils/cacheutils.cpp +++ b/utils/cacheutils/cacheutils.cpp @@ -354,4 +354,3 @@ int purgePrimProcFdCache(const std::vector files, const int pmId) } } // namespace cacheutils -// vim:ts=4 sw=4: diff --git a/utils/cacheutils/cacheutils.h b/utils/cacheutils/cacheutils.h index 013ef93c3..23624e521 100644 --- a/utils/cacheutils/cacheutils.h +++ b/utils/cacheutils/cacheutils.h @@ -72,4 +72,3 @@ int dropPrimProcFdCache(); int purgePrimProcFdCache(const std::vector files, const int pmId); } // namespace cacheutils -// vim:ts=4 sw=4: diff --git a/utils/common/CMakeLists.txt b/utils/common/CMakeLists.txt index 137ddcb06..59e072ec1 100644 --- a/utils/common/CMakeLists.txt +++ b/utils/common/CMakeLists.txt @@ -11,11 +11,12 @@ set(common_LIB_SRCS nullvaluemanip.cpp threadnaming.cpp utils_utf8.cpp - statistics.cpp) + statistics.cpp + string_prefixes.cpp) add_library(common SHARED ${common_LIB_SRCS}) -target_link_libraries(common boost_filesystem) +target_link_libraries(common Boost::filesystem) add_dependencies(common loggingcpp) diff --git a/utils/common/cgroupconfigurator.cpp b/utils/common/cgroupconfigurator.cpp index 65b8b1139..45dcc1e26 100644 --- a/utils/common/cgroupconfigurator.cpp +++ b/utils/common/cgroupconfigurator.cpp @@ -20,13 +20,8 @@ #include "logger.h" #include #include -#include -#ifdef _MSC_VER -#include "unistd.h" -#include "sysinfo.h" -#else #include -#endif + using namespace boost; using namespace std; diff --git a/utils/common/collation.h b/utils/common/collation.h index f80fa1004..721c98d16 100644 --- a/utils/common/collation.h +++ b/utils/common/collation.h @@ -1,5 +1,5 @@ /* - Copyright (C) 2020 MariaDB Corporation + Copyright (C) 2020-2022 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -88,6 +88,8 @@ typedef double pfloat; /* Mixed prototypes can't take float */ typedef const struct charset_info_st CHARSET_INFO; extern "C" MYSQL_PLUGIN_IMPORT CHARSET_INFO* default_charset_info; +#define HAVE_PSI_INTERFACE + #include "m_ctype.h" #undef FALSE @@ -133,6 +135,8 @@ class Charset { protected: const struct charset_info_st* mCharset; + private: + static constexpr uint flags_ = MY_STRXFRM_PAD_WITH_SPACE | MY_STRXFRM_PAD_TO_MAXLEN; public: Charset(CHARSET_INFO& cs) : mCharset(&cs) @@ -177,6 +181,34 @@ class Charset bool res = !mCharset->wildcmp(subject.str(), subject.end(), pattern.str(), pattern.end(), '\\', '_', '%'); return neg ? !res : res; } + size_t strnxfrm(uchar* dst, size_t dstlen, uint nweights, const uchar* src, size_t srclen, uint flags) + { + idbassert(mCharset->coll); + return mCharset->coll->strnxfrm(mCharset, dst, dstlen, nweights, src, srclen, flags); + } + // The magic check that tells that bytes are mapped to weights as 1:1 + bool strnxfrmIsValid() const + { + return (mCharset->state & MY_CS_NON1TO1) == 0; + } + template + T strnxfrm(const char* src) const + { + T ret = 0; + size_t len __attribute__((unused)) = mCharset->strnxfrm((char*)&ret, sizeof(T), sizeof(T), + src, sizeof(T), flags_); + assert(len <= sizeof(T)); + return ret; + } + template + T strnxfrm(const utils::ConstString &src) const + { + T ret = 0; + size_t len __attribute__((unused)) = mCharset->strnxfrm((char*)&ret, sizeof(T), sizeof(T), + (char*)src.str(), src.length(), flags_); + assert(len <= sizeof(T)); + return ret; + } }; class CollationAwareHasher : public Charset diff --git a/utils/common/hashfamily.h b/utils/common/hashfamily.h index fa86b7a15..4b09e01a8 100644 --- a/utils/common/hashfamily.h +++ b/utils/common/hashfamily.h @@ -50,4 +50,3 @@ class HashFamily }; } // namespace utils -// vim:ts=2 sw=2: diff --git a/utils/common/mcs_basic_types.h b/utils/common/mcs_basic_types.h index ff64c6a00..3e877ed24 100644 --- a/utils/common/mcs_basic_types.h +++ b/utils/common/mcs_basic_types.h @@ -21,4 +21,3 @@ using int128_t = __int128; using uint128_t = unsigned __int128; -// vim:ts=2 sw=2: diff --git a/utils/common/nullvaluemanip.cpp b/utils/common/nullvaluemanip.cpp index da2fd76e6..76e373511 100644 --- a/utils/common/nullvaluemanip.cpp +++ b/utils/common/nullvaluemanip.cpp @@ -198,6 +198,7 @@ int64_t getSignedNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidt os << "getSignedNullValue(): got bad column type (" << t << "). Width=" << colWidth << endl; throw logic_error(os.str()); } + } } // namespace utils diff --git a/utils/common/simd_sse.h b/utils/common/simd_sse.h index 9815029c7..d6407f58e 100644 --- a/utils/common/simd_sse.h +++ b/utils/common/simd_sse.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2021 Mariadb Corporation. +/* Copyright (C) 2021-2022 Mariadb Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -17,6 +17,17 @@ #pragma once +// Column filtering is dispatched 4-way based on the column type, +// which defines implementation of comparison operations for the column values +enum ENUM_KIND +{ + KIND_DEFAULT, // compared as signed integers + KIND_UNSIGNED, // compared as unsigned integers + KIND_FLOAT, // compared as floating-point numbers + KIND_TEXT +}; // whitespace-trimmed and then compared as signed integers + + #if defined(__x86_64__) #include @@ -36,16 +47,6 @@ #include -// Column filtering is dispatched 4-way based on the column type, -// which defines implementation of comparison operations for the column values -enum ENUM_KIND -{ - KIND_DEFAULT, // compared as signed integers - KIND_UNSIGNED, // compared as unsigned integers - KIND_FLOAT, // compared as floating-point numbers - KIND_TEXT -}; // whitespace-trimmed and then compared as signed integers - namespace simd { using vi128_t = __m128i; @@ -115,6 +116,15 @@ struct StorageToFiltering:: using type = T; }; +template +static inline vi128_t constant4i() { + static const union { + int i[4]; + vi128_t xmm; + } u = {{i0,i1,i2,i3}}; + return u.xmm; +} + template class SimdFilterProcessor; @@ -461,7 +471,7 @@ class SimdFilterProcessor< template class SimdFilterProcessor::value && sizeof(CHECK_T) == 8 && + typename std::enable_if::value && std::is_same::value && !std::is_same::value>::type> { public: @@ -568,7 +578,117 @@ class SimdFilterProcessor class SimdFilterProcessor::value && sizeof(CHECK_T) == 4 && + typename std::enable_if::value && std::is_same::value && + !std::is_same::value>::type> +{ + public: + constexpr static const uint16_t vecByteSize = 16U; + constexpr static const uint16_t vecBitSize = 128U; + using T = typename datatypes::WidthToSIntegralType::type; + using SimdWrapperType = vi128_wr; + using SimdType = vi128_t; + using FilterType = T; + using StorageType = T; + // Mask calculation for int and float types differs. + // See corresponding intrinsics algos for details. + constexpr static const uint16_t FilterMaskStep = sizeof(T); + // Load value + MCS_FORCE_INLINE SimdType emptyNullLoadValue(const T fill) + { + return loadValue(fill); + } + + MCS_FORCE_INLINE SimdType loadValue(const T fill) + { + return _mm_set_epi64x(fill, fill); + } + + // Load from + MCS_FORCE_INLINE SimdType loadFrom(const char* from) + { + return _mm_loadu_si128(reinterpret_cast(from)); + } + + // Compare + MCS_FORCE_INLINE MT cmpGe(SimdType& x, SimdType& y) + { + return cmpGt(y, x) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpGt(SimdType& x, SimdType& y) + { + SimdType signVec = constant4i<0,(int32_t)0x80000000,0,(int32_t)0x80000000>(); + SimdType xFlip = _mm_xor_si128(x, signVec); + SimdType yFlip = _mm_xor_si128(y, signVec); + return _mm_movemask_epi8(_mm_cmpgt_epi64(xFlip, yFlip)); + } + + MCS_FORCE_INLINE MT cmpEq(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi64(x, y)); + } + + MCS_FORCE_INLINE MT cmpLe(SimdType& x, SimdType& y) + { + return cmpGt(x, y) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpLt(SimdType& x, SimdType& y) + { + return cmpGt(y, x); + } + + MCS_FORCE_INLINE MT cmpNe(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi64(x, y)) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpAlwaysFalse(SimdType& x, SimdType& y) + { + return 0; + } + + MCS_FORCE_INLINE MT cmpAlwaysTrue(SimdType& x, SimdType& y) + { + return 0xFFFF; + } + + // misc + MCS_FORCE_INLINE MT convertVectorToBitMask(SimdType& vmask) + { + return _mm_movemask_epi8(vmask); + } + + MCS_FORCE_INLINE SimdType setToZero() + { + return _mm_setzero_si128(); + } + + MCS_FORCE_INLINE MT nullEmptyCmpNe(SimdType& x, SimdType& y) + { + return cmpNe(x, y); + } + + MCS_FORCE_INLINE MT nullEmptyCmpEq(SimdType& x, SimdType& y) + { + return cmpEq(x, y); + } + + // store + MCS_FORCE_INLINE void storeWMask(SimdType& x, SimdType& vmask, char* dst) + { + _mm_maskmoveu_si128(x, vmask, dst); + } + + MCS_FORCE_INLINE void store(char* dst, SimdType& x) + { + _mm_storeu_si128(reinterpret_cast(dst), x); + } +}; + +template +class SimdFilterProcessor::value && std::is_same::value && !std::is_same::value>::type> { public: @@ -673,9 +793,119 @@ class SimdFilterProcessor +class SimdFilterProcessor::value && std::is_same::value && + !std::is_same::value>::type> +{ + public: + constexpr static const uint16_t vecByteSize = 16U; + constexpr static const uint16_t vecBitSize = 128U; + using T = typename datatypes::WidthToSIntegralType::type; + using SimdWrapperType = vi128_wr; + using SimdType = vi128_t; + using FilterType = T; + using StorageType = T; + // Mask calculation for int and float types differs. + // See corresponding intrinsics algos for details. + constexpr static const uint16_t FilterMaskStep = sizeof(T); + // Load value + MCS_FORCE_INLINE SimdType emptyNullLoadValue(const T fill) + { + return loadValue(fill); + } + + MCS_FORCE_INLINE SimdType loadValue(const T fill) + { + return _mm_set1_epi32(fill); + } + + // Load from + MCS_FORCE_INLINE SimdType loadFrom(const char* from) + { + return _mm_loadu_si128(reinterpret_cast(from)); + } + + // Compare + MCS_FORCE_INLINE MT cmpEq(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi32(x, y)); + } + + MCS_FORCE_INLINE MT cmpGe(SimdType& x, SimdType& y) + { + return cmpGt(y, x) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpGt(SimdType& x, SimdType& y) + { + SimdType signVec = constant4i<(int32_t)0x80000000,(int32_t)0x80000000,(int32_t)0x80000000,(int32_t)0x80000000>(); + SimdType xFlip = _mm_xor_si128(x, signVec); + SimdType yFlip = _mm_xor_si128(y, signVec); + return _mm_movemask_epi8(_mm_cmpgt_epi32(xFlip, yFlip)); + } + + MCS_FORCE_INLINE MT cmpLe(SimdType& x, SimdType& y) + { + return cmpGt(x, y) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpLt(SimdType& x, SimdType& y) + { + return cmpGt(y, x); + } + + MCS_FORCE_INLINE MT cmpNe(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi32(x, y)) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpAlwaysFalse(SimdType& x, SimdType& y) + { + return 0; + } + + MCS_FORCE_INLINE MT cmpAlwaysTrue(SimdType& x, SimdType& y) + { + return 0xFFFF; + } + + // misc + MCS_FORCE_INLINE MT convertVectorToBitMask(SimdType& vmask) + { + return _mm_movemask_epi8(vmask); + } + + MCS_FORCE_INLINE MT nullEmptyCmpNe(SimdType& x, SimdType& y) + { + return cmpNe(x, y); + } + + MCS_FORCE_INLINE MT nullEmptyCmpEq(SimdType& x, SimdType& y) + { + return cmpEq(x, y); + } + + MCS_FORCE_INLINE SimdType setToZero() + { + return _mm_setzero_si128(); + } + + // store + MCS_FORCE_INLINE void storeWMask(SimdType& x, SimdType& vmask, char* dst) + { + _mm_maskmoveu_si128(x, vmask, dst); + } + + MCS_FORCE_INLINE void store(char* dst, SimdType& x) + { + _mm_storeu_si128(reinterpret_cast(dst), x); + } +}; + template class SimdFilterProcessor< - VT, CHECK_T, typename std::enable_if::value && sizeof(CHECK_T) == 2>::type> + VT, CHECK_T, typename std::enable_if::value && std::is_same::value>::type> { public: constexpr static const uint16_t vecByteSize = 16U; @@ -781,7 +1011,227 @@ class SimdFilterProcessor< template class SimdFilterProcessor< - VT, CHECK_T, typename std::enable_if::value && sizeof(CHECK_T) == 1>::type> + VT, CHECK_T, typename std::enable_if::value && std::is_same::value>::type> +{ + public: + constexpr static const uint16_t vecByteSize = 16U; + constexpr static const uint16_t vecBitSize = 128U; + using T = typename datatypes::WidthToSIntegralType::type; + using SimdWrapperType = simd::vi128_wr; + using SimdType = simd::vi128_t; + using FilterType = T; + using StorageType = T; + // Mask calculation for int and float types differs. + // See corresponding intrinsics algos for details. + constexpr static const uint16_t FilterMaskStep = sizeof(T); + // Load value + MCS_FORCE_INLINE SimdType emptyNullLoadValue(const T fill) + { + return loadValue(fill); + } + + MCS_FORCE_INLINE SimdType loadValue(const T fill) + { + return _mm_set1_epi16(fill); + } + + // Load from + MCS_FORCE_INLINE SimdType loadFrom(const char* from) + { + return _mm_loadu_si128(reinterpret_cast(from)); + } + + // Compare + MCS_FORCE_INLINE MT cmpEq(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi16(x, y)); + } + + MCS_FORCE_INLINE MT cmpGe(SimdType& x, SimdType& y) + { + SimdType maxOfTwo = _mm_max_epu16(x, y); // max(x, y), unsigned + return _mm_movemask_epi8(_mm_cmpeq_epi16(x, maxOfTwo)); + } + + MCS_FORCE_INLINE MT cmpGt(SimdType& x, SimdType& y) + { + return cmpGe(y, x) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpLe(SimdType& x, SimdType& y) + { + return cmpGe(y, x); + } + + MCS_FORCE_INLINE MT cmpLt(SimdType& x, SimdType& y) + { + return cmpGe(x, y) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpNe(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi16(x, y)) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpAlwaysFalse(SimdType& x, SimdType& y) + { + return 0; + } + + MCS_FORCE_INLINE MT cmpAlwaysTrue(SimdType& x, SimdType& y) + { + return 0xFFFF; + } + + // misc + MCS_FORCE_INLINE MT convertVectorToBitMask(SimdType& vmask) + { + return _mm_movemask_epi8(vmask); + } + + MCS_FORCE_INLINE MT nullEmptyCmpNe(SimdType& x, SimdType& y) + { + return cmpNe(x, y); + } + + MCS_FORCE_INLINE MT nullEmptyCmpEq(SimdType& x, SimdType& y) + { + return cmpEq(x, y); + } + + MCS_FORCE_INLINE SimdType setToZero() + { + return _mm_setzero_si128(); + } + + // store + MCS_FORCE_INLINE void storeWMask(SimdType& x, SimdType& vmask, char* dst) + { + _mm_maskmoveu_si128(x, vmask, dst); + } + + MCS_FORCE_INLINE void store(char* dst, SimdType& x) + { + _mm_storeu_si128(reinterpret_cast(dst), x); + } +}; + +template +class SimdFilterProcessor< + VT, CHECK_T, typename std::enable_if::value && std::is_same::value>::type> +{ + public: + constexpr static const uint16_t vecByteSize = 16U; + constexpr static const uint16_t vecBitSize = 128U; + using T = typename datatypes::WidthToSIntegralType::type; + using SimdWrapperType = vi128_wr; + using SimdType = vi128_t; + using FilterType = T; + using StorageType = T; + // Mask calculation for int and float types differs. + // See corresponding intrinsics algos for details. + constexpr static const uint16_t FilterMaskStep = sizeof(T); + // Load value + MCS_FORCE_INLINE SimdType emptyNullLoadValue(const T fill) + { + return loadValue(fill); + } + + MCS_FORCE_INLINE SimdType loadValue(const T fill) + { + return _mm_set1_epi8(fill); + } + + // Load from + MCS_FORCE_INLINE SimdType loadFrom(const char* from) + { + return _mm_loadu_si128(reinterpret_cast(from)); + } + + // Compare + MCS_FORCE_INLINE MT cmpEq(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi8(x, y)); + } + + MCS_FORCE_INLINE MT cmpGe(SimdType& x, SimdType& y) + { + return cmpLt(x, y) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpGt(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpgt_epi8(x, y)); + } + + MCS_FORCE_INLINE MT cmpLe(SimdType& x, SimdType& y) + { + return cmpGt(x, y) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpLt(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmplt_epi8(x, y)); + } + + MCS_FORCE_INLINE MT cmpNe(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi8(x, y)) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpAlwaysFalse(SimdType& x, SimdType& y) + { + return 0; + } + + MCS_FORCE_INLINE MT cmpAlwaysTrue(SimdType& x, SimdType& y) + { + return 0xFFFF; + } + + // permute + /* TODO Available in AVX-512 + MCS_FORCE_INLINE SimdType perm8Bits(SimdType& x, SimdType& idx) + { + return _mm_permutexvar_epi8(x, idx); + } + */ + // misc + MCS_FORCE_INLINE MT convertVectorToBitMask(SimdType& vmask) + { + return _mm_movemask_epi8(vmask); + } + + MCS_FORCE_INLINE MT nullEmptyCmpNe(SimdType& x, SimdType& y) + { + return cmpNe(x, y); + } + + MCS_FORCE_INLINE MT nullEmptyCmpEq(SimdType& x, SimdType& y) + { + return cmpEq(x, y); + } + + MCS_FORCE_INLINE SimdType setToZero() + { + return _mm_setzero_si128(); + } + + // store + MCS_FORCE_INLINE void storeWMask(SimdType& x, SimdType& vmask, char* dst) + { + _mm_maskmoveu_si128(x, vmask, dst); + } + + MCS_FORCE_INLINE void store(char* dst, SimdType& x) + { + _mm_storeu_si128(reinterpret_cast(dst), x); + } +}; + +template +class SimdFilterProcessor< + VT, CHECK_T, typename std::enable_if::value && std::is_same::value>::type> { public: constexpr static const uint16_t vecByteSize = 16U; @@ -895,4 +1345,3 @@ class SimdFilterProcessor< } // namespace simd #endif // if defined(__x86_64__ ) -// vim:ts=2 sw=2: diff --git a/utils/common/spinlock.h b/utils/common/spinlock.h index 2549230c7..8fab8661f 100644 --- a/utils/common/spinlock.h +++ b/utils/common/spinlock.h @@ -2,6 +2,10 @@ #include +// To be refactored. There are two issues with this implentation. +// 1. It can be replaced with a lock despite the memory_order_acquire/release mem order. Needs +// std::atomic_flag instead. +// 2. https://www.realworldtech.com/forum/?threadid=189711&curpostid=189723 namespace utils { inline void getSpinlock(std::atomic& lock) @@ -23,4 +27,29 @@ inline void releaseSpinlock(std::atomic& lock) lock.store(false, std::memory_order_release); } +// c++20 offers a combination of wait/notify methods but +// I prefer to use a simpler version of uspace spin lock. +class USpaceSpinLock +{ + public: + USpaceSpinLock(std::atomic_flag& flag) : flag_(flag) + { + while (flag_.test_and_set(std::memory_order_acquire)) + { + ; + } + }; + ~USpaceSpinLock() + { + release(); + }; + inline void release() + { + flag_.clear(std::memory_order_release); + } + + private: + std::atomic_flag& flag_; +}; + } // namespace utils diff --git a/utils/common/stlpoolallocator.h b/utils/common/stlpoolallocator.h index 46c9bc1ac..1ca6f0880 100644 --- a/utils/common/stlpoolallocator.h +++ b/utils/common/stlpoolallocator.h @@ -72,7 +72,7 @@ class STLPoolAllocator void usePoolAllocator(boost::shared_ptr b); boost::shared_ptr getPoolAllocator(); - pointer allocate(size_type, const void* hint = 0); + pointer allocate(size_type, typename STLPoolAllocator::const_pointer hint = 0); void deallocate(pointer p, size_type n); size_type max_size() const throw(); inline uint64_t getMemUsage() const @@ -131,7 +131,7 @@ boost::shared_ptr STLPoolAllocator::getPoolAllocator() template typename STLPoolAllocator::pointer STLPoolAllocator::allocate( - typename STLPoolAllocator::size_type s, typename std::allocator::const_pointer hint) + typename STLPoolAllocator::size_type s, typename STLPoolAllocator::const_pointer hint) { return (pointer)pa->allocate(s * sizeof(T)); } diff --git a/utils/common/string_prefixes.cpp b/utils/common/string_prefixes.cpp new file mode 100644 index 000000000..9f9c5c2e7 --- /dev/null +++ b/utils/common/string_prefixes.cpp @@ -0,0 +1,51 @@ +/* + Copyright (C) 2021, 2022 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +/* handling of the conversion of string prefixes to int64_t for quick range checking */ + +#include "collation.h" +#include "joblisttypes.h" + +#include "string_prefixes.h" + +// XXX: string (or, actually, a BLOB) with all NUL chars will be encoded into zero. Which corresponds to +// encoding of empty string, or NULL. +int64_t encodeStringPrefix(const uint8_t* str, size_t len, int charsetNumber) +{ + datatypes::Charset cset(charsetNumber); + uint8_t fixedLenPrefix[8]; + memset(fixedLenPrefix, 0, sizeof(fixedLenPrefix)); + cset.strnxfrm(fixedLenPrefix, sizeof(fixedLenPrefix), 8, str, len, 0); + int64_t acc = 0; + size_t i; + for (i = 0; i < 8; i++) + { + uint8_t byte = fixedLenPrefix[i]; + acc = (acc << 8) + byte; + } + return acc; +} + +int64_t encodeStringPrefix_check_null(const uint8_t* str, size_t len, int charsetNumber) +{ + if (len < 1) + { + return joblist::UBIGINTNULL; + } + return encodeStringPrefix(str, len, charsetNumber); +} diff --git a/utils/common/string_prefixes.h b/utils/common/string_prefixes.h new file mode 100644 index 000000000..750c552b5 --- /dev/null +++ b/utils/common/string_prefixes.h @@ -0,0 +1,32 @@ +/* + Copyright (C) 2021, 2022 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +/* handling of the conversion of string prefixes to int64_t for quick range checking */ + +#pragma once + +#include +#include + +// Encode string prefix into an int64_t, packing as many chars from string as possible +// into the result and respecting the collation provided by charsetNumber. +// +// For one example, for CI Czech collation, encodeStringPrefix("cz") < encodeStringPrefix("CH"). +int64_t encodeStringPrefix(const uint8_t* str, size_t len, int charsetNumber); + +int64_t encodeStringPrefix_check_null(const uint8_t* str, size_t len, int charsetNumber); diff --git a/utils/compress/idbcompress.cpp b/utils/compress/idbcompress.cpp index cffe8d268..2c64c3387 100644 --- a/utils/compress/idbcompress.cpp +++ b/utils/compress/idbcompress.cpp @@ -672,4 +672,3 @@ std::shared_ptr getCompressorByType( #endif } // namespace compress -// vim:ts=4 sw=4: diff --git a/utils/configcpp/configcpp.cpp b/utils/configcpp/configcpp.cpp index 6e2101f0c..57c995964 100644 --- a/utils/configcpp/configcpp.cpp +++ b/utils/configcpp/configcpp.cpp @@ -684,4 +684,3 @@ std::string Config::getTempFileDir(Config::TempDirPurpose what) } } // namespace config -// vim:ts=4 sw=4: diff --git a/utils/configcpp/configcpp.h b/utils/configcpp/configcpp.h index 41bb1ad7e..a413c3f52 100644 --- a/utils/configcpp/configcpp.h +++ b/utils/configcpp/configcpp.h @@ -254,4 +254,3 @@ class Config #undef EXPORT -// vim:ts=4 sw=4: diff --git a/utils/configcpp/configstream.cpp b/utils/configcpp/configstream.cpp index 059f690ed..cfb1a4acf 100644 --- a/utils/configcpp/configstream.cpp +++ b/utils/configcpp/configstream.cpp @@ -64,4 +64,3 @@ void ConfigStream::init(const xmlChar* xp) } } // namespace config -// vim:ts=4 sw=4: diff --git a/utils/configcpp/configstream.h b/utils/configcpp/configstream.h index 25ed12f67..deefb02f7 100644 --- a/utils/configcpp/configstream.h +++ b/utils/configcpp/configstream.h @@ -61,4 +61,3 @@ class ConfigStream } // namespace config -// vim:ts=4 sw=4: diff --git a/utils/configcpp/tdriver2.cpp b/utils/configcpp/tdriver2.cpp index e652ac045..f6d00cc49 100644 --- a/utils/configcpp/tdriver2.cpp +++ b/utils/configcpp/tdriver2.cpp @@ -65,7 +65,6 @@ class WOConfigFileTest : public CppUnit::TestFixture void test1() { WriteOnceConfig woc(cf); - CPPUNIT_ASSERT(woc.owns("PrimitiveServers", "LBID_Shift")); CPPUNIT_ASSERT(woc.owns("SystemConfig", "DBRootCount")); CPPUNIT_ASSERT(woc.owns("SystemConfig", "DBRMRoot")); @@ -73,9 +72,6 @@ class WOConfigFileTest : public CppUnit::TestFixture int vali; - vali = Config::fromText(woc.getConfig("PrimitiveServers", "LBID_Shift")); - CPPUNIT_ASSERT(vali == 13); - woc.setConfig("SystemConfig", "DBRootCount", "10"); vali = Config::fromText(woc.getConfig("SystemConfig", "DBRootCount")); CPPUNIT_ASSERT(vali == 10); diff --git a/utils/configcpp/xmlparser.cpp b/utils/configcpp/xmlparser.cpp index 03da1f959..37b6bddea 100644 --- a/utils/configcpp/xmlparser.cpp +++ b/utils/configcpp/xmlparser.cpp @@ -285,4 +285,3 @@ const vector XMLParser::enumSection(const xmlDocPtr doc, const string& s } } // namespace config -// vim:ts=4 sw=4: diff --git a/utils/configcpp/xmlparser.h b/utils/configcpp/xmlparser.h index c7576890b..e31659fea 100644 --- a/utils/configcpp/xmlparser.h +++ b/utils/configcpp/xmlparser.h @@ -63,4 +63,3 @@ class XMLParser } // namespace config -// vim:ts=4 sw=4: diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 5d5e9d346..37ff38c4e 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1193,7 +1193,7 @@ bool stringToTimeStruct(const string& data, Time& dtime, long decimals) return true; } -bool stringToTimestampStruct(const string& data, TimeStamp& timeStamp, const string& timeZone) +bool stringToTimestampStruct(const string& data, TimeStamp& timeStamp, long timeZone) { // special handling for 0000-00-00 00:00:00 // "0" is sent by the server when checking for default value @@ -1418,7 +1418,7 @@ boost::any DataConvert::StringToUDecimal(const datatypes::SystemCatalog::TypeAtt { int64_t val64; number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); - char ival = (char)val64; + signed char ival = (signed char)val64; if (ival < 0 && ival != static_cast(joblist::TINYINTEMPTYROW) && ival != static_cast(joblist::TINYINTNULL)) @@ -1879,10 +1879,11 @@ int64_t DataConvert::convertColumnDatetime(const char* dataOrg, CalpontDateTimeF // Most of this code is taken from DataConvert::convertColumnDatetime //------------------------------------------------------------------------------ int64_t DataConvert::convertColumnTimestamp(const char* dataOrg, CalpontDateTimeFormat datetimeFormat, - int& status, unsigned int dataOrgLen, const std::string& timeZone) + int& status, unsigned int dataOrgLen, long timeZone) { char tmbuf[64]; std::string dataOrgTemp = dataOrg; + status = 0; if (dataOrgTemp.substr(0, 19) == "0000-00-00 00:00:00") { return 0; @@ -1902,7 +1903,6 @@ int64_t DataConvert::convertColumnTimestamp(const char* dataOrg, CalpontDateTime dataOrgLen = strlen(tmbuf); } - status = 0; const char* p; p = dataOrg; char fld[10]; @@ -2258,8 +2258,7 @@ std::string DataConvert::datetimeToString(long long datetimevalue, long decimals return buf; } -std::string DataConvert::timestampToString(long long timestampvalue, const std::string& timezone, - long decimals) +std::string DataConvert::timestampToString(long long timestampvalue, long timezone, long decimals) { // 10 is default which means we don't need microseconds if (decimals > 6 || decimals < 0) @@ -2343,7 +2342,7 @@ std::string DataConvert::datetimeToString1(long long datetimevalue) return buf; } -std::string DataConvert::timestampToString1(long long timestampvalue, const std::string& timezone) +std::string DataConvert::timestampToString1(long long timestampvalue, long timezone) { const int TIMESTAMPTOSTRING1_LEN = 22; // YYYYMMDDHHMMSSmmmmmm\0 char buf[TIMESTAMPTOSTRING1_LEN]; @@ -2381,7 +2380,7 @@ int64_t DataConvert::datetimeToInt(const string& datetime) return stringToDatetime(datetime); } -int64_t DataConvert::timestampToInt(const string& timestamp, const string& timeZone) +int64_t DataConvert::timestampToInt(const string& timestamp, long timeZone) { return stringToTimestamp(timestamp, timeZone); } @@ -2414,7 +2413,7 @@ int64_t DataConvert::stringToDatetime(const string& data, bool* date) return -1; } -int64_t DataConvert::stringToTimestamp(const string& data, const string& timeZone) +int64_t DataConvert::stringToTimestamp(const string& data, long timeZone) { TimeStamp aTimestamp; @@ -3389,4 +3388,3 @@ void DataConvert::joinColTypeForUnion(datatypes::SystemCatalog::TypeHolderStd& u } } // namespace dataconvert -// vim:ts=4 sw=4: diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 1df077670..908961caa 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -28,13 +28,8 @@ #include #include #include -#ifdef _MSC_VER -#include -#include -#include -#else + #include -#endif #ifdef __linux__ #define POSIX_REGEX @@ -43,7 +38,7 @@ #ifdef POSIX_REGEX #include #else -#include +#include #endif #include "mcs_datatype.h" @@ -289,6 +284,14 @@ inline void unserializeTimezoneInfo(messageqcpp::ByteStream& bs, TIME_ZONE_INFO* bs >> (uint&)tz->revcnt; }; +inline long systemTimeZoneOffset() +{ + time_t t = time(NULL); + struct tm lt; + localtime_r(&t, <); + return lt.tm_gmtoff; +} + /** * This function converts the timezone represented as a string * in the format "+HH:MM" or "-HH:MM" to a signed offset in seconds @@ -296,20 +299,32 @@ inline void unserializeTimezoneInfo(messageqcpp::ByteStream& bs, TIME_ZONE_INFO* */ inline bool timeZoneToOffset(const char* str, std::string::size_type length, long* offset) { + if (strcmp(str, "SYSTEM") == 0) + { + *offset = systemTimeZoneOffset(); + return 0; + } + const char* end = str + length; bool negative; unsigned long number_tmp; long offset_tmp; if (length < 4) + { + *offset = 0; return 1; + } if (*str == '+') negative = 0; else if (*str == '-') negative = 1; else + { + *offset = 0; return 1; + } str++; number_tmp = 0; @@ -321,7 +336,10 @@ inline bool timeZoneToOffset(const char* str, std::string::size_type length, lon } if (str + 1 >= end || *str != ':') + { + *offset = 0; return 1; + } str++; offset_tmp = number_tmp * 60L; @@ -334,7 +352,10 @@ inline bool timeZoneToOffset(const char* str, std::string::size_type length, lon } if (str != end) + { + *offset = 0; return 1; + } offset_tmp = (offset_tmp + number_tmp) * 60L; @@ -347,10 +368,12 @@ inline bool timeZoneToOffset(const char* str, std::string::size_type length, lon */ if (number_tmp > 59 || offset_tmp < -13 * 3600L + 1 || offset_tmp > 13 * 3600L) + { + *offset = 0; return 1; + } *offset = offset_tmp; - return 0; } @@ -483,10 +506,10 @@ inline bool isTimestampValid(uint64_t second, uint64_t microsecond) * * @param seconds the value to be converted * @param time the broken-down representation of the timestamp - * @param timeZone a string with the server timezone of the machine - * which initiated the query + @param offset a timeZone offset (in seconds) relative to UTC. + For example, for EST which is UTC-5:00, offset will be -18000s. */ -inline void gmtSecToMySQLTime(int64_t seconds, MySQLTime& time, const std::string& timeZone) +inline void gmtSecToMySQLTime(int64_t seconds, MySQLTime& time, long offset) { if (seconds == 0) { @@ -494,78 +517,52 @@ inline void gmtSecToMySQLTime(int64_t seconds, MySQLTime& time, const std::strin return; } - if (timeZone == "SYSTEM") + int64_t days; + int32_t rem; + int32_t y; + int32_t yleap; + const unsigned int* ip; + + days = (int64_t)(seconds / SECS_PER_DAY); + rem = (int32_t)(seconds % SECS_PER_DAY); + + rem += offset; + while (rem < 0) { - struct tm tmp_tm; - time_t tmp_t = (time_t)seconds; - localtime_r(&tmp_t, &tmp_tm); - time.second_part = 0; - time.year = (int)((tmp_tm.tm_year + 1900) % 10000); - time.month = (int)tmp_tm.tm_mon + 1; - time.day = (int)tmp_tm.tm_mday; - time.hour = (int)tmp_tm.tm_hour; - time.minute = (int)tmp_tm.tm_min; - time.second = (int)tmp_tm.tm_sec; - time.time_type = CALPONTDATETIME_ENUM; - if (time.second == 60 || time.second == 61) - time.second = 59; + rem += SECS_PER_DAY; + days--; } - else + while (rem >= SECS_PER_DAY) { - long offset; - if (timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset)) - { - time.reset(); - return; - } - - int64_t days; - int32_t rem; - int32_t y; - int32_t yleap; - const unsigned int* ip; - - days = (int64_t)(seconds / SECS_PER_DAY); - rem = (int32_t)(seconds % SECS_PER_DAY); - - rem += offset; - while (rem < 0) - { - rem += SECS_PER_DAY; - days--; - } - while (rem >= SECS_PER_DAY) - { - rem -= SECS_PER_DAY; - days++; - } - time.hour = (unsigned int)(rem / SECS_PER_HOUR); - rem = rem % SECS_PER_HOUR; - time.minute = (unsigned int)(rem / SECS_PER_MIN); - time.second = (unsigned int)(rem % SECS_PER_MIN); - - y = EPOCH_YEAR; - while (days < 0 || days >= (int64_t)(year_lengths[yleap = isLeapYear(y)])) - { - int32_t newy; - - newy = y + days / DAYS_PER_NYEAR; - if (days < 0) - newy--; - days -= (newy - y) * DAYS_PER_NYEAR + leapsThruEndOf(newy - 1) - leapsThruEndOf(y - 1); - y = newy; - } - time.year = y; - - ip = mon_lengths[yleap]; - for (time.month = 0; days >= (int64_t)ip[time.month]; time.month++) - days -= (int64_t)ip[time.month]; - time.month++; - time.day = (unsigned int)(days + 1); - - time.second_part = 0; - time.time_type = CALPONTDATETIME_ENUM; + rem -= SECS_PER_DAY; + days++; } + time.hour = (unsigned int)(rem / SECS_PER_HOUR); + rem = rem % SECS_PER_HOUR; + time.minute = (unsigned int)(rem / SECS_PER_MIN); + time.second = (unsigned int)(rem % SECS_PER_MIN); + + y = EPOCH_YEAR; + while (days < 0 || days >= (int64_t)(year_lengths[yleap = isLeapYear(y)])) + { + int32_t newy; + + newy = y + days / DAYS_PER_NYEAR; + if (days < 0) + newy--; + days -= (newy - y) * DAYS_PER_NYEAR + leapsThruEndOf(newy - 1) - leapsThruEndOf(y - 1); + y = newy; + } + time.year = y; + + ip = mon_lengths[yleap]; + for (time.month = 0; days >= (int64_t)ip[time.month]; time.month++) + days -= (int64_t)ip[time.month]; + time.month++; + time.day = (unsigned int)(days + 1); + + time.second_part = 0; + time.time_type = CALPONTDATETIME_ENUM; } /** @@ -593,41 +590,15 @@ inline int64_t secSinceEpoch(int year, int month, int day, int hour, int min, in return ((days * HOURS_PER_DAY + hour) * MINS_PER_HOUR + min) * SECS_PER_MIN + sec; } -// This is duplicate of funchelpers.h:calc_mysql_daynr, -// with one additional function parameter -inline uint32_t calc_mysql_daynr(uint32_t year, uint32_t month, uint32_t day, bool& isValid) -{ - int temp; - int y = year; - long delsum; - - if (!isDateValid(day, month, year)) - { - isValid = false; - return 0; - } - - delsum = (long)(365 * y + 31 * ((int)month - 1) + (int)day); - - if (month <= 2) - y--; - else - delsum -= (long)((int)month * 4 + 23) / 10; - - temp = (int)((y / 100 + 1) * 3) / 4; - - return delsum + (int)y / 4 - temp; -} - /** * @brief converts a timestamp from broken-down representation * to seconds since UTC epoch * * @param time the broken-down representation of the timestamp - @param timeZone a string with the server timezone of the machine - which initiated the query + @param offset a timeZone offset (in seconds) relative to UTC. + For example, for EST which is UTC-5:00, offset will be -18000s. */ -inline int64_t mySQLTimeToGmtSec(const MySQLTime& time, const std::string& timeZone, bool& isValid) +inline int64_t mySQLTimeToGmtSec(const MySQLTime& time, long offset, bool& isValid) { int64_t seconds; @@ -637,88 +608,7 @@ inline int64_t mySQLTimeToGmtSec(const MySQLTime& time, const std::string& timeZ return 0; } - if (timeZone == "SYSTEM") - { - // This is mirror of code in func_unix_timestamp.cpp - uint32_t loop; - time_t tmp_t = 0; - int shift = 0; - struct tm *l_time, tm_tmp; - int64_t diff; - localtime_r(&tmp_t, &tm_tmp); - // Get the system timezone offset at 0 seconds since epoch - int64_t my_time_zone = tm_tmp.tm_gmtoff; - int day = time.day; - - if ((time.year == MAX_TIMESTAMP_YEAR) && (time.month == 1) && (day > 4)) - { - day -= 2; - shift = 2; - } - - tmp_t = (time_t)(((calc_mysql_daynr(time.year, time.month, day, isValid) - 719528) * 86400L + - (int64_t)time.hour * 3600L + (int64_t)(time.minute * 60 + time.second)) - - (time_t)my_time_zone); - if (!isValid) - return 0; - - localtime_r(&tmp_t, &tm_tmp); - l_time = &tm_tmp; - - for (loop = 0; - loop < 2 && (time.hour != (uint32_t)l_time->tm_hour || time.minute != (uint32_t)l_time->tm_min || - time.second != (uint32_t)l_time->tm_sec); - loop++) - { - int days = day - l_time->tm_mday; - - if (days < -1) - days = 1; /* Month has wrapped */ - else if (days > 1) - days = -1; - - diff = (3600L * (int64_t)(days * 24 + ((int)time.hour - (int)l_time->tm_hour)) + - (int64_t)(60 * ((int)time.minute - (int)l_time->tm_min)) + - (int64_t)((int)time.second - (int)l_time->tm_sec)); - tmp_t += (time_t)diff; - localtime_r(&tmp_t, &tm_tmp); - l_time = &tm_tmp; - } - - if (loop == 2 && time.hour != (uint32_t)l_time->tm_hour) - { - int days = day - l_time->tm_mday; - - if (days < -1) - days = 1; /* Month has wrapped */ - else if (days > 1) - days = -1; - - diff = (3600L * (int64_t)(days * 24 + ((int)time.hour - (int)l_time->tm_hour)) + - (int64_t)(60 * ((int)time.minute - (int)l_time->tm_min)) + - (int64_t)((int)time.second - (int)l_time->tm_sec)); - - if (diff == 3600) - tmp_t += 3600 - time.minute * 60 - time.second; /* Move to next hour */ - else if (diff == -3600) - tmp_t -= time.minute * 60 + time.second; /* Move to previous hour */ - } - - /* shift back, if we were dealing with boundary dates */ - tmp_t += shift * 86400L; - - seconds = (int64_t)tmp_t; - } - else - { - long offset; - if (timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset)) - { - isValid = false; - return -1; - } - seconds = secSinceEpoch(time.year, time.month, time.day, time.hour, time.minute, time.second) - offset; - } + seconds = secSinceEpoch(time.year, time.month, time.day, time.hour, time.minute, time.second) - offset; /* make sure we have legit timestamps (i.e. we didn't over/underflow anywhere above) */ if (seconds >= MIN_TIMESTAMP_VALUE && seconds <= MAX_TIMESTAMP_VALUE) return seconds; @@ -1151,11 +1041,11 @@ struct TimeStamp { } - int64_t convertToMySQLint(const std::string& timeZone) const; + int64_t convertToMySQLint(long timeZone) const; void reset(); }; -inline int64_t TimeStamp::convertToMySQLint(const std::string& timeZone) const +inline int64_t TimeStamp::convertToMySQLint(long timeZone) const { const int TIMESTAMPTOSTRING1_LEN = 22; // YYYYMMDDHHMMSSmmmmmm\0 char buf[TIMESTAMPTOSTRING1_LEN]; @@ -1262,10 +1152,9 @@ class DataConvert * @param type the columns database type * @param data the columns string representation of it's data */ - EXPORT static std::string timestampToString(long long timestampvalue, const std::string& timezone, - long decimals = 0); + EXPORT static std::string timestampToString(long long timestampvalue, long timezone, long decimals = 0); static inline void timestampToString(long long timestampvalue, char* buf, unsigned int buflen, - const std::string& timezone, long decimals = 0); + long timezone, long decimals = 0); /** * @brief convert a columns data from native format to a string @@ -1300,9 +1189,9 @@ class DataConvert * @param type the columns database type * @param data the columns string representation of it's data */ - EXPORT static std::string timestampToString1(long long timestampvalue, const std::string& timezone); + EXPORT static std::string timestampToString1(long long timestampvalue, long timezone); static inline void timestampToString1(long long timestampvalue, char* buf, unsigned int buflen, - const std::string& timezone); + long timezone); /** * @brief convert a columns data from native format to a string @@ -1352,11 +1241,11 @@ class DataConvert * @param datetimeFormat the format the date value in * @param status 0 - success, -1 - fail * @param dataOrgLen length specification of dataOrg - * @param timeZone the timezone used for conversion to native format + * @param timeZone an offset (in seconds) relative to UTC. + For example, for EST which is UTC-5:00, offset will be -18000s. */ EXPORT static int64_t convertColumnTimestamp(const char* dataOrg, CalpontDateTimeFormat datetimeFormat, - int& status, unsigned int dataOrgLen, - const std::string& timeZone); + int& status, unsigned int dataOrgLen, long timeZone); /** * @brief convert a time column data, represented as a string, @@ -1385,7 +1274,7 @@ class DataConvert // convert string to datetime EXPORT static int64_t stringToDatetime(const std::string& data, bool* isDate = NULL); // convert string to timestamp - EXPORT static int64_t stringToTimestamp(const std::string& data, const std::string& timeZone); + EXPORT static int64_t stringToTimestamp(const std::string& data, long timeZone); // convert integer to date EXPORT static int64_t intToDate(int64_t data); // convert integer to datetime @@ -1396,7 +1285,7 @@ class DataConvert EXPORT static int64_t dateToInt(const std::string& date); // convert string to datetime. alias to datetimeToInt EXPORT static int64_t datetimeToInt(const std::string& datetime); - EXPORT static int64_t timestampToInt(const std::string& timestamp, const std::string& timeZone); + EXPORT static int64_t timestampToInt(const std::string& timestamp, long timeZone); EXPORT static int64_t timeToInt(const std::string& time); EXPORT static int64_t stringToTime(const std::string& data); // bug4388, union type conversion @@ -1467,7 +1356,7 @@ inline void DataConvert::datetimeToString(long long datetimevalue, char* buf, un } inline void DataConvert::timestampToString(long long timestampvalue, char* buf, unsigned int buflen, - const std::string& timezone, long decimals) + long timezone, long decimals) { // 10 is default which means we don't need microseconds if (decimals > 6 || decimals < 0) @@ -1545,7 +1434,7 @@ inline void DataConvert::datetimeToString1(long long datetimevalue, char* buf, u } inline void DataConvert::timestampToString1(long long timestampvalue, char* buf, unsigned int buflen, - const std::string& timezone) + long timezone) { TimeStamp timestamp(timestampvalue); int64_t seconds = timestamp.second; diff --git a/utils/ddlcleanup/ddlcleanuputil.cpp b/utils/ddlcleanup/ddlcleanuputil.cpp index a9defdf45..ca196c295 100644 --- a/utils/ddlcleanup/ddlcleanuputil.cpp +++ b/utils/ddlcleanup/ddlcleanuputil.cpp @@ -270,4 +270,3 @@ int ddl_cleanup() } } // namespace ddlcleanuputil -// vim:ts=4 sw=4: diff --git a/utils/ddlcleanup/ddlcleanuputil.h b/utils/ddlcleanup/ddlcleanuputil.h index f7523d43a..bfd8e87cd 100644 --- a/utils/ddlcleanup/ddlcleanuputil.h +++ b/utils/ddlcleanup/ddlcleanuputil.h @@ -28,4 +28,3 @@ namespace ddlcleanuputil { int ddl_cleanup(); } -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_abs.cpp b/utils/funcexp/func_abs.cpp index 64ac5d014..cd05ade7a 100644 --- a/utils/funcexp/func_abs.cpp +++ b/utils/funcexp/func_abs.cpp @@ -79,4 +79,3 @@ long double Func_abs::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_add_time.cpp b/utils/funcexp/func_add_time.cpp index 09b1d2d0c..82c173edc 100644 --- a/utils/funcexp/func_add_time.cpp +++ b/utils/funcexp/func_add_time.cpp @@ -171,7 +171,7 @@ int64_t Func_add_time::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm TimeStamp timestamp(val1); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); + gmtSecToMySQLTime(seconds, m_time, ct.getTimeZone()); dt1.year = m_time.year; dt1.month = m_time.month; dt1.day = m_time.day; @@ -303,4 +303,3 @@ int64_t Func_add_time::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, boo } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_ascii.cpp b/utils/funcexp/func_ascii.cpp index eb93f6011..5cf8653d2 100644 --- a/utils/funcexp/func_ascii.cpp +++ b/utils/funcexp/func_ascii.cpp @@ -56,4 +56,3 @@ int64_t Func_ascii::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNu } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_between.cpp b/utils/funcexp/func_between.cpp index 3bd2b907b..657123635 100644 --- a/utils/funcexp/func_between.cpp +++ b/utils/funcexp/func_between.cpp @@ -328,7 +328,7 @@ CalpontSystemCatalog::ColType Func_between::operationType(FunctionParm& fp, if (cc) { Result result = cc->result(); - result.intVal = dataconvert::DataConvert::timestampToInt(result.strVal, timeZone()); + result.intVal = dataconvert::DataConvert::timestampToInt(result.strVal, resultType.getTimeZone()); cc->result(result); } } @@ -376,4 +376,3 @@ bool Func_notbetween::getBoolVal(rowgroup::Row& row, FunctionParm& pm, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_bitwise.cpp b/utils/funcexp/func_bitwise.cpp index 76d09b9bd..998696ae0 100644 --- a/utils/funcexp/func_bitwise.cpp +++ b/utils/funcexp/func_bitwise.cpp @@ -105,7 +105,8 @@ static datatypes::TUInt64Null DecimalToBitOperand(Row& row, const execplan::SPTP // and could be extracted into a utility class with its own header // if that is the case - this is left as future exercise datatypes::TUInt64Null GenericToBitOperand(Row& row, const execplan::SPTP& parm, - const funcexp::Func& thisFunc, bool temporalRounding) + const funcexp::Func& thisFunc, bool temporalRounding, + long timeZone) { switch (parm->data()->resultType().colDataType) { @@ -186,7 +187,7 @@ datatypes::TUInt64Null GenericToBitOperand(Row& row, const execplan::SPTP& parm, return datatypes::TUInt64Null(); TimeStamp dt(time); - int64_t value = dt.convertToMySQLint(thisFunc.timeZone()); + int64_t value = dt.convertToMySQLint(timeZone); if (temporalRounding && dt.msecond >= 500000) value++; return datatypes::TUInt64Null((uint64_t)value); @@ -222,8 +223,8 @@ class BitOperandGeneric : public datatypes::TUInt64Null BitOperandGeneric() { } - BitOperandGeneric(Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc) - : TUInt64Null(GenericToBitOperand(row, parm, thisFunc, true)) + BitOperandGeneric(Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc, long timeZone) + : TUInt64Null(GenericToBitOperand(row, parm, thisFunc, true, timeZone)) { } }; @@ -236,8 +237,9 @@ class BitOperandGenericShiftAmount : public datatypes::TUInt64Null BitOperandGenericShiftAmount() { } - BitOperandGenericShiftAmount(Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc) - : TUInt64Null(GenericToBitOperand(row, parm, thisFunc, false)) + BitOperandGenericShiftAmount(Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc, + long timeZone) + : TUInt64Null(GenericToBitOperand(row, parm, thisFunc, false, timeZone)) { } }; @@ -327,7 +329,7 @@ class Func_bitand_return_uint64 : public Func_bitand CalpontSystemCatalog::ColType& operationColType) override { idbassert(parm.size() == 2); - Arg2Lazy args(row, parm, *this); + Arg2Lazy args(row, parm, *this, operationColType.getTimeZone()); return (int64_t)(args.a & args.b).nullSafeValue(isNull); } }; @@ -353,7 +355,7 @@ class Func_leftshift_return_uint64 : public Func_leftshift CalpontSystemCatalog::ColType& operationColType) override { idbassert(parm.size() == 2); - Arg2Eager args(row, parm, *this); + Arg2Eager args(row, parm, *this, operationColType.getTimeZone()); return (int64_t)args.a.MariaDBShiftLeft(args.b).nullSafeValue(isNull); } }; @@ -378,7 +380,7 @@ class Func_rightshift_return_uint64 : public Func_rightshift CalpontSystemCatalog::ColType& operationColType) override { idbassert(parm.size() == 2); - Arg2Eager args(row, parm, *this); + Arg2Eager args(row, parm, *this, operationColType.getTimeZone()); return (int64_t)args.a.MariaDBShiftRight(args.b).nullSafeValue(isNull); } }; @@ -409,7 +411,7 @@ class Func_bitor_return_uint64 : public Func_bitor CalpontSystemCatalog::ColType& operationColType) override { idbassert(parm.size() == 2); - Arg2Lazy args(row, parm, *this); + Arg2Lazy args(row, parm, *this, operationColType.getTimeZone()); return (int64_t)(args.a | args.b).nullSafeValue(isNull); } }; @@ -435,7 +437,7 @@ class Func_bitxor_return_uint64 : public Func_bitxor CalpontSystemCatalog::ColType& operationColType) override { idbassert(parm.size() == 2); - Arg2Eager args(row, parm, *this); + Arg2Eager args(row, parm, *this, operationColType.getTimeZone()); return (int64_t)(args.a ^ args.b).nullSafeValue(isNull); } }; @@ -475,7 +477,7 @@ class Func_bit_count_return_uint64 : public Func_bit_count CalpontSystemCatalog::ColType& operationColType) override { idbassert(parm.size() == 1); - return bitCount((uint64_t)TA(row, parm[0], *this).nullSafeValue(isNull)); + return bitCount((uint64_t)TA(row, parm[0], *this, operationColType.getTimeZone()).nullSafeValue(isNull)); } }; @@ -492,4 +494,3 @@ bool Func_bit_count::fix(execplan::FunctionColumn& col) const } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_case.cpp b/utils/funcexp/func_case.cpp index e8bfbd0f4..fba7691e0 100644 --- a/utils/funcexp/func_case.cpp +++ b/utils/funcexp/func_case.cpp @@ -723,4 +723,3 @@ int64_t Func_searched_case::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_cast.cpp b/utils/funcexp/func_cast.cpp index b3917b2a5..ef0e2f515 100644 --- a/utils/funcexp/func_cast.cpp +++ b/utils/funcexp/func_cast.cpp @@ -186,7 +186,7 @@ int64_t Func_cast_signed::getIntVal(Row& row, FunctionParm& parm, bool& isNull, int64_t time = parm[0]->data()->getTimestampIntVal(row, isNull); TimeStamp dt(time); - return dt.convertToMySQLint(timeZone()); + return dt.convertToMySQLint(operationColType.getTimeZone()); } break; @@ -304,7 +304,7 @@ uint64_t Func_cast_unsigned::getUintVal(Row& row, FunctionParm& parm, bool& isNu int64_t time = parm[0]->data()->getTimestampIntVal(row, isNull); TimeStamp dt(time); - return dt.convertToMySQLint(timeZone()); + return dt.convertToMySQLint(operationColType.getTimeZone()); } break; @@ -433,7 +433,7 @@ string Func_cast_char::getStrVal(Row& row, FunctionParm& parm, bool& isNull, case execplan::CalpontSystemCatalog::TIMESTAMP: { return dataconvert::DataConvert::timestampToString(parm[0]->data()->getTimestampIntVal(row, isNull), - timeZone()) + operationColType.getTimeZone()) .substr(0, length); } break; @@ -568,7 +568,7 @@ int32_t Func_cast_date::getDateIntVal(rowgroup::Row& row, FunctionParm& parm, bo case execplan::CalpontSystemCatalog::TIMESTAMP: { int64_t val1 = parm[0]->data()->getTimestampIntVal(row, isNull); - string value = dataconvert::DataConvert::timestampToString(val1, timeZone()); + string value = dataconvert::DataConvert::timestampToString(val1, op_ct.getTimeZone()); value = value.substr(0, 10); return dataconvert::DataConvert::stringToDate(value); } @@ -691,7 +691,7 @@ int64_t Func_cast_date::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); + gmtSecToMySQLTime(seconds, m_time, operationColType.getTimeZone()); DateTime dt; dt.year = m_time.year; dt.month = m_time.month; @@ -847,7 +847,7 @@ int64_t Func_cast_datetime::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); + gmtSecToMySQLTime(seconds, m_time, operationColType.getTimeZone()); DateTime dt; dt.year = m_time.year; dt.month = m_time.month; @@ -958,7 +958,7 @@ int64_t Func_cast_datetime::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); + gmtSecToMySQLTime(seconds, m_time, operationColType.getTimeZone()); Time time; time.hour = m_time.hour; time.minute = m_time.minute; @@ -1362,7 +1362,7 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, FunctionParm& parm, bool& int32_t s = 0; string value = dataconvert::DataConvert::timestampToString1( - parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); + parm[0]->data()->getTimestampIntVal(row, isNull), operationColType.getTimeZone()); // strip off micro seconds string date = value.substr(0, 14); @@ -1475,8 +1475,8 @@ double Func_cast_double::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull case execplan::CalpontSystemCatalog::TIMESTAMP: { - string str = - DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); + string str = DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), + operationColType.getTimeZone()); // strip off micro seconds str = str.substr(0, 14); @@ -1549,4 +1549,3 @@ double Func_cast_double::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_ceil.cpp b/utils/funcexp/func_ceil.cpp index f0c016847..e5c1f8728 100644 --- a/utils/funcexp/func_ceil.cpp +++ b/utils/funcexp/func_ceil.cpp @@ -571,7 +571,7 @@ IDB_Decimal Func_ceil::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, TimeStamp dt(parm[0]->data()->getTimestampIntVal(row, isNull)); if (!isNull) - ret.value = dt.convertToMySQLint(timeZone()); + ret.value = dt.convertToMySQLint(op_ct.getTimeZone()); } break; @@ -598,4 +598,3 @@ IDB_Decimal Func_ceil::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_char.cpp b/utils/funcexp/func_char.cpp index 7ead2b947..0c4adeaf9 100644 --- a/utils/funcexp/func_char.cpp +++ b/utils/funcexp/func_char.cpp @@ -178,4 +178,3 @@ string Func_char::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontS } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_char_length.cpp b/utils/funcexp/func_char_length.cpp index 749914188..5270d1d8a 100644 --- a/utils/funcexp/func_char_length.cpp +++ b/utils/funcexp/func_char_length.cpp @@ -99,7 +99,7 @@ int64_t Func_char_length::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool case execplan::CalpontSystemCatalog::TIMESTAMP: { string date = dataconvert::DataConvert::timestampToString( - parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); + parm[0]->data()->getTimestampIntVal(row, isNull), op_ct.getTimeZone()); return (int64_t)date.size(); } @@ -121,4 +121,3 @@ int64_t Func_char_length::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_coalesce.cpp b/utils/funcexp/func_coalesce.cpp index ebabde050..bbb5a279e 100644 --- a/utils/funcexp/func_coalesce.cpp +++ b/utils/funcexp/func_coalesce.cpp @@ -244,4 +244,3 @@ execplan::IDB_Decimal Func_coalesce::getDecimalVal(rowgroup::Row& row, FunctionP } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_concat.cpp b/utils/funcexp/func_concat.cpp index 73b2a2b2d..05835d23a 100644 --- a/utils/funcexp/func_concat.cpp +++ b/utils/funcexp/func_concat.cpp @@ -65,4 +65,3 @@ string Func_concat::getStrVal(Row& row, FunctionParm& parm, bool& isNull, Calpon } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_concat_oracle.cpp b/utils/funcexp/func_concat_oracle.cpp index 28713a107..46631e7a4 100644 --- a/utils/funcexp/func_concat_oracle.cpp +++ b/utils/funcexp/func_concat_oracle.cpp @@ -71,4 +71,3 @@ string Func_concat_oracle::getStrVal(Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_concat_ws.cpp b/utils/funcexp/func_concat_ws.cpp index 2c07c5b0f..ea0cf22d4 100644 --- a/utils/funcexp/func_concat_ws.cpp +++ b/utils/funcexp/func_concat_ws.cpp @@ -121,4 +121,3 @@ string Func_concat_ws::getStrVal(Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_conv.cpp b/utils/funcexp/func_conv.cpp index 5f73f5ea4..037ce83e8 100644 --- a/utils/funcexp/func_conv.cpp +++ b/utils/funcexp/func_conv.cpp @@ -304,4 +304,3 @@ string Func_conv::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_convert_tz.cpp b/utils/funcexp/func_convert_tz.cpp index 424b7494c..363392796 100644 --- a/utils/funcexp/func_convert_tz.cpp +++ b/utils/funcexp/func_convert_tz.cpp @@ -154,7 +154,9 @@ int64_t Func_convert_tz::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& } else { - seconds = dataconvert::mySQLTimeToGmtSec(my_start_time, from_tz, valid); + long from_tz_offset; + dataconvert::timeZoneToOffset(from_tz.c_str(), from_tz.size(), &from_tz_offset); + seconds = dataconvert::mySQLTimeToGmtSec(my_start_time, from_tz_offset, valid); if (!valid) { if (seconds != 0) @@ -196,7 +198,9 @@ int64_t Func_convert_tz::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& } else { - dataconvert::gmtSecToMySQLTime(seconds, my_time_tmp, to_tz); + long to_tz_offset; + dataconvert::timeZoneToOffset(to_tz.c_str(), to_tz.size(), &to_tz_offset); + dataconvert::gmtSecToMySQLTime(seconds, my_time_tmp, to_tz_offset); } dataconvert::DateTime result_datetime(my_time_tmp.year, my_time_tmp.month, my_time_tmp.day, @@ -216,4 +220,3 @@ string Func_convert_tz::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_crc32.cpp b/utils/funcexp/func_crc32.cpp index a40717378..786c3f33e 100644 --- a/utils/funcexp/func_crc32.cpp +++ b/utils/funcexp/func_crc32.cpp @@ -64,4 +64,3 @@ int64_t Func_crc32::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNu } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_date.cpp b/utils/funcexp/func_date.cpp index 8ebbd2250..b6e554e5f 100644 --- a/utils/funcexp/func_date.cpp +++ b/utils/funcexp/func_date.cpp @@ -47,7 +47,7 @@ CalpontSystemCatalog::ColType Func_date::operationType(FunctionParm& fp, } int64_t Func_date::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, - CalpontSystemCatalog::ColType&) + CalpontSystemCatalog::ColType& ct) { CalpontSystemCatalog::ColDataType type = parm[0]->data()->resultType().colDataType; @@ -75,7 +75,7 @@ int64_t Func_date::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNul case execplan::CalpontSystemCatalog::TIMESTAMP: { int64_t val1 = parm[0]->data()->getTimestampIntVal(row, isNull); - value = dataconvert::DataConvert::timestampToString(val1, timeZone()); + value = dataconvert::DataConvert::timestampToString(val1, ct.getTimeZone()); value = value.substr(0, 10); break; } @@ -149,4 +149,3 @@ string Func_date::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_date_add.cpp b/utils/funcexp/func_date_add.cpp index c98d54581..6cc47fdae 100644 --- a/utils/funcexp/func_date_add.cpp +++ b/utils/funcexp/func_date_add.cpp @@ -759,7 +759,7 @@ int64_t Func_date_add::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& i TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); + gmtSecToMySQLTime(seconds, m_time, ct.getTimeZone()); DateTime dt; dt.year = m_time.year; dt.month = m_time.month; @@ -816,4 +816,3 @@ string Func_date_add::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_date_format.cpp b/utils/funcexp/func_date_format.cpp index 0aff75d9e..8a3ba9749 100644 --- a/utils/funcexp/func_date_format.cpp +++ b/utils/funcexp/func_date_format.cpp @@ -242,7 +242,7 @@ CalpontSystemCatalog::ColType Func_date_format::operationType(FunctionParm& fp, } string Func_date_format::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, - CalpontSystemCatalog::ColType&) + CalpontSystemCatalog::ColType& ct) { int64_t val = 0; DateTime dt = 0; @@ -273,7 +273,7 @@ string Func_date_format::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& TimeStamp timestamp(val); int64_t seconds = timestamp.second; MySQLTime time; - gmtSecToMySQLTime(seconds, time, timeZone()); + gmtSecToMySQLTime(seconds, time, ct.getTimeZone()); dt.year = time.year; dt.month = time.month; dt.day = time.day; @@ -412,8 +412,7 @@ int64_t Func_date_format::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& pa int64_t Func_date_format::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), timeZone()); + return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), ct.getTimeZone()); } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_day.cpp b/utils/funcexp/func_day.cpp index 29baf4466..e7954e3d0 100644 --- a/utils/funcexp/func_day.cpp +++ b/utils/funcexp/func_day.cpp @@ -64,7 +64,7 @@ int64_t Func_day::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); return m_time.day; } @@ -143,4 +143,3 @@ int64_t Func_day::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_dayname.cpp b/utils/funcexp/func_dayname.cpp index 9a94882c6..e8e069db5 100644 --- a/utils/funcexp/func_dayname.cpp +++ b/utils/funcexp/func_dayname.cpp @@ -76,7 +76,7 @@ int64_t Func_dayname::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is dataconvert::TimeStamp timestamp(val); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, time, op_ct.getTimeZone()); year = time.year; month = time.month; day = time.day; @@ -179,4 +179,3 @@ string Func_dayname::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_dayofweek.cpp b/utils/funcexp/func_dayofweek.cpp index 64f04bcf3..c8fb40ab2 100644 --- a/utils/funcexp/func_dayofweek.cpp +++ b/utils/funcexp/func_dayofweek.cpp @@ -75,7 +75,7 @@ int64_t Func_dayofweek::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& dataconvert::TimeStamp timestamp(val); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, time, ct.getTimeZone()); year = time.year; month = time.month; day = time.day; @@ -166,4 +166,3 @@ int64_t Func_dayofweek::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_dayofyear.cpp b/utils/funcexp/func_dayofyear.cpp index 487635c7c..215ec2150 100644 --- a/utils/funcexp/func_dayofyear.cpp +++ b/utils/funcexp/func_dayofyear.cpp @@ -74,7 +74,7 @@ int64_t Func_dayofyear::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, ct.getTimeZone()); year = m_time.year; month = m_time.month; day = m_time.day; @@ -160,4 +160,3 @@ int64_t Func_dayofyear::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_div.cpp b/utils/funcexp/func_div.cpp index 7169992bb..d36e75c9e 100644 --- a/utils/funcexp/func_div.cpp +++ b/utils/funcexp/func_div.cpp @@ -108,4 +108,3 @@ string Func_div::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_elt.cpp b/utils/funcexp/func_elt.cpp index 22446095f..860dd1ae3 100644 --- a/utils/funcexp/func_elt.cpp +++ b/utils/funcexp/func_elt.cpp @@ -96,4 +96,3 @@ string Func_elt::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_exp.cpp b/utils/funcexp/func_exp.cpp index bab2fc926..d57b28556 100644 --- a/utils/funcexp/func_exp.cpp +++ b/utils/funcexp/func_exp.cpp @@ -110,4 +110,3 @@ long double Func_exp::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_extract.cpp b/utils/funcexp/func_extract.cpp index 066151aa3..b051f2676 100644 --- a/utils/funcexp/func_extract.cpp +++ b/utils/funcexp/func_extract.cpp @@ -202,7 +202,7 @@ int64_t Func_extract::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, ct.getTimeZone()); dataconvert::DateTime dt; dt.year = m_time.year; dt.month = m_time.month; @@ -254,4 +254,3 @@ int64_t Func_extract::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_find_in_set.cpp b/utils/funcexp/func_find_in_set.cpp index 94c1243dc..85b0c7e25 100644 --- a/utils/funcexp/func_find_in_set.cpp +++ b/utils/funcexp/func_find_in_set.cpp @@ -127,4 +127,3 @@ execplan::IDB_Decimal Func_find_in_set::getDecimalVal(rowgroup::Row& row, Functi } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_floor.cpp b/utils/funcexp/func_floor.cpp index c212d19c0..f497400b5 100644 --- a/utils/funcexp/func_floor.cpp +++ b/utils/funcexp/func_floor.cpp @@ -225,7 +225,7 @@ uint64_t Func_floor::getUintVal(Row& row, FunctionParm& parm, bool& isNull, case execplan::CalpontSystemCatalog::TIMESTAMP: { string str = - DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), op_ct.getTimeZone()); // strip off micro seconds str = str.substr(0, 14); @@ -519,7 +519,7 @@ IDB_Decimal Func_floor::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull case execplan::CalpontSystemCatalog::TIMESTAMP: { string str = - DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), op_ct.getTimeZone()); // strip off micro seconds str = str.substr(0, 14); @@ -555,4 +555,3 @@ IDB_Decimal Func_floor::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_from_days.cpp b/utils/funcexp/func_from_days.cpp index e9f45b421..a3e964ee8 100644 --- a/utils/funcexp/func_from_days.cpp +++ b/utils/funcexp/func_from_days.cpp @@ -80,4 +80,3 @@ int64_t Func_from_days::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_from_unixtime.cpp b/utils/funcexp/func_from_unixtime.cpp index cca4083be..b01bfab34 100644 --- a/utils/funcexp/func_from_unixtime.cpp +++ b/utils/funcexp/func_from_unixtime.cpp @@ -212,4 +212,3 @@ long double Func_from_unixtime::getLongDoubleVal(rowgroup::Row& row, FunctionPar } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_get_format.cpp b/utils/funcexp/func_get_format.cpp index d902c0873..00f513643 100644 --- a/utils/funcexp/func_get_format.cpp +++ b/utils/funcexp/func_get_format.cpp @@ -107,4 +107,3 @@ string Func_get_format::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_greatest.cpp b/utils/funcexp/func_greatest.cpp index e8562be20..b3dea5beb 100644 --- a/utils/funcexp/func_greatest.cpp +++ b/utils/funcexp/func_greatest.cpp @@ -240,4 +240,3 @@ int64_t Func_greatest::getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_hex.cpp b/utils/funcexp/func_hex.cpp index 133eca923..c1a111cb7 100644 --- a/utils/funcexp/func_hex.cpp +++ b/utils/funcexp/func_hex.cpp @@ -95,7 +95,7 @@ string Func_hex::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, if ((val <= (double)numeric_limits::min()) || (val >= (double)numeric_limits::max())) dec = ~(int64_t)0; else - dec = (uint64_t)(val + (val > 0 ? 0.5 : -0.5)); + dec = static_cast(static_cast(val + (val > 0 ? 0.5 : -0.5))); retval = helpers::convNumToStr(dec, ans, 16); break; @@ -140,4 +140,3 @@ string Func_hex::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_hour.cpp b/utils/funcexp/func_hour.cpp index b9ef7f823..0550ef3ef 100644 --- a/utils/funcexp/func_hour.cpp +++ b/utils/funcexp/func_hour.cpp @@ -117,7 +117,7 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNul dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); return m_time.hour; } @@ -156,4 +156,3 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_idbpartition.cpp b/utils/funcexp/func_idbpartition.cpp index 3b71cdd97..b7b5e6971 100644 --- a/utils/funcexp/func_idbpartition.cpp +++ b/utils/funcexp/func_idbpartition.cpp @@ -63,4 +63,3 @@ string Func_idbpartition::getStrVal(Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_if.cpp b/utils/funcexp/func_if.cpp index 3e208dc7d..d5941df8d 100644 --- a/utils/funcexp/func_if.cpp +++ b/utils/funcexp/func_if.cpp @@ -35,7 +35,7 @@ using namespace rowgroup; namespace { -bool boolVal(SPTP& parm, Row& row, const string& timeZone) +bool boolVal(SPTP& parm, Row& row, long timeZone) { bool ret = true; bool isNull = false; // Keep it local. We don't want to mess with the global one here. @@ -122,9 +122,9 @@ CalpontSystemCatalog::ColType Func_if::operationType(FunctionParm& fp, return ct; } -int64_t Func_if::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) +int64_t Func_if::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - if (boolVal(parm[0], row, timeZone())) + if (boolVal(parm[0], row, ct.getTimeZone())) { return parm[1]->data()->getIntVal(row, isNull); } @@ -134,9 +134,9 @@ int64_t Func_if::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSy } } -string Func_if::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) +string Func_if::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - if (boolVal(parm[0], row, timeZone())) + if (boolVal(parm[0], row, ct.getTimeZone())) { return parm[1]->data()->getStrVal(row, isNull); } @@ -146,9 +146,10 @@ string Func_if::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSys } } -IDB_Decimal Func_if::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) +IDB_Decimal Func_if::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - if (boolVal(parm[0], row, timeZone())) + if (boolVal(parm[0], row, ct.getTimeZone())) { return parm[1]->data()->getDecimalVal(row, isNull); } @@ -158,9 +159,9 @@ IDB_Decimal Func_if::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, C } } -double Func_if::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) +double Func_if::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - if (boolVal(parm[0], row, timeZone())) + if (boolVal(parm[0], row, ct.getTimeZone())) { return parm[1]->data()->getDoubleVal(row, isNull); } @@ -171,9 +172,9 @@ double Func_if::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, Calpont } long double Func_if::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNull, - CalpontSystemCatalog::ColType&) + CalpontSystemCatalog::ColType& ct) { - if (boolVal(parm[0], row, timeZone())) + if (boolVal(parm[0], row, ct.getTimeZone())) { return parm[1]->data()->getLongDoubleVal(row, isNull); } @@ -183,9 +184,9 @@ long double Func_if::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNull } } -int32_t Func_if::getDateIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) +int32_t Func_if::getDateIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - if (boolVal(parm[0], row, timeZone())) + if (boolVal(parm[0], row, ct.getTimeZone())) { return parm[1]->data()->getDateIntVal(row, isNull); } @@ -195,9 +196,10 @@ int32_t Func_if::getDateIntVal(Row& row, FunctionParm& parm, bool& isNull, Calpo } } -int64_t Func_if::getDatetimeIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) +int64_t Func_if::getDatetimeIntVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - if (boolVal(parm[0], row, timeZone())) + if (boolVal(parm[0], row, ct.getTimeZone())) { return parm[1]->data()->getDatetimeIntVal(row, isNull); } @@ -208,9 +210,9 @@ int64_t Func_if::getDatetimeIntVal(Row& row, FunctionParm& parm, bool& isNull, C } int64_t Func_if::getTimestampIntVal(Row& row, FunctionParm& parm, bool& isNull, - CalpontSystemCatalog::ColType&) + CalpontSystemCatalog::ColType& ct) { - if (boolVal(parm[0], row, timeZone())) + if (boolVal(parm[0], row, ct.getTimeZone())) { return parm[1]->data()->getTimestampIntVal(row, isNull); } @@ -220,9 +222,9 @@ int64_t Func_if::getTimestampIntVal(Row& row, FunctionParm& parm, bool& isNull, } } -int64_t Func_if::getTimeIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) +int64_t Func_if::getTimeIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - if (boolVal(parm[0], row, timeZone())) + if (boolVal(parm[0], row, ct.getTimeZone())) { return parm[1]->data()->getTimeIntVal(row, isNull); } @@ -232,4 +234,3 @@ int64_t Func_if::getTimeIntVal(Row& row, FunctionParm& parm, bool& isNull, Calpo } } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_ifnull.cpp b/utils/funcexp/func_ifnull.cpp index 52dae2556..a2450a5a9 100644 --- a/utils/funcexp/func_ifnull.cpp +++ b/utils/funcexp/func_ifnull.cpp @@ -205,4 +205,3 @@ bool Func_ifnull::getBoolVal(Row& row, FunctionParm& parm, bool& isNull, Calpont } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_in.cpp b/utils/funcexp/func_in.cpp index 89ed89593..d452fc242 100644 --- a/utils/funcexp/func_in.cpp +++ b/utils/funcexp/func_in.cpp @@ -386,4 +386,3 @@ bool Func_notin::getBoolVal(rowgroup::Row& row, FunctionParm& pm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_insert.cpp b/utils/funcexp/func_insert.cpp index 236752e96..72fd06e46 100644 --- a/utils/funcexp/func_insert.cpp +++ b/utils/funcexp/func_insert.cpp @@ -102,4 +102,3 @@ std::string Func_insert::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& i } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_instr.cpp b/utils/funcexp/func_instr.cpp index ab0379cce..71eb81d0a 100644 --- a/utils/funcexp/func_instr.cpp +++ b/utils/funcexp/func_instr.cpp @@ -85,4 +85,3 @@ int64_t Func_instr::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNu } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_isnull.cpp b/utils/funcexp/func_isnull.cpp index 48337b3ec..4ff7e643a 100644 --- a/utils/funcexp/func_isnull.cpp +++ b/utils/funcexp/func_isnull.cpp @@ -79,4 +79,3 @@ bool Func_isnull::getBoolVal(Row& row, FunctionParm& parm, bool& isNull, Calpont } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_last_day.cpp b/utils/funcexp/func_last_day.cpp index 9040c47d2..e74a2d514 100644 --- a/utils/funcexp/func_last_day.cpp +++ b/utils/funcexp/func_last_day.cpp @@ -74,7 +74,7 @@ int64_t Func_last_day::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& i TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); + gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); year = m_time.year; month = m_time.month; day = m_time.day; @@ -195,4 +195,3 @@ int64_t Func_last_day::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& i } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_lcase.cpp b/utils/funcexp/func_lcase.cpp index 7de396e47..ccc2dccb5 100644 --- a/utils/funcexp/func_lcase.cpp +++ b/utils/funcexp/func_lcase.cpp @@ -64,4 +64,3 @@ std::string Func_lcase::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_least.cpp b/utils/funcexp/func_least.cpp index add78cfa7..d488b7516 100644 --- a/utils/funcexp/func_least.cpp +++ b/utils/funcexp/func_least.cpp @@ -218,4 +218,3 @@ int64_t Func_least::getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_left.cpp b/utils/funcexp/func_left.cpp index a90d5f46c..de31d1770 100644 --- a/utils/funcexp/func_left.cpp +++ b/utils/funcexp/func_left.cpp @@ -73,4 +73,3 @@ std::string Func_left::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_length.cpp b/utils/funcexp/func_length.cpp index f63e2f1bb..a6f7236f5 100644 --- a/utils/funcexp/func_length.cpp +++ b/utils/funcexp/func_length.cpp @@ -55,4 +55,3 @@ int64_t Func_length::getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_lpad.cpp b/utils/funcexp/func_lpad.cpp index 0ad0bdc25..8a5544a6f 100644 --- a/utils/funcexp/func_lpad.cpp +++ b/utils/funcexp/func_lpad.cpp @@ -127,4 +127,3 @@ std::string Func_lpad::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_ltrim.cpp b/utils/funcexp/func_ltrim.cpp index 39fd7165f..3d7e96918 100644 --- a/utils/funcexp/func_ltrim.cpp +++ b/utils/funcexp/func_ltrim.cpp @@ -93,4 +93,3 @@ std::string Func_ltrim::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_ltrim_oracle.cpp b/utils/funcexp/func_ltrim_oracle.cpp index 234808e74..70fc1dc8b 100644 --- a/utils/funcexp/func_ltrim_oracle.cpp +++ b/utils/funcexp/func_ltrim_oracle.cpp @@ -97,4 +97,3 @@ std::string Func_ltrim_oracle::getStrVal(rowgroup::Row& row, FunctionParm& fp, b } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_makedate.cpp b/utils/funcexp/func_makedate.cpp index 0418508d3..fb49f54cc 100644 --- a/utils/funcexp/func_makedate.cpp +++ b/utils/funcexp/func_makedate.cpp @@ -190,4 +190,3 @@ string Func_makedate::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_maketime.cpp b/utils/funcexp/func_maketime.cpp index 6d0a9d903..ad918ccce 100644 --- a/utils/funcexp/func_maketime.cpp +++ b/utils/funcexp/func_maketime.cpp @@ -171,4 +171,3 @@ string Func_maketime::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_math.cpp b/utils/funcexp/func_math.cpp index 5f822d539..74a12aa7d 100644 --- a/utils/funcexp/func_math.cpp +++ b/utils/funcexp/func_math.cpp @@ -1786,7 +1786,7 @@ string Func_format::getStrVal(Row& row, FunctionParm& parm, bool& isNull, case execplan::CalpontSystemCatalog::TIMESTAMP: { value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), - timeZone()); + operationColType.getTimeZone()); } break; @@ -2276,4 +2276,3 @@ double Func_degrees::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, Ca } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_md5.cpp b/utils/funcexp/func_md5.cpp index 6ef058e84..0ce6b0a05 100644 --- a/utils/funcexp/func_md5.cpp +++ b/utils/funcexp/func_md5.cpp @@ -512,4 +512,3 @@ string Func_md5::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_microsecond.cpp b/utils/funcexp/func_microsecond.cpp index aca1e3aea..8c94647bf 100644 --- a/utils/funcexp/func_microsecond.cpp +++ b/utils/funcexp/func_microsecond.cpp @@ -137,4 +137,3 @@ int64_t Func_microsecond::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_minute.cpp b/utils/funcexp/func_minute.cpp index 4a5f594a7..8e5724547 100644 --- a/utils/funcexp/func_minute.cpp +++ b/utils/funcexp/func_minute.cpp @@ -117,7 +117,7 @@ int64_t Func_minute::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isN dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); return m_time.minute; } @@ -137,4 +137,3 @@ int64_t Func_minute::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_mod.cpp b/utils/funcexp/func_mod.cpp index bf33a6491..dff1a4e07 100644 --- a/utils/funcexp/func_mod.cpp +++ b/utils/funcexp/func_mod.cpp @@ -531,4 +531,3 @@ std::string Func_mod::getStrVal(Row& row, FunctionParm& fp, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_month.cpp b/utils/funcexp/func_month.cpp index 3f9136a04..ea66abf32 100644 --- a/utils/funcexp/func_month.cpp +++ b/utils/funcexp/func_month.cpp @@ -63,7 +63,7 @@ int64_t Func_month::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNu dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); return m_time.month; } @@ -142,4 +142,3 @@ int64_t Func_month::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNu } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_monthname.cpp b/utils/funcexp/func_monthname.cpp index 3f76db8ca..217a25a67 100644 --- a/utils/funcexp/func_monthname.cpp +++ b/utils/funcexp/func_monthname.cpp @@ -45,7 +45,7 @@ CalpontSystemCatalog::ColType Func_monthname::operationType(FunctionParm& fp, string Func_monthname::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - int32_t month = getIntVal(row, parm, isNull, op_ct); + int32_t month = getIntValInternal(row, parm, isNull, op_ct); if (month == -1) return ""; @@ -74,8 +74,8 @@ int64_t Func_monthname::getTimestampIntVal(rowgroup::Row& row, FunctionParm& par return val; } -int64_t Func_monthname::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, - CalpontSystemCatalog::ColType& op_ct) +int64_t Func_monthname::getIntValInternal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& op_ct) { int64_t val = 0; dataconvert::DateTime aDateTime; @@ -97,7 +97,7 @@ int64_t Func_monthname::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& dataconvert::TimeStamp timestamp(val); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, time, op_ct.getTimeZone()); return time.month; } @@ -165,12 +165,20 @@ int64_t Func_monthname::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& break; - default: isNull = true; return -1; + default: + isNull = true; + return -1; } return -1; } +int64_t Func_monthname::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + return getIntValInternal(row, parm, isNull, op_ct); +} + double Func_monthname::getDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { @@ -191,4 +199,3 @@ execplan::IDB_Decimal Func_monthname::getDecimalVal(rowgroup::Row& row, Function } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_nullif.cpp b/utils/funcexp/func_nullif.cpp index 77c07c674..3821151e4 100644 --- a/utils/funcexp/func_nullif.cpp +++ b/utils/funcexp/func_nullif.cpp @@ -955,7 +955,8 @@ execplan::IDB_Decimal Func_nullif::getDecimalVal(rowgroup::Row& row, FunctionPar string value; if (parm[1]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) - value = DataConvert::timestampToString1(parm[1]->data()->getTimestampIntVal(row, isNull), timeZone()); + value = + DataConvert::timestampToString1(parm[1]->data()->getTimestampIntVal(row, isNull), op_ct.getTimeZone()); else value = DataConvert::datetimeToString1(parm[1]->data()->getDatetimeIntVal(row, isNull)); @@ -963,7 +964,7 @@ execplan::IDB_Decimal Func_nullif::getDecimalVal(rowgroup::Row& row, FunctionPar { // strip off micro seconds value = value.substr(0, 14); - int64_t x = atoll(value.c_str()); + x = atoll(value.c_str()); if (s > 5) s = 0; @@ -1021,4 +1022,3 @@ execplan::IDB_Decimal Func_nullif::getDecimalVal(rowgroup::Row& row, FunctionPar } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_period_add.cpp b/utils/funcexp/func_period_add.cpp index 23b964151..4038651ed 100644 --- a/utils/funcexp/func_period_add.cpp +++ b/utils/funcexp/func_period_add.cpp @@ -94,4 +94,3 @@ int64_t Func_period_add::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_period_diff.cpp b/utils/funcexp/func_period_diff.cpp index 470ab60c9..080d15a42 100644 --- a/utils/funcexp/func_period_diff.cpp +++ b/utils/funcexp/func_period_diff.cpp @@ -113,4 +113,3 @@ int64_t Func_period_diff::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_pow.cpp b/utils/funcexp/func_pow.cpp index 734b50a22..236e7e2c5 100644 --- a/utils/funcexp/func_pow.cpp +++ b/utils/funcexp/func_pow.cpp @@ -121,4 +121,3 @@ long double Func_pow::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_quarter.cpp b/utils/funcexp/func_quarter.cpp index c4417497f..5f1fd0033 100644 --- a/utils/funcexp/func_quarter.cpp +++ b/utils/funcexp/func_quarter.cpp @@ -67,7 +67,7 @@ int64_t Func_quarter::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); month = m_time.month; break; } @@ -139,4 +139,3 @@ int64_t Func_quarter::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_quote.cpp b/utils/funcexp/func_quote.cpp index 05ce4cecf..b82cf7baa 100644 --- a/utils/funcexp/func_quote.cpp +++ b/utils/funcexp/func_quote.cpp @@ -80,4 +80,3 @@ std::string Func_quote::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_rand.cpp b/utils/funcexp/func_rand.cpp index 223949bbf..b7da263d9 100644 --- a/utils/funcexp/func_rand.cpp +++ b/utils/funcexp/func_rand.cpp @@ -98,4 +98,3 @@ double Func_rand::getDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_regexp.cpp b/utils/funcexp/func_regexp.cpp index f918aa7b4..eef59d784 100644 --- a/utils/funcexp/func_regexp.cpp +++ b/utils/funcexp/func_regexp.cpp @@ -28,7 +28,7 @@ using namespace std; #ifdef __linux__ #include #else -#include +#include using namespace boost; #endif @@ -48,7 +48,7 @@ using namespace logging; namespace { inline bool getBool(rowgroup::Row& row, funcexp::FunctionParm& pm, bool& isNull, - CalpontSystemCatalog::ColType& ct, const string& timeZone) + CalpontSystemCatalog::ColType& ct, long timeZone) { string expr; string pattern; @@ -226,8 +226,8 @@ inline bool getBool(rowgroup::Row& row, funcexp::FunctionParm& pm, bool& isNull, return false; #else - regex pat(pattern.c_str()); - return regex_search(expr.c_str(), pat); + std::regex pat(pattern.c_str()); + return std::regex_search(expr.c_str(), pat); #endif } @@ -244,8 +244,7 @@ CalpontSystemCatalog::ColType Func_regexp::operationType(FunctionParm& fp, bool Func_regexp::getBoolVal(rowgroup::Row& row, FunctionParm& pm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return getBool(row, pm, isNull, ct, timeZone()) && !isNull; + return getBool(row, pm, isNull, ct, ct.getTimeZone()) && !isNull; } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_repeat.cpp b/utils/funcexp/func_repeat.cpp index 6b0e6ace0..ebbfc355a 100644 --- a/utils/funcexp/func_repeat.cpp +++ b/utils/funcexp/func_repeat.cpp @@ -90,4 +90,3 @@ std::string Func_repeat::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& i } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_replace.cpp b/utils/funcexp/func_replace.cpp index 05e0fe359..da18fb3f1 100644 --- a/utils/funcexp/func_replace.cpp +++ b/utils/funcexp/func_replace.cpp @@ -173,4 +173,3 @@ std::string Func_replace::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_replace_oracle.cpp b/utils/funcexp/func_replace_oracle.cpp index b8eb7f6ef..113b49d96 100644 --- a/utils/funcexp/func_replace_oracle.cpp +++ b/utils/funcexp/func_replace_oracle.cpp @@ -167,4 +167,3 @@ std::string Func_replace_oracle::getStrVal(rowgroup::Row& row, FunctionParm& fp, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_reverse.cpp b/utils/funcexp/func_reverse.cpp index 2b383ee7f..db57d0953 100644 --- a/utils/funcexp/func_reverse.cpp +++ b/utils/funcexp/func_reverse.cpp @@ -93,4 +93,3 @@ std::string Func_reverse::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_right.cpp b/utils/funcexp/func_right.cpp index 4615256e5..2c1ec612a 100644 --- a/utils/funcexp/func_right.cpp +++ b/utils/funcexp/func_right.cpp @@ -73,4 +73,3 @@ std::string Func_right::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_round.cpp b/utils/funcexp/func_round.cpp index 293159570..d5a973792 100644 --- a/utils/funcexp/func_round.cpp +++ b/utils/funcexp/func_round.cpp @@ -41,6 +41,8 @@ using namespace logging; #include "funchelpers.h" +#include "exceptclasses.h" + namespace { using namespace funcexp; @@ -136,18 +138,27 @@ int64_t Func_round::getIntVal(Row& row, FunctionParm& parm, bool& isNull, uint64_t Func_round::getUintVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - uint64_t x; - if (UNLIKELY(op_ct.colDataType == execplan::CalpontSystemCatalog::DATE)) + IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); + + if (!op_ct.isWideDecimalType()) { - IDB_Decimal d = getDecimalVal(row, parm, isNull, op_ct); - x = static_cast(d.value); + if (x.scale > 0) + { + while (x.scale-- > 0) + x.value /= 10; + } + else + { + while (x.scale++ < 0) + x.value *= 10; + } + + return x.value; } else { - x = parm[0]->data()->getUintVal(row, isNull); + return static_cast(x.getIntegralPart()); } - - return x; } double Func_round::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, @@ -434,10 +445,11 @@ IDB_Decimal Func_round::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull { uint64_t x = parm[0]->data()->getUintVal(row, isNull); - if (x > (uint64_t)helpers::maxNumber_c[18]) - { - x = helpers::maxNumber_c[18]; - } + // why it is here at all??? + // if (x > (uint64_t)helpers::maxNumber_c[18]) + //{ + // x = helpers::maxNumber_c[18]; + //} decimal.value = x; decimal.scale = 0; @@ -569,7 +581,7 @@ IDB_Decimal Func_round::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull string value; if (op_ct.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), - timeZone()); + op_ct.getTimeZone()); else value = dataconvert::DataConvert::datetimeToString1(parm[0]->data()->getDatetimeIntVal(row, isNull)); @@ -645,7 +657,7 @@ string Func_round::getStrVal(Row& row, FunctionParm& parm, bool& isNull, Calpont { IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); int64_t e = (x.scale < 0) ? (-x.scale) : x.scale; - int64_t p = 1; + [[maybe_unused]] int64_t p = 1; while (e-- > 0) p *= 10; @@ -709,5 +721,10 @@ int64_t Func_round::getDatetimeIntVal(Row& row, FunctionParm& parm, bool& isNull return parm[0]->data()->getIntVal(row, isNull); } +int64_t Func_round::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + return parm[0]->data()->getTimestampIntVal(row, isNull); +} + } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_rpad.cpp b/utils/funcexp/func_rpad.cpp index fba47bfae..a2a538ff3 100644 --- a/utils/funcexp/func_rpad.cpp +++ b/utils/funcexp/func_rpad.cpp @@ -127,4 +127,3 @@ std::string Func_rpad::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_rtrim.cpp b/utils/funcexp/func_rtrim.cpp index 426ec8712..ee61906b4 100644 --- a/utils/funcexp/func_rtrim.cpp +++ b/utils/funcexp/func_rtrim.cpp @@ -157,4 +157,3 @@ std::string Func_rtrim::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_rtrim_oracle.cpp b/utils/funcexp/func_rtrim_oracle.cpp index 1b47d7ddf..93713e113 100644 --- a/utils/funcexp/func_rtrim_oracle.cpp +++ b/utils/funcexp/func_rtrim_oracle.cpp @@ -161,4 +161,3 @@ std::string Func_rtrim_oracle::getStrVal(rowgroup::Row& row, FunctionParm& fp, b } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_sec_to_time.cpp b/utils/funcexp/func_sec_to_time.cpp index d3b4af675..9a2fef4c4 100644 --- a/utils/funcexp/func_sec_to_time.cpp +++ b/utils/funcexp/func_sec_to_time.cpp @@ -231,4 +231,3 @@ execplan::IDB_Decimal Func_sec_to_time::getDecimalVal(rowgroup::Row& row, Functi } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_second.cpp b/utils/funcexp/func_second.cpp index e07d29176..6136c3ede 100644 --- a/utils/funcexp/func_second.cpp +++ b/utils/funcexp/func_second.cpp @@ -116,7 +116,7 @@ int64_t Func_second::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isN dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); return m_time.second; } @@ -144,4 +144,3 @@ int64_t Func_second::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_sha.cpp b/utils/funcexp/func_sha.cpp index bc74a0a2a..ec3fac1f4 100644 --- a/utils/funcexp/func_sha.cpp +++ b/utils/funcexp/func_sha.cpp @@ -655,4 +655,3 @@ string Func_sha::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_sign.cpp b/utils/funcexp/func_sign.cpp index a0cece67a..426fbb0c3 100644 --- a/utils/funcexp/func_sign.cpp +++ b/utils/funcexp/func_sign.cpp @@ -71,4 +71,3 @@ string Func_sign::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_space.cpp b/utils/funcexp/func_space.cpp index 93e765a56..9a9a85789 100644 --- a/utils/funcexp/func_space.cpp +++ b/utils/funcexp/func_space.cpp @@ -57,4 +57,3 @@ std::string Func_space::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_str_to_date.cpp b/utils/funcexp/func_str_to_date.cpp index ff1f53356..ab0fb4f61 100644 --- a/utils/funcexp/func_str_to_date.cpp +++ b/utils/funcexp/func_str_to_date.cpp @@ -42,7 +42,7 @@ namespace using namespace funcexp; dataconvert::DateTime getDateTime(rowgroup::Row& row, FunctionParm& parm, bool& isNull, - CalpontSystemCatalog::ColType& ct, const string& timeZone) + CalpontSystemCatalog::ColType& ct, long timeZone) { TimeExtractor extractor; dataconvert::DateTime dateTime; @@ -186,7 +186,7 @@ string Func_str_to_date::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, timeZone()); + dateTime = getDateTime(row, parm, isNull, ct, ct.getTimeZone()); string convertedDate = dataconvert::DataConvert::datetimeToString(*((long long*)&dateTime)); return convertedDate; } @@ -195,7 +195,7 @@ int32_t Func_str_to_date::getDateIntVal(rowgroup::Row& row, FunctionParm& parm, CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, timeZone()); + dateTime = getDateTime(row, parm, isNull, ct, ct.getTimeZone()); int64_t time = *(reinterpret_cast(&dateTime)); return ((((int32_t)(time >> 32)) & 0xFFFFFFC0) | 0x3E); } @@ -204,7 +204,7 @@ int64_t Func_str_to_date::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& pa CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, timeZone()); + dateTime = getDateTime(row, parm, isNull, ct, ct.getTimeZone()); int64_t time = *(reinterpret_cast(&dateTime)); return time; } @@ -213,7 +213,7 @@ int64_t Func_str_to_date::getTimestampIntVal(rowgroup::Row& row, FunctionParm& p CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, timeZone()); + dateTime = getDateTime(row, parm, isNull, ct, ct.getTimeZone()); dataconvert::TimeStamp timestamp; dataconvert::MySQLTime m_time; m_time.year = dateTime.year; @@ -223,7 +223,7 @@ int64_t Func_str_to_date::getTimestampIntVal(rowgroup::Row& row, FunctionParm& p m_time.minute = dateTime.minute; m_time.second = dateTime.second; bool isValid = true; - int64_t seconds = mySQLTimeToGmtSec(m_time, timeZone(), isValid); + int64_t seconds = mySQLTimeToGmtSec(m_time, ct.getTimeZone(), isValid); if (!isValid) { timestamp = -1; @@ -243,7 +243,7 @@ int64_t Func_str_to_date::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, { dataconvert::DateTime dateTime; dataconvert::Time retTime; - dateTime = getDateTime(row, parm, isNull, ct, timeZone()); + dateTime = getDateTime(row, parm, isNull, ct, ct.getTimeZone()); retTime.day = 0; retTime.is_neg = false; retTime.hour = dateTime.hour; @@ -258,10 +258,9 @@ int64_t Func_str_to_date::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, timeZone()); + dateTime = getDateTime(row, parm, isNull, ct, ct.getTimeZone()); int64_t time = *(reinterpret_cast(&dateTime)); return time; } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_strcmp.cpp b/utils/funcexp/func_strcmp.cpp index a16c68cc4..63b3b0b22 100644 --- a/utils/funcexp/func_strcmp.cpp +++ b/utils/funcexp/func_strcmp.cpp @@ -74,4 +74,3 @@ std::string Func_strcmp::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& i } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_substr.cpp b/utils/funcexp/func_substr.cpp index cb45acc0b..9a39a4f10 100644 --- a/utils/funcexp/func_substr.cpp +++ b/utils/funcexp/func_substr.cpp @@ -98,4 +98,3 @@ std::string Func_substr::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& i } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_substring_index.cpp b/utils/funcexp/func_substring_index.cpp index ffae25e5e..82f94a7ee 100644 --- a/utils/funcexp/func_substring_index.cpp +++ b/utils/funcexp/func_substring_index.cpp @@ -205,4 +205,3 @@ std::string Func_substring_index::getStrVal(rowgroup::Row& row, FunctionParm& fp } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_sysdate.cpp b/utils/funcexp/func_sysdate.cpp index b30e66c53..780492f37 100644 --- a/utils/funcexp/func_sysdate.cpp +++ b/utils/funcexp/func_sysdate.cpp @@ -105,4 +105,3 @@ int64_t Func_sysdate::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_time.cpp b/utils/funcexp/func_time.cpp index a859c4843..e898f7820 100644 --- a/utils/funcexp/func_time.cpp +++ b/utils/funcexp/func_time.cpp @@ -45,7 +45,7 @@ CalpontSystemCatalog::ColType Func_time::operationType(FunctionParm& fp, } string Func_time::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, - CalpontSystemCatalog::ColType&) + CalpontSystemCatalog::ColType& ct) { int64_t val = 0; @@ -126,7 +126,7 @@ string Func_time::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, ct.getTimeZone()); dataconvert::Time time; time.hour = m_time.hour; time.minute = m_time.minute; @@ -169,4 +169,3 @@ double Func_time::getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_time_format.cpp b/utils/funcexp/func_time_format.cpp index 5b971b958..1205e21ba 100644 --- a/utils/funcexp/func_time_format.cpp +++ b/utils/funcexp/func_time_format.cpp @@ -45,7 +45,7 @@ CalpontSystemCatalog::ColType Func_time_format::operationType(FunctionParm& fp, } string Func_time_format::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, - CalpontSystemCatalog::ColType&) + CalpontSystemCatalog::ColType& ct) { // assume 256 is enough. assume not allowing incomplete date char buf[256]; @@ -72,7 +72,7 @@ string Func_time_format::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, ct.getTimeZone()); hour = m_time.hour; min = m_time.minute; sec = m_time.second; @@ -229,4 +229,3 @@ string Func_time_format::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_time_to_sec.cpp b/utils/funcexp/func_time_to_sec.cpp index d6b5de7cf..3cf718aa2 100644 --- a/utils/funcexp/func_time_to_sec.cpp +++ b/utils/funcexp/func_time_to_sec.cpp @@ -70,7 +70,7 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool dataconvert::TimeStamp timestamp(val); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, time, op_ct.getTimeZone()); hour = time.hour; min = time.minute; sec = time.second; @@ -195,4 +195,3 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_timediff.cpp b/utils/funcexp/func_timediff.cpp index 2477943ab..030004761 100644 --- a/utils/funcexp/func_timediff.cpp +++ b/utils/funcexp/func_timediff.cpp @@ -142,16 +142,11 @@ string Func_timediff::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& is case execplan::CalpontSystemCatalog::TIMESTAMP: { - if (type1 != type2) - { - isNull = true; - break; - } int64_t temp = parm[0]->data()->getTimestampIntVal(row, isNull); dataconvert::TimeStamp timestamp(temp); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, time, ct.getTimeZone()); dataconvert::DateTime dt; dt.year = time.year; dt.month = time.month; @@ -239,7 +234,7 @@ string Func_timediff::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& is dataconvert::TimeStamp timestamp(temp); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, time, ct.getTimeZone()); dataconvert::DateTime dt; dt.year = time.year; dt.month = time.month; @@ -328,7 +323,7 @@ int64_t Func_timediff::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, int64_t Func_timediff::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), timeZone()); + return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), ct.getTimeZone()); } int64_t Func_timediff::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, @@ -354,4 +349,3 @@ double Func_timediff::getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& i } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_timestampdiff.cpp b/utils/funcexp/func_timestampdiff.cpp index 83024c069..6ad2f3952 100644 --- a/utils/funcexp/func_timestampdiff.cpp +++ b/utils/funcexp/func_timestampdiff.cpp @@ -55,7 +55,7 @@ int64_t Func_timestampdiff::getIntVal(rowgroup::Row& row, FunctionParm& parm, bo TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); + gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); dt1.year = m_time.year; dt1.month = m_time.month; dt1.day = m_time.day; @@ -82,7 +82,7 @@ int64_t Func_timestampdiff::getIntVal(rowgroup::Row& row, FunctionParm& parm, bo TimeStamp timestamp(parm[1]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); + gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); dt2.year = m_time.year; dt2.month = m_time.month; dt2.day = m_time.day; @@ -196,4 +196,3 @@ int64_t Func_timestampdiff::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_to_days.cpp b/utils/funcexp/func_to_days.cpp index 79fc165b4..d6f831be1 100644 --- a/utils/funcexp/func_to_days.cpp +++ b/utils/funcexp/func_to_days.cpp @@ -85,7 +85,7 @@ int64_t Func_to_days::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); year = m_time.year; month = m_time.month; day = m_time.day; @@ -151,4 +151,3 @@ int64_t Func_to_days::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_trim.cpp b/utils/funcexp/func_trim.cpp index 636afe44e..d47823563 100644 --- a/utils/funcexp/func_trim.cpp +++ b/utils/funcexp/func_trim.cpp @@ -168,4 +168,3 @@ std::string Func_trim::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_trim_oracle.cpp b/utils/funcexp/func_trim_oracle.cpp index 5e1226c58..4daec213f 100644 --- a/utils/funcexp/func_trim_oracle.cpp +++ b/utils/funcexp/func_trim_oracle.cpp @@ -166,4 +166,3 @@ std::string Func_trim_oracle::getStrVal(rowgroup::Row& row, FunctionParm& fp, bo } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_truncate.cpp b/utils/funcexp/func_truncate.cpp index f9b7826d0..7a529c00f 100644 --- a/utils/funcexp/func_truncate.cpp +++ b/utils/funcexp/func_truncate.cpp @@ -522,7 +522,7 @@ IDB_Decimal Func_truncate::getDecimalVal(Row& row, FunctionParm& parm, bool& isN { // strip off micro seconds value = value.substr(0, 14); - int64_t x = atoll(value.c_str()); + x = atoll(value.c_str()); if (s > 5) s = 0; @@ -560,7 +560,7 @@ IDB_Decimal Func_truncate::getDecimalVal(Row& row, FunctionParm& parm, bool& isN int64_t x = 0; string value = - DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), op_ct.getTimeZone()); s = parm[1]->data()->getIntVal(row, isNull); @@ -613,7 +613,7 @@ IDB_Decimal Func_truncate::getDecimalVal(Row& row, FunctionParm& parm, bool& isN { // strip off micro seconds value = value.substr(0, 14); - int64_t x = atoll(value.c_str()); + x = atoll(value.c_str()); if (s > 5) s = 0; @@ -714,5 +714,10 @@ string Func_truncate::getStrVal(Row& row, FunctionParm& parm, bool& isNull, return x.toString(true); } +int64_t Func_truncate::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + return parm[0]->data()->getTimestampIntVal(row, isNull); +} + } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_ucase.cpp b/utils/funcexp/func_ucase.cpp index 4d78fb8b2..578ad5e07 100644 --- a/utils/funcexp/func_ucase.cpp +++ b/utils/funcexp/func_ucase.cpp @@ -73,4 +73,3 @@ std::string Func_ucase::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_unhex.cpp b/utils/funcexp/func_unhex.cpp index 38da2cb2d..92ef5fa3e 100644 --- a/utils/funcexp/func_unhex.cpp +++ b/utils/funcexp/func_unhex.cpp @@ -112,4 +112,3 @@ string Func_unhex::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_unix_timestamp.cpp b/utils/funcexp/func_unix_timestamp.cpp index d5b65fe25..f9c81e6ba 100644 --- a/utils/funcexp/func_unix_timestamp.cpp +++ b/utils/funcexp/func_unix_timestamp.cpp @@ -223,4 +223,3 @@ string Func_unix_timestamp::getStrVal(rowgroup::Row& row, FunctionParm& parm, bo } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_week.cpp b/utils/funcexp/func_week.cpp index f0fe91afc..e6ab52f35 100644 --- a/utils/funcexp/func_week.cpp +++ b/utils/funcexp/func_week.cpp @@ -76,7 +76,7 @@ int64_t Func_week::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNul dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); year = m_time.year; month = m_time.month; day = m_time.day; @@ -169,4 +169,3 @@ int64_t Func_week::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_weekday.cpp b/utils/funcexp/func_weekday.cpp index a6fbbcceb..e78d12c66 100644 --- a/utils/funcexp/func_weekday.cpp +++ b/utils/funcexp/func_weekday.cpp @@ -73,7 +73,7 @@ int64_t Func_weekday::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); year = m_time.year; month = m_time.month; day = m_time.day; @@ -164,4 +164,3 @@ int64_t Func_weekday::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_year.cpp b/utils/funcexp/func_year.cpp index 0b64c2534..9ea12fbdf 100644 --- a/utils/funcexp/func_year.cpp +++ b/utils/funcexp/func_year.cpp @@ -63,7 +63,7 @@ int64_t Func_year::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNul dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); return m_time.year; } @@ -140,4 +140,3 @@ int64_t Func_year::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_yearweek.cpp b/utils/funcexp/func_yearweek.cpp index 3e59dd120..2ad941396 100644 --- a/utils/funcexp/func_yearweek.cpp +++ b/utils/funcexp/func_yearweek.cpp @@ -78,7 +78,7 @@ int64_t Func_yearweek::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& i dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); year = m_time.year; month = m_time.month; day = m_time.day; @@ -173,4 +173,3 @@ int64_t Func_yearweek::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& i } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/funchelpers.h b/utils/funcexp/funchelpers.h index 73586aef6..ec0e6f22c 100644 --- a/utils/funcexp/funchelpers.h +++ b/utils/funcexp/funchelpers.h @@ -32,7 +32,6 @@ #include #include -#include #include #include "dataconvert.h" diff --git a/utils/funcexp/functor.cpp b/utils/funcexp/functor.cpp index ce83903cb..3da34944c 100644 --- a/utils/funcexp/functor.cpp +++ b/utils/funcexp/functor.cpp @@ -105,9 +105,9 @@ uint64_t Func::stringToDatetime(const string str) return ret; } -uint64_t Func::stringToTimestamp(const string str) +uint64_t Func::stringToTimestamp(const std::string& str, long timeZone) { - int64_t ret = DataConvert::stringToTimestamp(str, timeZone()); + int64_t ret = DataConvert::stringToTimestamp(str, timeZone); if (ret == -1) { @@ -361,4 +361,3 @@ string Func::longDoubleToString(long double ld) } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/functor.h b/utils/funcexp/functor.h index 26b7d04c7..1cc73879f 100644 --- a/utils/funcexp/functor.h +++ b/utils/funcexp/functor.h @@ -73,17 +73,6 @@ class Func fFuncName = funcName; } - const std::string timeZone() const - { - std::unique_lock l(tzMutex); - return fTimeZone; - } - void timeZone(const std::string timeZone) - { - std::unique_lock l(tzMutex); - fTimeZone = timeZone; - } - void raiseIllegalParameterDataTypeError(const execplan::CalpontSystemCatalog::ColType& colType) const { std::ostringstream oss; @@ -177,7 +166,7 @@ class Func protected: virtual uint32_t stringToDate(std::string); virtual uint64_t stringToDatetime(std::string); - virtual uint64_t stringToTimestamp(std::string); + virtual uint64_t stringToTimestamp(const std::string&, long); virtual int64_t stringToTime(std::string); virtual uint32_t intToDate(int64_t); @@ -205,9 +194,6 @@ class Func float fFloatNullVal; double fDoubleNullVal; long double fLongDoubleNullVal; - - std::string fTimeZone; - mutable std::mutex tzMutex; }; class ParmTSInt64 : public datatypes::TSInt64Null @@ -216,7 +202,7 @@ class ParmTSInt64 : public datatypes::TSInt64Null ParmTSInt64() { } - ParmTSInt64(rowgroup::Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc) + ParmTSInt64(rowgroup::Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc, long timeZone) : TSInt64Null(parm->data()->toTSInt64Null(row)) { } @@ -228,7 +214,7 @@ class ParmTUInt64 : public datatypes::TUInt64Null ParmTUInt64() { } - ParmTUInt64(rowgroup::Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc) + ParmTUInt64(rowgroup::Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc, long timeZone) : TUInt64Null(parm->data()->toTUInt64Null(row)) { } @@ -240,8 +226,8 @@ class Arg2Lazy public: TA a; TB b; - Arg2Lazy(rowgroup::Row& row, FunctionParm& parm, const Func& thisFunc) - : a(row, parm[0], thisFunc), b(a.isNull() ? TB() : TB(row, parm[1], thisFunc)) + Arg2Lazy(rowgroup::Row& row, FunctionParm& parm, const Func& thisFunc, long timeZone) + : a(row, parm[0], thisFunc, timeZone), b(a.isNull() ? TB() : TB(row, parm[1], thisFunc, timeZone)) { } }; @@ -252,8 +238,8 @@ class Arg2Eager public: TA a; TB b; - Arg2Eager(rowgroup::Row& row, FunctionParm& parm, const Func& thisFunc) - : a(row, parm[0], thisFunc), b(row, parm[1], thisFunc) + Arg2Eager(rowgroup::Row& row, FunctionParm& parm, const Func& thisFunc, long timeZone) + : a(row, parm[0], thisFunc, timeZone), b(row, parm[1], thisFunc, timeZone) { } }; diff --git a/utils/funcexp/functor_real.h b/utils/funcexp/functor_real.h index 59d4a05ab..60ccbe5e0 100644 --- a/utils/funcexp/functor_real.h +++ b/utils/funcexp/functor_real.h @@ -180,6 +180,9 @@ class Func_round : public Func_Real int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; /** @brief Func_truncate class @@ -214,6 +217,9 @@ class Func_truncate : public Func_Real execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; /** @brief Func_ceil class diff --git a/utils/funcexp/functor_str.h b/utils/funcexp/functor_str.h index 4ed4abc95..37657b0f3 100644 --- a/utils/funcexp/functor_str.h +++ b/utils/funcexp/functor_str.h @@ -94,7 +94,7 @@ class Func_Str : public Func execplan::CalpontSystemCatalog::ColType& op_ct) { std::string str = getStrVal(row, fp, isNull, op_ct); - return (isNull ? 0 : stringToTimestamp(str)); + return (isNull ? 0 : stringToTimestamp(str, op_ct.getTimeZone())); } int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -651,6 +651,8 @@ class Func_monthname : public Func_Str int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntValInternal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); diff --git a/utils/idbdatafile/CMakeLists.txt b/utils/idbdatafile/CMakeLists.txt index c51cc60fb..a4b62074f 100644 --- a/utils/idbdatafile/CMakeLists.txt +++ b/utils/idbdatafile/CMakeLists.txt @@ -15,7 +15,7 @@ set(idbdatafile_LIB_SRCS add_library(idbdatafile SHARED ${idbdatafile_LIB_SRCS}) -target_link_libraries(idbdatafile ${NETSNMP_LIBRARIES} ${ENGINE_OAM_LIBS}) +target_link_libraries(idbdatafile ${NETSNMP_LIBRARIES} ${ENGINE_OAM_LIBS} Boost::filesystem Boost::system) target_compile_definitions(idbdatafile PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) diff --git a/utils/joiner/joiner.h b/utils/joiner/joiner.h index e61370dd6..58661489b 100644 --- a/utils/joiner/joiner.h +++ b/utils/joiner/joiner.h @@ -20,41 +20,11 @@ #include #include #include -#ifdef _MSC_VER -#include -#else + #include -#endif #include "../common/simpleallocator.h" - -#ifndef _HASHFIX_ -#define _HASHFIX_ -#ifndef __LP64__ -#if __GNUC__ == 4 && __GNUC_MINOR__ < 2 -// This is needed for /usr/include/c++/4.1.1/tr1/functional on 32-bit compiles -// tr1_hashtable_define_trivial_hash(long long unsigned int); -namespace std -{ -namespace tr1 -{ -template <> -struct hash : public std::unary_function -{ - std::size_t operator()(long long unsigned int val) const - { - return static_cast(val); - } -}; -} // namespace tr1 -} // namespace std -#endif -#endif -#endif - -#define NO_DATALISTS #include "../joblist/elementtype.h" -#undef NO_DATALISTS namespace joiner { diff --git a/utils/joiner/joinpartition.cpp b/utils/joiner/joinpartition.cpp index 5fa2ec706..75680ca88 100644 --- a/utils/joiner/joinpartition.cpp +++ b/utils/joiner/joinpartition.cpp @@ -155,7 +155,6 @@ JoinPartition::JoinPartition(const JoinPartition& jp, bool splitMode) , nextSmallOffset(0) , nextLargeOffset(0) { - boost::posix_time::ptime t; ostringstream os; fileMode = true; diff --git a/utils/joiner/tuplejoiner.cpp b/utils/joiner/tuplejoiner.cpp index fc5a5052f..ac925e6ea 100644 --- a/utils/joiner/tuplejoiner.cpp +++ b/utils/joiner/tuplejoiner.cpp @@ -603,7 +603,7 @@ void TupleJoiner::match(rowgroup::Row& largeSideRow, uint32_t largeRowIndex, uin if (UNLIKELY(inUM() && (joinType & MATCHNULLS) && !isNull && !typelessJoin)) { - if (smallRG.getColType(largeKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE) + if (largeRG.getColType(largeKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE) { uint bucket = bucketPicker((char*)&(joblist::LONGDOUBLENULL), sizeof(joblist::LONGDOUBLENULL), bpSeed) & bucketMask; @@ -612,7 +612,7 @@ void TupleJoiner::match(rowgroup::Row& largeSideRow, uint32_t largeRowIndex, uin for (; range.first != range.second; ++range.first) matches->push_back(range.first->second); } - else if (!smallRG.usesStringTable()) + else if (!largeRG.usesStringTable()) { auto nullVal = getJoinNullValue(); uint bucket = bucketPicker((char*)&nullVal, sizeof(nullVal), bpSeed) & bucketMask; diff --git a/utils/json/json.hpp b/utils/json/json.hpp new file mode 100644 index 000000000..e40e3b05b --- /dev/null +++ b/utils/json/json.hpp @@ -0,0 +1,22109 @@ +/* + __ _____ _____ _____ + __| | __| | | | JSON for Modern C++ +| | |__ | | | | | | version 3.10.5 +|_____|_____|_____|_|___| https://github.com/nlohmann/json + +Licensed under the MIT License . +SPDX-License-Identifier: MIT +Copyright (c) 2013-2022 Niels Lohmann . + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +/****************************************************************************\ + * Note on documentation: The source files contain links to the online * + * documentation of the public API at https://json.nlohmann.me. This URL * + * contains the most recent documentation and should also be applicable to * + * previous versions; documentation for deprecated functions is not * + * removed, but marked deprecated. See "Generate documentation" section in * + * file doc/README.md. * +\****************************************************************************/ + +#ifndef INCLUDE_NLOHMANN_JSON_HPP_ +#define INCLUDE_NLOHMANN_JSON_HPP_ + +#define NLOHMANN_JSON_VERSION_MAJOR 3 +#define NLOHMANN_JSON_VERSION_MINOR 10 +#define NLOHMANN_JSON_VERSION_PATCH 5 + +#include // all_of, find, for_each +#include // nullptr_t, ptrdiff_t, size_t +#include // hash, less +#include // initializer_list +#ifndef JSON_NO_IO + #include // istream, ostream +#endif // JSON_NO_IO +#include // random_access_iterator_tag +#include // unique_ptr +#include // accumulate +#include // string, stoi, to_string +#include // declval, forward, move, pair, swap +#include // vector + +// #include + + +#include +#include + +// #include + + +#include // transform +#include // array +#include // forward_list +#include // inserter, front_inserter, end +#include // map +#include // string +#include // tuple, make_tuple +#include // is_arithmetic, is_same, is_enum, underlying_type, is_convertible +#include // unordered_map +#include // pair, declval +#include // valarray + +// #include + + +#include // exception +#include // runtime_error +#include // to_string +#include // vector + +// #include + + +#include // array +#include // size_t +#include // uint8_t +#include // string + +namespace nlohmann +{ +namespace detail +{ +/////////////////////////// +// JSON type enumeration // +/////////////////////////// + +/*! +@brief the JSON type enumeration + +This enumeration collects the different JSON types. It is internally used to +distinguish the stored values, and the functions @ref basic_json::is_null(), +@ref basic_json::is_object(), @ref basic_json::is_array(), +@ref basic_json::is_string(), @ref basic_json::is_boolean(), +@ref basic_json::is_number() (with @ref basic_json::is_number_integer(), +@ref basic_json::is_number_unsigned(), and @ref basic_json::is_number_float()), +@ref basic_json::is_discarded(), @ref basic_json::is_primitive(), and +@ref basic_json::is_structured() rely on it. + +@note There are three enumeration entries (number_integer, number_unsigned, and +number_float), because the library distinguishes these three types for numbers: +@ref basic_json::number_unsigned_t is used for unsigned integers, +@ref basic_json::number_integer_t is used for signed integers, and +@ref basic_json::number_float_t is used for floating-point numbers or to +approximate integers which do not fit in the limits of their respective type. + +@sa see @ref basic_json::basic_json(const value_t value_type) -- create a JSON +value with the default value for a given type + +@since version 1.0.0 +*/ +enum class value_t : std::uint8_t +{ + null, ///< null value + object, ///< object (unordered set of name/value pairs) + array, ///< array (ordered collection of values) + string, ///< string value + boolean, ///< boolean value + number_integer, ///< number value (signed integer) + number_unsigned, ///< number value (unsigned integer) + number_float, ///< number value (floating-point) + binary, ///< binary array (ordered collection of bytes) + discarded ///< discarded by the parser callback function +}; + +/*! +@brief comparison operator for JSON types + +Returns an ordering that is similar to Python: +- order: null < boolean < number < object < array < string < binary +- furthermore, each type is not smaller than itself +- discarded values are not comparable +- binary is represented as a b"" string in python and directly comparable to a + string; however, making a binary array directly comparable with a string would + be surprising behavior in a JSON file. + +@since version 1.0.0 +*/ +inline bool operator<(const value_t lhs, const value_t rhs) noexcept +{ + static constexpr std::array order = {{ + 0 /* null */, 3 /* object */, 4 /* array */, 5 /* string */, + 1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */, + 6 /* binary */ + } + }; + + const auto l_index = static_cast(lhs); + const auto r_index = static_cast(rhs); + return l_index < order.size() && r_index < order.size() && order[l_index] < order[r_index]; +} +} // namespace detail +} // namespace nlohmann + +// #include + + +#include +// #include + + +#include // declval, pair +// #include + + +/* Hedley - https://nemequ.github.io/hedley + * Created by Evan Nemerson + * + * To the extent possible under law, the author(s) have dedicated all + * copyright and related and neighboring rights to this software to + * the public domain worldwide. This software is distributed without + * any warranty. + * + * For details, see . + * SPDX-License-Identifier: CC0-1.0 + */ + +#if !defined(JSON_HEDLEY_VERSION) || (JSON_HEDLEY_VERSION < 15) +#if defined(JSON_HEDLEY_VERSION) + #undef JSON_HEDLEY_VERSION +#endif +#define JSON_HEDLEY_VERSION 15 + +#if defined(JSON_HEDLEY_STRINGIFY_EX) + #undef JSON_HEDLEY_STRINGIFY_EX +#endif +#define JSON_HEDLEY_STRINGIFY_EX(x) #x + +#if defined(JSON_HEDLEY_STRINGIFY) + #undef JSON_HEDLEY_STRINGIFY +#endif +#define JSON_HEDLEY_STRINGIFY(x) JSON_HEDLEY_STRINGIFY_EX(x) + +#if defined(JSON_HEDLEY_CONCAT_EX) + #undef JSON_HEDLEY_CONCAT_EX +#endif +#define JSON_HEDLEY_CONCAT_EX(a,b) a##b + +#if defined(JSON_HEDLEY_CONCAT) + #undef JSON_HEDLEY_CONCAT +#endif +#define JSON_HEDLEY_CONCAT(a,b) JSON_HEDLEY_CONCAT_EX(a,b) + +#if defined(JSON_HEDLEY_CONCAT3_EX) + #undef JSON_HEDLEY_CONCAT3_EX +#endif +#define JSON_HEDLEY_CONCAT3_EX(a,b,c) a##b##c + +#if defined(JSON_HEDLEY_CONCAT3) + #undef JSON_HEDLEY_CONCAT3 +#endif +#define JSON_HEDLEY_CONCAT3(a,b,c) JSON_HEDLEY_CONCAT3_EX(a,b,c) + +#if defined(JSON_HEDLEY_VERSION_ENCODE) + #undef JSON_HEDLEY_VERSION_ENCODE +#endif +#define JSON_HEDLEY_VERSION_ENCODE(major,minor,revision) (((major) * 1000000) + ((minor) * 1000) + (revision)) + +#if defined(JSON_HEDLEY_VERSION_DECODE_MAJOR) + #undef JSON_HEDLEY_VERSION_DECODE_MAJOR +#endif +#define JSON_HEDLEY_VERSION_DECODE_MAJOR(version) ((version) / 1000000) + +#if defined(JSON_HEDLEY_VERSION_DECODE_MINOR) + #undef JSON_HEDLEY_VERSION_DECODE_MINOR +#endif +#define JSON_HEDLEY_VERSION_DECODE_MINOR(version) (((version) % 1000000) / 1000) + +#if defined(JSON_HEDLEY_VERSION_DECODE_REVISION) + #undef JSON_HEDLEY_VERSION_DECODE_REVISION +#endif +#define JSON_HEDLEY_VERSION_DECODE_REVISION(version) ((version) % 1000) + +#if defined(JSON_HEDLEY_GNUC_VERSION) + #undef JSON_HEDLEY_GNUC_VERSION +#endif +#if defined(__GNUC__) && defined(__GNUC_PATCHLEVEL__) + #define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__) +#elif defined(__GNUC__) + #define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, 0) +#endif + +#if defined(JSON_HEDLEY_GNUC_VERSION_CHECK) + #undef JSON_HEDLEY_GNUC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_GNUC_VERSION) + #define JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_GNUC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_MSVC_VERSION) + #undef JSON_HEDLEY_MSVC_VERSION +#endif +#if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 140000000) && !defined(__ICL) + #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 10000000, (_MSC_FULL_VER % 10000000) / 100000, (_MSC_FULL_VER % 100000) / 100) +#elif defined(_MSC_FULL_VER) && !defined(__ICL) + #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 1000000, (_MSC_FULL_VER % 1000000) / 10000, (_MSC_FULL_VER % 10000) / 10) +#elif defined(_MSC_VER) && !defined(__ICL) + #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_VER / 100, _MSC_VER % 100, 0) +#endif + +#if defined(JSON_HEDLEY_MSVC_VERSION_CHECK) + #undef JSON_HEDLEY_MSVC_VERSION_CHECK +#endif +#if !defined(JSON_HEDLEY_MSVC_VERSION) + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (0) +#elif defined(_MSC_VER) && (_MSC_VER >= 1400) + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_FULL_VER >= ((major * 10000000) + (minor * 100000) + (patch))) +#elif defined(_MSC_VER) && (_MSC_VER >= 1200) + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_FULL_VER >= ((major * 1000000) + (minor * 10000) + (patch))) +#else + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_VER >= ((major * 100) + (minor))) +#endif + +#if defined(JSON_HEDLEY_INTEL_VERSION) + #undef JSON_HEDLEY_INTEL_VERSION +#endif +#if defined(__INTEL_COMPILER) && defined(__INTEL_COMPILER_UPDATE) && !defined(__ICL) + #define JSON_HEDLEY_INTEL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, __INTEL_COMPILER_UPDATE) +#elif defined(__INTEL_COMPILER) && !defined(__ICL) + #define JSON_HEDLEY_INTEL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, 0) +#endif + +#if defined(JSON_HEDLEY_INTEL_VERSION_CHECK) + #undef JSON_HEDLEY_INTEL_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_INTEL_VERSION) + #define JSON_HEDLEY_INTEL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_INTEL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_INTEL_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_INTEL_CL_VERSION) + #undef JSON_HEDLEY_INTEL_CL_VERSION +#endif +#if defined(__INTEL_COMPILER) && defined(__INTEL_COMPILER_UPDATE) && defined(__ICL) + #define JSON_HEDLEY_INTEL_CL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER, __INTEL_COMPILER_UPDATE, 0) +#endif + +#if defined(JSON_HEDLEY_INTEL_CL_VERSION_CHECK) + #undef JSON_HEDLEY_INTEL_CL_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_INTEL_CL_VERSION) + #define JSON_HEDLEY_INTEL_CL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_INTEL_CL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_INTEL_CL_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_PGI_VERSION) + #undef JSON_HEDLEY_PGI_VERSION +#endif +#if defined(__PGI) && defined(__PGIC__) && defined(__PGIC_MINOR__) && defined(__PGIC_PATCHLEVEL__) + #define JSON_HEDLEY_PGI_VERSION JSON_HEDLEY_VERSION_ENCODE(__PGIC__, __PGIC_MINOR__, __PGIC_PATCHLEVEL__) +#endif + +#if defined(JSON_HEDLEY_PGI_VERSION_CHECK) + #undef JSON_HEDLEY_PGI_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_PGI_VERSION) + #define JSON_HEDLEY_PGI_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_PGI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_PGI_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_SUNPRO_VERSION) + #undef JSON_HEDLEY_SUNPRO_VERSION +#endif +#if defined(__SUNPRO_C) && (__SUNPRO_C > 0x1000) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_C >> 16) & 0xf) * 10) + ((__SUNPRO_C >> 12) & 0xf), (((__SUNPRO_C >> 8) & 0xf) * 10) + ((__SUNPRO_C >> 4) & 0xf), (__SUNPRO_C & 0xf) * 10) +#elif defined(__SUNPRO_C) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_C >> 8) & 0xf, (__SUNPRO_C >> 4) & 0xf, (__SUNPRO_C) & 0xf) +#elif defined(__SUNPRO_CC) && (__SUNPRO_CC > 0x1000) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_CC >> 16) & 0xf) * 10) + ((__SUNPRO_CC >> 12) & 0xf), (((__SUNPRO_CC >> 8) & 0xf) * 10) + ((__SUNPRO_CC >> 4) & 0xf), (__SUNPRO_CC & 0xf) * 10) +#elif defined(__SUNPRO_CC) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_CC >> 8) & 0xf, (__SUNPRO_CC >> 4) & 0xf, (__SUNPRO_CC) & 0xf) +#endif + +#if defined(JSON_HEDLEY_SUNPRO_VERSION_CHECK) + #undef JSON_HEDLEY_SUNPRO_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_SUNPRO_VERSION) + #define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_SUNPRO_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION) + #undef JSON_HEDLEY_EMSCRIPTEN_VERSION +#endif +#if defined(__EMSCRIPTEN__) + #define JSON_HEDLEY_EMSCRIPTEN_VERSION JSON_HEDLEY_VERSION_ENCODE(__EMSCRIPTEN_major__, __EMSCRIPTEN_minor__, __EMSCRIPTEN_tiny__) +#endif + +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK) + #undef JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION) + #define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_EMSCRIPTEN_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_ARM_VERSION) + #undef JSON_HEDLEY_ARM_VERSION +#endif +#if defined(__CC_ARM) && defined(__ARMCOMPILER_VERSION) + #define JSON_HEDLEY_ARM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ARMCOMPILER_VERSION / 1000000, (__ARMCOMPILER_VERSION % 1000000) / 10000, (__ARMCOMPILER_VERSION % 10000) / 100) +#elif defined(__CC_ARM) && defined(__ARMCC_VERSION) + #define JSON_HEDLEY_ARM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ARMCC_VERSION / 1000000, (__ARMCC_VERSION % 1000000) / 10000, (__ARMCC_VERSION % 10000) / 100) +#endif + +#if defined(JSON_HEDLEY_ARM_VERSION_CHECK) + #undef JSON_HEDLEY_ARM_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_ARM_VERSION) + #define JSON_HEDLEY_ARM_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_ARM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_ARM_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_IBM_VERSION) + #undef JSON_HEDLEY_IBM_VERSION +#endif +#if defined(__ibmxl__) + #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ibmxl_version__, __ibmxl_release__, __ibmxl_modification__) +#elif defined(__xlC__) && defined(__xlC_ver__) + #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, (__xlC_ver__ >> 8) & 0xff) +#elif defined(__xlC__) + #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, 0) +#endif + +#if defined(JSON_HEDLEY_IBM_VERSION_CHECK) + #undef JSON_HEDLEY_IBM_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_IBM_VERSION) + #define JSON_HEDLEY_IBM_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_IBM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_IBM_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_VERSION) + #undef JSON_HEDLEY_TI_VERSION +#endif +#if \ + defined(__TI_COMPILER_VERSION__) && \ + ( \ + defined(__TMS470__) || defined(__TI_ARM__) || \ + defined(__MSP430__) || \ + defined(__TMS320C2000__) \ + ) +#if (__TI_COMPILER_VERSION__ >= 16000000) + #define JSON_HEDLEY_TI_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif +#endif + +#if defined(JSON_HEDLEY_TI_VERSION_CHECK) + #undef JSON_HEDLEY_TI_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_VERSION) + #define JSON_HEDLEY_TI_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL2000_VERSION) + #undef JSON_HEDLEY_TI_CL2000_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__TMS320C2000__) + #define JSON_HEDLEY_TI_CL2000_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL2000_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL2000_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL2000_VERSION) + #define JSON_HEDLEY_TI_CL2000_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL2000_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL2000_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL430_VERSION) + #undef JSON_HEDLEY_TI_CL430_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__MSP430__) + #define JSON_HEDLEY_TI_CL430_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL430_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL430_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL430_VERSION) + #define JSON_HEDLEY_TI_CL430_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL430_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL430_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_ARMCL_VERSION) + #undef JSON_HEDLEY_TI_ARMCL_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && (defined(__TMS470__) || defined(__TI_ARM__)) + #define JSON_HEDLEY_TI_ARMCL_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_ARMCL_VERSION_CHECK) + #undef JSON_HEDLEY_TI_ARMCL_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_ARMCL_VERSION) + #define JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_ARMCL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL6X_VERSION) + #undef JSON_HEDLEY_TI_CL6X_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__TMS320C6X__) + #define JSON_HEDLEY_TI_CL6X_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL6X_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL6X_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL6X_VERSION) + #define JSON_HEDLEY_TI_CL6X_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL6X_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL6X_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL7X_VERSION) + #undef JSON_HEDLEY_TI_CL7X_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__C7000__) + #define JSON_HEDLEY_TI_CL7X_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL7X_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL7X_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL7X_VERSION) + #define JSON_HEDLEY_TI_CL7X_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL7X_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL7X_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CLPRU_VERSION) + #undef JSON_HEDLEY_TI_CLPRU_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__PRU__) + #define JSON_HEDLEY_TI_CLPRU_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CLPRU_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CLPRU_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CLPRU_VERSION) + #define JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CLPRU_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_CRAY_VERSION) + #undef JSON_HEDLEY_CRAY_VERSION +#endif +#if defined(_CRAYC) + #if defined(_RELEASE_PATCHLEVEL) + #define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, _RELEASE_PATCHLEVEL) + #else + #define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, 0) + #endif +#endif + +#if defined(JSON_HEDLEY_CRAY_VERSION_CHECK) + #undef JSON_HEDLEY_CRAY_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_CRAY_VERSION) + #define JSON_HEDLEY_CRAY_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_CRAY_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_CRAY_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_IAR_VERSION) + #undef JSON_HEDLEY_IAR_VERSION +#endif +#if defined(__IAR_SYSTEMS_ICC__) + #if __VER__ > 1000 + #define JSON_HEDLEY_IAR_VERSION JSON_HEDLEY_VERSION_ENCODE((__VER__ / 1000000), ((__VER__ / 1000) % 1000), (__VER__ % 1000)) + #else + #define JSON_HEDLEY_IAR_VERSION JSON_HEDLEY_VERSION_ENCODE(__VER__ / 100, __VER__ % 100, 0) + #endif +#endif + +#if defined(JSON_HEDLEY_IAR_VERSION_CHECK) + #undef JSON_HEDLEY_IAR_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_IAR_VERSION) + #define JSON_HEDLEY_IAR_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_IAR_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_IAR_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TINYC_VERSION) + #undef JSON_HEDLEY_TINYC_VERSION +#endif +#if defined(__TINYC__) + #define JSON_HEDLEY_TINYC_VERSION JSON_HEDLEY_VERSION_ENCODE(__TINYC__ / 1000, (__TINYC__ / 100) % 10, __TINYC__ % 100) +#endif + +#if defined(JSON_HEDLEY_TINYC_VERSION_CHECK) + #undef JSON_HEDLEY_TINYC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TINYC_VERSION) + #define JSON_HEDLEY_TINYC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TINYC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TINYC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_DMC_VERSION) + #undef JSON_HEDLEY_DMC_VERSION +#endif +#if defined(__DMC__) + #define JSON_HEDLEY_DMC_VERSION JSON_HEDLEY_VERSION_ENCODE(__DMC__ >> 8, (__DMC__ >> 4) & 0xf, __DMC__ & 0xf) +#endif + +#if defined(JSON_HEDLEY_DMC_VERSION_CHECK) + #undef JSON_HEDLEY_DMC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_DMC_VERSION) + #define JSON_HEDLEY_DMC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_DMC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_DMC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_COMPCERT_VERSION) + #undef JSON_HEDLEY_COMPCERT_VERSION +#endif +#if defined(__COMPCERT_VERSION__) + #define JSON_HEDLEY_COMPCERT_VERSION JSON_HEDLEY_VERSION_ENCODE(__COMPCERT_VERSION__ / 10000, (__COMPCERT_VERSION__ / 100) % 100, __COMPCERT_VERSION__ % 100) +#endif + +#if defined(JSON_HEDLEY_COMPCERT_VERSION_CHECK) + #undef JSON_HEDLEY_COMPCERT_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_COMPCERT_VERSION) + #define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_COMPCERT_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_PELLES_VERSION) + #undef JSON_HEDLEY_PELLES_VERSION +#endif +#if defined(__POCC__) + #define JSON_HEDLEY_PELLES_VERSION JSON_HEDLEY_VERSION_ENCODE(__POCC__ / 100, __POCC__ % 100, 0) +#endif + +#if defined(JSON_HEDLEY_PELLES_VERSION_CHECK) + #undef JSON_HEDLEY_PELLES_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_PELLES_VERSION) + #define JSON_HEDLEY_PELLES_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_PELLES_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_PELLES_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_MCST_LCC_VERSION) + #undef JSON_HEDLEY_MCST_LCC_VERSION +#endif +#if defined(__LCC__) && defined(__LCC_MINOR__) + #define JSON_HEDLEY_MCST_LCC_VERSION JSON_HEDLEY_VERSION_ENCODE(__LCC__ / 100, __LCC__ % 100, __LCC_MINOR__) +#endif + +#if defined(JSON_HEDLEY_MCST_LCC_VERSION_CHECK) + #undef JSON_HEDLEY_MCST_LCC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_MCST_LCC_VERSION) + #define JSON_HEDLEY_MCST_LCC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_MCST_LCC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_MCST_LCC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_GCC_VERSION) + #undef JSON_HEDLEY_GCC_VERSION +#endif +#if \ + defined(JSON_HEDLEY_GNUC_VERSION) && \ + !defined(__clang__) && \ + !defined(JSON_HEDLEY_INTEL_VERSION) && \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_ARM_VERSION) && \ + !defined(JSON_HEDLEY_CRAY_VERSION) && \ + !defined(JSON_HEDLEY_TI_VERSION) && \ + !defined(JSON_HEDLEY_TI_ARMCL_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL430_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL2000_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL6X_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL7X_VERSION) && \ + !defined(JSON_HEDLEY_TI_CLPRU_VERSION) && \ + !defined(__COMPCERT__) && \ + !defined(JSON_HEDLEY_MCST_LCC_VERSION) + #define JSON_HEDLEY_GCC_VERSION JSON_HEDLEY_GNUC_VERSION +#endif + +#if defined(JSON_HEDLEY_GCC_VERSION_CHECK) + #undef JSON_HEDLEY_GCC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_GCC_VERSION) + #define JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_GCC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_HAS_ATTRIBUTE +#endif +#if \ + defined(__has_attribute) && \ + ( \ + (!defined(JSON_HEDLEY_IAR_VERSION) || JSON_HEDLEY_IAR_VERSION_CHECK(8,5,9)) \ + ) +# define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) __has_attribute(attribute) +#else +# define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_GNUC_HAS_ATTRIBUTE +#endif +#if defined(__has_attribute) + #define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_HAS_ATTRIBUTE(attribute) +#else + #define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_GCC_HAS_ATTRIBUTE +#endif +#if defined(__has_attribute) + #define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_HAS_ATTRIBUTE(attribute) +#else + #define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE +#endif +#if \ + defined(__has_cpp_attribute) && \ + defined(__cplusplus) && \ + (!defined(JSON_HEDLEY_SUNPRO_VERSION) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0)) + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) __has_cpp_attribute(attribute) +#else + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS) + #undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS +#endif +#if !defined(__cplusplus) || !defined(__has_cpp_attribute) + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) (0) +#elif \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_IAR_VERSION) && \ + (!defined(JSON_HEDLEY_SUNPRO_VERSION) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0)) && \ + (!defined(JSON_HEDLEY_MSVC_VERSION) || JSON_HEDLEY_MSVC_VERSION_CHECK(19,20,0)) + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) JSON_HEDLEY_HAS_CPP_ATTRIBUTE(ns::attribute) +#else + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE +#endif +#if defined(__has_cpp_attribute) && defined(__cplusplus) + #define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) __has_cpp_attribute(attribute) +#else + #define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE +#endif +#if defined(__has_cpp_attribute) && defined(__cplusplus) + #define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) __has_cpp_attribute(attribute) +#else + #define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_BUILTIN) + #undef JSON_HEDLEY_HAS_BUILTIN +#endif +#if defined(__has_builtin) + #define JSON_HEDLEY_HAS_BUILTIN(builtin) __has_builtin(builtin) +#else + #define JSON_HEDLEY_HAS_BUILTIN(builtin) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_BUILTIN) + #undef JSON_HEDLEY_GNUC_HAS_BUILTIN +#endif +#if defined(__has_builtin) + #define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin,major,minor,patch) __has_builtin(builtin) +#else + #define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_BUILTIN) + #undef JSON_HEDLEY_GCC_HAS_BUILTIN +#endif +#if defined(__has_builtin) + #define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin,major,minor,patch) __has_builtin(builtin) +#else + #define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_FEATURE) + #undef JSON_HEDLEY_HAS_FEATURE +#endif +#if defined(__has_feature) + #define JSON_HEDLEY_HAS_FEATURE(feature) __has_feature(feature) +#else + #define JSON_HEDLEY_HAS_FEATURE(feature) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_FEATURE) + #undef JSON_HEDLEY_GNUC_HAS_FEATURE +#endif +#if defined(__has_feature) + #define JSON_HEDLEY_GNUC_HAS_FEATURE(feature,major,minor,patch) __has_feature(feature) +#else + #define JSON_HEDLEY_GNUC_HAS_FEATURE(feature,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_FEATURE) + #undef JSON_HEDLEY_GCC_HAS_FEATURE +#endif +#if defined(__has_feature) + #define JSON_HEDLEY_GCC_HAS_FEATURE(feature,major,minor,patch) __has_feature(feature) +#else + #define JSON_HEDLEY_GCC_HAS_FEATURE(feature,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_EXTENSION) + #undef JSON_HEDLEY_HAS_EXTENSION +#endif +#if defined(__has_extension) + #define JSON_HEDLEY_HAS_EXTENSION(extension) __has_extension(extension) +#else + #define JSON_HEDLEY_HAS_EXTENSION(extension) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_EXTENSION) + #undef JSON_HEDLEY_GNUC_HAS_EXTENSION +#endif +#if defined(__has_extension) + #define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension,major,minor,patch) __has_extension(extension) +#else + #define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_EXTENSION) + #undef JSON_HEDLEY_GCC_HAS_EXTENSION +#endif +#if defined(__has_extension) + #define JSON_HEDLEY_GCC_HAS_EXTENSION(extension,major,minor,patch) __has_extension(extension) +#else + #define JSON_HEDLEY_GCC_HAS_EXTENSION(extension,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) + #define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) __has_declspec_attribute(attribute) +#else + #define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) + #define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) __has_declspec_attribute(attribute) +#else + #define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) + #define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) __has_declspec_attribute(attribute) +#else + #define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_WARNING) + #undef JSON_HEDLEY_HAS_WARNING +#endif +#if defined(__has_warning) + #define JSON_HEDLEY_HAS_WARNING(warning) __has_warning(warning) +#else + #define JSON_HEDLEY_HAS_WARNING(warning) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_WARNING) + #undef JSON_HEDLEY_GNUC_HAS_WARNING +#endif +#if defined(__has_warning) + #define JSON_HEDLEY_GNUC_HAS_WARNING(warning,major,minor,patch) __has_warning(warning) +#else + #define JSON_HEDLEY_GNUC_HAS_WARNING(warning,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_WARNING) + #undef JSON_HEDLEY_GCC_HAS_WARNING +#endif +#if defined(__has_warning) + #define JSON_HEDLEY_GCC_HAS_WARNING(warning,major,minor,patch) __has_warning(warning) +#else + #define JSON_HEDLEY_GCC_HAS_WARNING(warning,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \ + defined(__clang__) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(18,4,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,7,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(2,0,1) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(5,0,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,17) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(8,0,0) || \ + (JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) && defined(__C99_PRAGMA_OPERATOR)) + #define JSON_HEDLEY_PRAGMA(value) _Pragma(#value) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) + #define JSON_HEDLEY_PRAGMA(value) __pragma(value) +#else + #define JSON_HEDLEY_PRAGMA(value) +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_PUSH) + #undef JSON_HEDLEY_DIAGNOSTIC_PUSH +#endif +#if defined(JSON_HEDLEY_DIAGNOSTIC_POP) + #undef JSON_HEDLEY_DIAGNOSTIC_POP +#endif +#if defined(__clang__) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("clang diagnostic push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("clang diagnostic pop") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("GCC diagnostic push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("GCC diagnostic pop") +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH __pragma(warning(push)) + #define JSON_HEDLEY_DIAGNOSTIC_POP __pragma(warning(pop)) +#elif JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("pop") +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,4,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("diag_push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("diag_pop") +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,90,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)") +#else + #define JSON_HEDLEY_DIAGNOSTIC_PUSH + #define JSON_HEDLEY_DIAGNOSTIC_POP +#endif + +/* JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_ is for + HEDLEY INTERNAL USE ONLY. API subject to change without notice. */ +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_ +#endif +#if defined(__cplusplus) +# if JSON_HEDLEY_HAS_WARNING("-Wc++98-compat") +# if JSON_HEDLEY_HAS_WARNING("-Wc++17-extensions") +# if JSON_HEDLEY_HAS_WARNING("-Wc++1z-extensions") +# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ + _Pragma("clang diagnostic ignored \"-Wc++17-extensions\"") \ + _Pragma("clang diagnostic ignored \"-Wc++1z-extensions\"") \ + xpr \ + JSON_HEDLEY_DIAGNOSTIC_POP +# else +# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ + _Pragma("clang diagnostic ignored \"-Wc++17-extensions\"") \ + xpr \ + JSON_HEDLEY_DIAGNOSTIC_POP +# endif +# else +# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ + xpr \ + JSON_HEDLEY_DIAGNOSTIC_POP +# endif +# endif +#endif +#if !defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(x) x +#endif + +#if defined(JSON_HEDLEY_CONST_CAST) + #undef JSON_HEDLEY_CONST_CAST +#endif +#if defined(__cplusplus) +# define JSON_HEDLEY_CONST_CAST(T, expr) (const_cast(expr)) +#elif \ + JSON_HEDLEY_HAS_WARNING("-Wcast-qual") || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) +# define JSON_HEDLEY_CONST_CAST(T, expr) (__extension__ ({ \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL \ + ((T) (expr)); \ + JSON_HEDLEY_DIAGNOSTIC_POP \ + })) +#else +# define JSON_HEDLEY_CONST_CAST(T, expr) ((T) (expr)) +#endif + +#if defined(JSON_HEDLEY_REINTERPRET_CAST) + #undef JSON_HEDLEY_REINTERPRET_CAST +#endif +#if defined(__cplusplus) + #define JSON_HEDLEY_REINTERPRET_CAST(T, expr) (reinterpret_cast(expr)) +#else + #define JSON_HEDLEY_REINTERPRET_CAST(T, expr) ((T) (expr)) +#endif + +#if defined(JSON_HEDLEY_STATIC_CAST) + #undef JSON_HEDLEY_STATIC_CAST +#endif +#if defined(__cplusplus) + #define JSON_HEDLEY_STATIC_CAST(T, expr) (static_cast(expr)) +#else + #define JSON_HEDLEY_STATIC_CAST(T, expr) ((T) (expr)) +#endif + +#if defined(JSON_HEDLEY_CPP_CAST) + #undef JSON_HEDLEY_CPP_CAST +#endif +#if defined(__cplusplus) +# if JSON_HEDLEY_HAS_WARNING("-Wold-style-cast") +# define JSON_HEDLEY_CPP_CAST(T, expr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wold-style-cast\"") \ + ((T) (expr)) \ + JSON_HEDLEY_DIAGNOSTIC_POP +# elif JSON_HEDLEY_IAR_VERSION_CHECK(8,3,0) +# define JSON_HEDLEY_CPP_CAST(T, expr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("diag_suppress=Pe137") \ + JSON_HEDLEY_DIAGNOSTIC_POP +# else +# define JSON_HEDLEY_CPP_CAST(T, expr) ((T) (expr)) +# endif +#else +# define JSON_HEDLEY_CPP_CAST(T, expr) (expr) +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wdeprecated-declarations") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("clang diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warning(disable:1478 1786)") +#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED __pragma(warning(disable:1478 1786)) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(20,7,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1216,1444,1445") +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1444") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED __pragma(warning(disable:4996)) +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1444") +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1291,1718") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) && !defined(__cplusplus) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,E_DEPRECATED_ATT,E_DEPRECATED_ATT_MESS)") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) && defined(__cplusplus) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,symdeprecated,symdeprecated2)") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress=Pe1444,Pe1215") +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,90,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warn(disable:2241)") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("clang diagnostic ignored \"-Wunknown-pragmas\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("warning(disable:161)") +#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS __pragma(warning(disable:161)) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 1675") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("GCC diagnostic ignored \"-Wunknown-pragmas\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS __pragma(warning(disable:4068)) +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(16,9,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,3,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 163") +#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 163") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress=Pe161") +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 161") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-attributes") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("clang diagnostic ignored \"-Wunknown-attributes\"") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(17,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("warning(disable:1292)") +#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES __pragma(warning(disable:1292)) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(19,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES __pragma(warning(disable:5030)) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(20,7,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097,1098") +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,14,0) && defined(__cplusplus) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("error_messages(off,attrskipunsup)") +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(18,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,3,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1173") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress=Pe1097") +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wcast-qual") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("clang diagnostic ignored \"-Wcast-qual\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("warning(disable:2203 2331)") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("GCC diagnostic ignored \"-Wcast-qual\"") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunused-function") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("clang diagnostic ignored \"-Wunused-function\"") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("GCC diagnostic ignored \"-Wunused-function\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(1,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION __pragma(warning(disable:4505)) +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("diag_suppress 3142") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION +#endif + +#if defined(JSON_HEDLEY_DEPRECATED) + #undef JSON_HEDLEY_DEPRECATED +#endif +#if defined(JSON_HEDLEY_DEPRECATED_FOR) + #undef JSON_HEDLEY_DEPRECATED_FOR +#endif +#if \ + JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DEPRECATED(since) __declspec(deprecated("Since " # since)) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated("Since " #since "; use " #replacement)) +#elif \ + (JSON_HEDLEY_HAS_EXTENSION(attribute_deprecated_with_message) && !defined(JSON_HEDLEY_IAR_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,5,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(18,1,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(18,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,3,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,3,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__("Since " #since))) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__("Since " #since "; use " #replacement))) +#elif defined(__cplusplus) && (__cplusplus >= 201402L) + #define JSON_HEDLEY_DEPRECATED(since) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[deprecated("Since " #since)]]) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[deprecated("Since " #since "; use " #replacement)]]) +#elif \ + JSON_HEDLEY_HAS_ATTRIBUTE(deprecated) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) + #define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__)) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__)) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_PELLES_VERSION_CHECK(6,50,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DEPRECATED(since) __declspec(deprecated) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated) +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DEPRECATED(since) _Pragma("deprecated") + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) _Pragma("deprecated") +#else + #define JSON_HEDLEY_DEPRECATED(since) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) +#endif + +#if defined(JSON_HEDLEY_UNAVAILABLE) + #undef JSON_HEDLEY_UNAVAILABLE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(warning) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_UNAVAILABLE(available_since) __attribute__((__warning__("Not available until " #available_since))) +#else + #define JSON_HEDLEY_UNAVAILABLE(available_since) +#endif + +#if defined(JSON_HEDLEY_WARN_UNUSED_RESULT) + #undef JSON_HEDLEY_WARN_UNUSED_RESULT +#endif +#if defined(JSON_HEDLEY_WARN_UNUSED_RESULT_MSG) + #undef JSON_HEDLEY_WARN_UNUSED_RESULT_MSG +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(warn_unused_result) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0) && defined(__cplusplus)) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_WARN_UNUSED_RESULT __attribute__((__warn_unused_result__)) + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) __attribute__((__warn_unused_result__)) +#elif (JSON_HEDLEY_HAS_CPP_ATTRIBUTE(nodiscard) >= 201907L) + #define JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard(msg)]]) +#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE(nodiscard) + #define JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) +#elif defined(_Check_return_) /* SAL */ + #define JSON_HEDLEY_WARN_UNUSED_RESULT _Check_return_ + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) _Check_return_ +#else + #define JSON_HEDLEY_WARN_UNUSED_RESULT + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) +#endif + +#if defined(JSON_HEDLEY_SENTINEL) + #undef JSON_HEDLEY_SENTINEL +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(sentinel) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,4,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_SENTINEL(position) __attribute__((__sentinel__(position))) +#else + #define JSON_HEDLEY_SENTINEL(position) +#endif + +#if defined(JSON_HEDLEY_NO_RETURN) + #undef JSON_HEDLEY_NO_RETURN +#endif +#if JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_NO_RETURN __noreturn +#elif \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__)) +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L + #define JSON_HEDLEY_NO_RETURN _Noreturn +#elif defined(__cplusplus) && (__cplusplus >= 201103L) + #define JSON_HEDLEY_NO_RETURN JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[noreturn]]) +#elif \ + JSON_HEDLEY_HAS_ATTRIBUTE(noreturn) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,2,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) + #define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__)) +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) + #define JSON_HEDLEY_NO_RETURN _Pragma("does_not_return") +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_NO_RETURN __declspec(noreturn) +#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,0,0) && defined(__cplusplus) + #define JSON_HEDLEY_NO_RETURN _Pragma("FUNC_NEVER_RETURNS;") +#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3,2,0) + #define JSON_HEDLEY_NO_RETURN __attribute((noreturn)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9,0,0) + #define JSON_HEDLEY_NO_RETURN __declspec(noreturn) +#else + #define JSON_HEDLEY_NO_RETURN +#endif + +#if defined(JSON_HEDLEY_NO_ESCAPE) + #undef JSON_HEDLEY_NO_ESCAPE +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(noescape) + #define JSON_HEDLEY_NO_ESCAPE __attribute__((__noescape__)) +#else + #define JSON_HEDLEY_NO_ESCAPE +#endif + +#if defined(JSON_HEDLEY_UNREACHABLE) + #undef JSON_HEDLEY_UNREACHABLE +#endif +#if defined(JSON_HEDLEY_UNREACHABLE_RETURN) + #undef JSON_HEDLEY_UNREACHABLE_RETURN +#endif +#if defined(JSON_HEDLEY_ASSUME) + #undef JSON_HEDLEY_ASSUME +#endif +#if \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_ASSUME(expr) __assume(expr) +#elif JSON_HEDLEY_HAS_BUILTIN(__builtin_assume) + #define JSON_HEDLEY_ASSUME(expr) __builtin_assume(expr) +#elif \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) + #if defined(__cplusplus) + #define JSON_HEDLEY_ASSUME(expr) std::_nassert(expr) + #else + #define JSON_HEDLEY_ASSUME(expr) _nassert(expr) + #endif +#endif +#if \ + (JSON_HEDLEY_HAS_BUILTIN(__builtin_unreachable) && (!defined(JSON_HEDLEY_ARM_VERSION))) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,5,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(18,10,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,5) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(10,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_UNREACHABLE() __builtin_unreachable() +#elif defined(JSON_HEDLEY_ASSUME) + #define JSON_HEDLEY_UNREACHABLE() JSON_HEDLEY_ASSUME(0) +#endif +#if !defined(JSON_HEDLEY_ASSUME) + #if defined(JSON_HEDLEY_UNREACHABLE) + #define JSON_HEDLEY_ASSUME(expr) JSON_HEDLEY_STATIC_CAST(void, ((expr) ? 1 : (JSON_HEDLEY_UNREACHABLE(), 1))) + #else + #define JSON_HEDLEY_ASSUME(expr) JSON_HEDLEY_STATIC_CAST(void, expr) + #endif +#endif +#if defined(JSON_HEDLEY_UNREACHABLE) + #if \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) + #define JSON_HEDLEY_UNREACHABLE_RETURN(value) return (JSON_HEDLEY_STATIC_CAST(void, JSON_HEDLEY_ASSUME(0)), (value)) + #else + #define JSON_HEDLEY_UNREACHABLE_RETURN(value) JSON_HEDLEY_UNREACHABLE() + #endif +#else + #define JSON_HEDLEY_UNREACHABLE_RETURN(value) return (value) +#endif +#if !defined(JSON_HEDLEY_UNREACHABLE) + #define JSON_HEDLEY_UNREACHABLE() JSON_HEDLEY_ASSUME(0) +#endif + +JSON_HEDLEY_DIAGNOSTIC_PUSH +#if JSON_HEDLEY_HAS_WARNING("-Wpedantic") + #pragma clang diagnostic ignored "-Wpedantic" +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wc++98-compat-pedantic") && defined(__cplusplus) + #pragma clang diagnostic ignored "-Wc++98-compat-pedantic" +#endif +#if JSON_HEDLEY_GCC_HAS_WARNING("-Wvariadic-macros",4,0,0) + #if defined(__clang__) + #pragma clang diagnostic ignored "-Wvariadic-macros" + #elif defined(JSON_HEDLEY_GCC_VERSION) + #pragma GCC diagnostic ignored "-Wvariadic-macros" + #endif +#endif +#if defined(JSON_HEDLEY_NON_NULL) + #undef JSON_HEDLEY_NON_NULL +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(nonnull) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) + #define JSON_HEDLEY_NON_NULL(...) __attribute__((__nonnull__(__VA_ARGS__))) +#else + #define JSON_HEDLEY_NON_NULL(...) +#endif +JSON_HEDLEY_DIAGNOSTIC_POP + +#if defined(JSON_HEDLEY_PRINTF_FORMAT) + #undef JSON_HEDLEY_PRINTF_FORMAT +#endif +#if defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format,4,4,0) && !defined(__USE_MINGW_ANSI_STDIO) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(ms_printf, string_idx, first_to_check))) +#elif defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format,4,4,0) && defined(__USE_MINGW_ANSI_STDIO) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(gnu_printf, string_idx, first_to_check))) +#elif \ + JSON_HEDLEY_HAS_ATTRIBUTE(format) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(__printf__, string_idx, first_to_check))) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(6,0,0) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __declspec(vaformat(printf,string_idx,first_to_check)) +#else + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) +#endif + +#if defined(JSON_HEDLEY_CONSTEXPR) + #undef JSON_HEDLEY_CONSTEXPR +#endif +#if defined(__cplusplus) + #if __cplusplus >= 201103L + #define JSON_HEDLEY_CONSTEXPR JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(constexpr) + #endif +#endif +#if !defined(JSON_HEDLEY_CONSTEXPR) + #define JSON_HEDLEY_CONSTEXPR +#endif + +#if defined(JSON_HEDLEY_PREDICT) + #undef JSON_HEDLEY_PREDICT +#endif +#if defined(JSON_HEDLEY_LIKELY) + #undef JSON_HEDLEY_LIKELY +#endif +#if defined(JSON_HEDLEY_UNLIKELY) + #undef JSON_HEDLEY_UNLIKELY +#endif +#if defined(JSON_HEDLEY_UNPREDICTABLE) + #undef JSON_HEDLEY_UNPREDICTABLE +#endif +#if JSON_HEDLEY_HAS_BUILTIN(__builtin_unpredictable) + #define JSON_HEDLEY_UNPREDICTABLE(expr) __builtin_unpredictable((expr)) +#endif +#if \ + (JSON_HEDLEY_HAS_BUILTIN(__builtin_expect_with_probability) && !defined(JSON_HEDLEY_PGI_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(9,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PREDICT(expr, value, probability) __builtin_expect_with_probability( (expr), (value), (probability)) +# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) __builtin_expect_with_probability(!!(expr), 1 , (probability)) +# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) __builtin_expect_with_probability(!!(expr), 0 , (probability)) +# define JSON_HEDLEY_LIKELY(expr) __builtin_expect (!!(expr), 1 ) +# define JSON_HEDLEY_UNLIKELY(expr) __builtin_expect (!!(expr), 0 ) +#elif \ + (JSON_HEDLEY_HAS_BUILTIN(__builtin_expect) && !defined(JSON_HEDLEY_INTEL_CL_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0) && defined(__cplusplus)) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,7,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,27) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PREDICT(expr, expected, probability) \ + (((probability) >= 0.9) ? __builtin_expect((expr), (expected)) : (JSON_HEDLEY_STATIC_CAST(void, expected), (expr))) +# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) \ + (__extension__ ({ \ + double hedley_probability_ = (probability); \ + ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 1) : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 0) : !!(expr))); \ + })) +# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) \ + (__extension__ ({ \ + double hedley_probability_ = (probability); \ + ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 0) : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 1) : !!(expr))); \ + })) +# define JSON_HEDLEY_LIKELY(expr) __builtin_expect(!!(expr), 1) +# define JSON_HEDLEY_UNLIKELY(expr) __builtin_expect(!!(expr), 0) +#else +# define JSON_HEDLEY_PREDICT(expr, expected, probability) (JSON_HEDLEY_STATIC_CAST(void, expected), (expr)) +# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) (!!(expr)) +# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) (!!(expr)) +# define JSON_HEDLEY_LIKELY(expr) (!!(expr)) +# define JSON_HEDLEY_UNLIKELY(expr) (!!(expr)) +#endif +#if !defined(JSON_HEDLEY_UNPREDICTABLE) + #define JSON_HEDLEY_UNPREDICTABLE(expr) JSON_HEDLEY_PREDICT(expr, 1, 0.5) +#endif + +#if defined(JSON_HEDLEY_MALLOC) + #undef JSON_HEDLEY_MALLOC +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(malloc) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(12,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_MALLOC __attribute__((__malloc__)) +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) + #define JSON_HEDLEY_MALLOC _Pragma("returns_new_memory") +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_MALLOC __declspec(restrict) +#else + #define JSON_HEDLEY_MALLOC +#endif + +#if defined(JSON_HEDLEY_PURE) + #undef JSON_HEDLEY_PURE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(pure) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(2,96,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PURE __attribute__((__pure__)) +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) +# define JSON_HEDLEY_PURE _Pragma("does_not_write_global_data") +#elif defined(__cplusplus) && \ + ( \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(2,0,1) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) \ + ) +# define JSON_HEDLEY_PURE _Pragma("FUNC_IS_PURE;") +#else +# define JSON_HEDLEY_PURE +#endif + +#if defined(JSON_HEDLEY_CONST) + #undef JSON_HEDLEY_CONST +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(const) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(2,5,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_CONST __attribute__((__const__)) +#elif \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) + #define JSON_HEDLEY_CONST _Pragma("no_side_effect") +#else + #define JSON_HEDLEY_CONST JSON_HEDLEY_PURE +#endif + +#if defined(JSON_HEDLEY_RESTRICT) + #undef JSON_HEDLEY_RESTRICT +#endif +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && !defined(__cplusplus) + #define JSON_HEDLEY_RESTRICT restrict +#elif \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,4) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,14,0) && defined(__cplusplus)) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) || \ + defined(__clang__) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_RESTRICT __restrict +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,3,0) && !defined(__cplusplus) + #define JSON_HEDLEY_RESTRICT _Restrict +#else + #define JSON_HEDLEY_RESTRICT +#endif + +#if defined(JSON_HEDLEY_INLINE) + #undef JSON_HEDLEY_INLINE +#endif +#if \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \ + (defined(__cplusplus) && (__cplusplus >= 199711L)) + #define JSON_HEDLEY_INLINE inline +#elif \ + defined(JSON_HEDLEY_GCC_VERSION) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(6,2,0) + #define JSON_HEDLEY_INLINE __inline__ +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(12,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,1,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_INLINE __inline +#else + #define JSON_HEDLEY_INLINE +#endif + +#if defined(JSON_HEDLEY_ALWAYS_INLINE) + #undef JSON_HEDLEY_ALWAYS_INLINE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(always_inline) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) +# define JSON_HEDLEY_ALWAYS_INLINE __attribute__((__always_inline__)) JSON_HEDLEY_INLINE +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(12,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) +# define JSON_HEDLEY_ALWAYS_INLINE __forceinline +#elif defined(__cplusplus) && \ + ( \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) \ + ) +# define JSON_HEDLEY_ALWAYS_INLINE _Pragma("FUNC_ALWAYS_INLINE;") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) +# define JSON_HEDLEY_ALWAYS_INLINE _Pragma("inline=forced") +#else +# define JSON_HEDLEY_ALWAYS_INLINE JSON_HEDLEY_INLINE +#endif + +#if defined(JSON_HEDLEY_NEVER_INLINE) + #undef JSON_HEDLEY_NEVER_INLINE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(noinline) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) + #define JSON_HEDLEY_NEVER_INLINE __attribute__((__noinline__)) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_NEVER_INLINE __declspec(noinline) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(10,2,0) + #define JSON_HEDLEY_NEVER_INLINE _Pragma("noinline") +#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,0,0) && defined(__cplusplus) + #define JSON_HEDLEY_NEVER_INLINE _Pragma("FUNC_CANNOT_INLINE;") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_NEVER_INLINE _Pragma("inline=never") +#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3,2,0) + #define JSON_HEDLEY_NEVER_INLINE __attribute((noinline)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9,0,0) + #define JSON_HEDLEY_NEVER_INLINE __declspec(noinline) +#else + #define JSON_HEDLEY_NEVER_INLINE +#endif + +#if defined(JSON_HEDLEY_PRIVATE) + #undef JSON_HEDLEY_PRIVATE +#endif +#if defined(JSON_HEDLEY_PUBLIC) + #undef JSON_HEDLEY_PUBLIC +#endif +#if defined(JSON_HEDLEY_IMPORT) + #undef JSON_HEDLEY_IMPORT +#endif +#if defined(_WIN32) || defined(__CYGWIN__) +# define JSON_HEDLEY_PRIVATE +# define JSON_HEDLEY_PUBLIC __declspec(dllexport) +# define JSON_HEDLEY_IMPORT __declspec(dllimport) +#else +# if \ + JSON_HEDLEY_HAS_ATTRIBUTE(visibility) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ + ( \ + defined(__TI_EABI__) && \ + ( \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) \ + ) \ + ) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PRIVATE __attribute__((__visibility__("hidden"))) +# define JSON_HEDLEY_PUBLIC __attribute__((__visibility__("default"))) +# else +# define JSON_HEDLEY_PRIVATE +# define JSON_HEDLEY_PUBLIC +# endif +# define JSON_HEDLEY_IMPORT extern +#endif + +#if defined(JSON_HEDLEY_NO_THROW) + #undef JSON_HEDLEY_NO_THROW +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(nothrow) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_NO_THROW __attribute__((__nothrow__)) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,1,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) + #define JSON_HEDLEY_NO_THROW __declspec(nothrow) +#else + #define JSON_HEDLEY_NO_THROW +#endif + +#if defined(JSON_HEDLEY_FALL_THROUGH) + #undef JSON_HEDLEY_FALL_THROUGH +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(fallthrough) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(7,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_FALL_THROUGH __attribute__((__fallthrough__)) +#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(clang,fallthrough) + #define JSON_HEDLEY_FALL_THROUGH JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[clang::fallthrough]]) +#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE(fallthrough) + #define JSON_HEDLEY_FALL_THROUGH JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[fallthrough]]) +#elif defined(__fallthrough) /* SAL */ + #define JSON_HEDLEY_FALL_THROUGH __fallthrough +#else + #define JSON_HEDLEY_FALL_THROUGH +#endif + +#if defined(JSON_HEDLEY_RETURNS_NON_NULL) + #undef JSON_HEDLEY_RETURNS_NON_NULL +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(returns_nonnull) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_RETURNS_NON_NULL __attribute__((__returns_nonnull__)) +#elif defined(_Ret_notnull_) /* SAL */ + #define JSON_HEDLEY_RETURNS_NON_NULL _Ret_notnull_ +#else + #define JSON_HEDLEY_RETURNS_NON_NULL +#endif + +#if defined(JSON_HEDLEY_ARRAY_PARAM) + #undef JSON_HEDLEY_ARRAY_PARAM +#endif +#if \ + defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \ + !defined(__STDC_NO_VLA__) && \ + !defined(__cplusplus) && \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_TINYC_VERSION) + #define JSON_HEDLEY_ARRAY_PARAM(name) (name) +#else + #define JSON_HEDLEY_ARRAY_PARAM(name) +#endif + +#if defined(JSON_HEDLEY_IS_CONSTANT) + #undef JSON_HEDLEY_IS_CONSTANT +#endif +#if defined(JSON_HEDLEY_REQUIRE_CONSTEXPR) + #undef JSON_HEDLEY_REQUIRE_CONSTEXPR +#endif +/* JSON_HEDLEY_IS_CONSTEXPR_ is for + HEDLEY INTERNAL USE ONLY. API subject to change without notice. */ +#if defined(JSON_HEDLEY_IS_CONSTEXPR_) + #undef JSON_HEDLEY_IS_CONSTEXPR_ +#endif +#if \ + JSON_HEDLEY_HAS_BUILTIN(__builtin_constant_p) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,19) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) && !defined(__cplusplus)) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_IS_CONSTANT(expr) __builtin_constant_p(expr) +#endif +#if !defined(__cplusplus) +# if \ + JSON_HEDLEY_HAS_BUILTIN(__builtin_types_compatible_p) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,4,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,24) +#if defined(__INTPTR_TYPE__) + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) __builtin_types_compatible_p(__typeof__((1 ? (void*) ((__INTPTR_TYPE__) ((expr) * 0)) : (int*) 0)), int*) +#else + #include + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) __builtin_types_compatible_p(__typeof__((1 ? (void*) ((intptr_t) ((expr) * 0)) : (int*) 0)), int*) +#endif +# elif \ + ( \ + defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) && \ + !defined(JSON_HEDLEY_SUNPRO_VERSION) && \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_IAR_VERSION)) || \ + (JSON_HEDLEY_HAS_EXTENSION(c_generic_selections) && !defined(JSON_HEDLEY_IAR_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(17,0,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(12,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,3,0) +#if defined(__INTPTR_TYPE__) + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) _Generic((1 ? (void*) ((__INTPTR_TYPE__) ((expr) * 0)) : (int*) 0), int*: 1, void*: 0) +#else + #include + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) _Generic((1 ? (void*) ((intptr_t) * 0) : (int*) 0), int*: 1, void*: 0) +#endif +# elif \ + defined(JSON_HEDLEY_GCC_VERSION) || \ + defined(JSON_HEDLEY_INTEL_VERSION) || \ + defined(JSON_HEDLEY_TINYC_VERSION) || \ + defined(JSON_HEDLEY_TI_ARMCL_VERSION) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(18,12,0) || \ + defined(JSON_HEDLEY_TI_CL2000_VERSION) || \ + defined(JSON_HEDLEY_TI_CL6X_VERSION) || \ + defined(JSON_HEDLEY_TI_CL7X_VERSION) || \ + defined(JSON_HEDLEY_TI_CLPRU_VERSION) || \ + defined(__clang__) +# define JSON_HEDLEY_IS_CONSTEXPR_(expr) ( \ + sizeof(void) != \ + sizeof(*( \ + 1 ? \ + ((void*) ((expr) * 0L) ) : \ +((struct { char v[sizeof(void) * 2]; } *) 1) \ + ) \ + ) \ + ) +# endif +#endif +#if defined(JSON_HEDLEY_IS_CONSTEXPR_) + #if !defined(JSON_HEDLEY_IS_CONSTANT) + #define JSON_HEDLEY_IS_CONSTANT(expr) JSON_HEDLEY_IS_CONSTEXPR_(expr) + #endif + #define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (JSON_HEDLEY_IS_CONSTEXPR_(expr) ? (expr) : (-1)) +#else + #if !defined(JSON_HEDLEY_IS_CONSTANT) + #define JSON_HEDLEY_IS_CONSTANT(expr) (0) + #endif + #define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (expr) +#endif + +#if defined(JSON_HEDLEY_BEGIN_C_DECLS) + #undef JSON_HEDLEY_BEGIN_C_DECLS +#endif +#if defined(JSON_HEDLEY_END_C_DECLS) + #undef JSON_HEDLEY_END_C_DECLS +#endif +#if defined(JSON_HEDLEY_C_DECL) + #undef JSON_HEDLEY_C_DECL +#endif +#if defined(__cplusplus) + #define JSON_HEDLEY_BEGIN_C_DECLS extern "C" { + #define JSON_HEDLEY_END_C_DECLS } + #define JSON_HEDLEY_C_DECL extern "C" +#else + #define JSON_HEDLEY_BEGIN_C_DECLS + #define JSON_HEDLEY_END_C_DECLS + #define JSON_HEDLEY_C_DECL +#endif + +#if defined(JSON_HEDLEY_STATIC_ASSERT) + #undef JSON_HEDLEY_STATIC_ASSERT +#endif +#if \ + !defined(__cplusplus) && ( \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) || \ + (JSON_HEDLEY_HAS_FEATURE(c_static_assert) && !defined(JSON_HEDLEY_INTEL_CL_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(6,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + defined(_Static_assert) \ + ) +# define JSON_HEDLEY_STATIC_ASSERT(expr, message) _Static_assert(expr, message) +#elif \ + (defined(__cplusplus) && (__cplusplus >= 201103L)) || \ + JSON_HEDLEY_MSVC_VERSION_CHECK(16,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) +# define JSON_HEDLEY_STATIC_ASSERT(expr, message) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(static_assert(expr, message)) +#else +# define JSON_HEDLEY_STATIC_ASSERT(expr, message) +#endif + +#if defined(JSON_HEDLEY_NULL) + #undef JSON_HEDLEY_NULL +#endif +#if defined(__cplusplus) + #if __cplusplus >= 201103L + #define JSON_HEDLEY_NULL JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(nullptr) + #elif defined(NULL) + #define JSON_HEDLEY_NULL NULL + #else + #define JSON_HEDLEY_NULL JSON_HEDLEY_STATIC_CAST(void*, 0) + #endif +#elif defined(NULL) + #define JSON_HEDLEY_NULL NULL +#else + #define JSON_HEDLEY_NULL ((void*) 0) +#endif + +#if defined(JSON_HEDLEY_MESSAGE) + #undef JSON_HEDLEY_MESSAGE +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") +# define JSON_HEDLEY_MESSAGE(msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \ + JSON_HEDLEY_PRAGMA(message msg) \ + JSON_HEDLEY_DIAGNOSTIC_POP +#elif \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message msg) +#elif JSON_HEDLEY_CRAY_VERSION_CHECK(5,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(_CRI message msg) +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#else +# define JSON_HEDLEY_MESSAGE(msg) +#endif + +#if defined(JSON_HEDLEY_WARNING) + #undef JSON_HEDLEY_WARNING +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") +# define JSON_HEDLEY_WARNING(msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \ + JSON_HEDLEY_PRAGMA(clang warning msg) \ + JSON_HEDLEY_DIAGNOSTIC_POP +#elif \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,8,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(18,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) +# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(GCC warning msg) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) +# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#else +# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_MESSAGE(msg) +#endif + +#if defined(JSON_HEDLEY_REQUIRE) + #undef JSON_HEDLEY_REQUIRE +#endif +#if defined(JSON_HEDLEY_REQUIRE_MSG) + #undef JSON_HEDLEY_REQUIRE_MSG +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(diagnose_if) +# if JSON_HEDLEY_HAS_WARNING("-Wgcc-compat") +# define JSON_HEDLEY_REQUIRE(expr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wgcc-compat\"") \ + __attribute__((diagnose_if(!(expr), #expr, "error"))) \ + JSON_HEDLEY_DIAGNOSTIC_POP +# define JSON_HEDLEY_REQUIRE_MSG(expr,msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wgcc-compat\"") \ + __attribute__((diagnose_if(!(expr), msg, "error"))) \ + JSON_HEDLEY_DIAGNOSTIC_POP +# else +# define JSON_HEDLEY_REQUIRE(expr) __attribute__((diagnose_if(!(expr), #expr, "error"))) +# define JSON_HEDLEY_REQUIRE_MSG(expr,msg) __attribute__((diagnose_if(!(expr), msg, "error"))) +# endif +#else +# define JSON_HEDLEY_REQUIRE(expr) +# define JSON_HEDLEY_REQUIRE_MSG(expr,msg) +#endif + +#if defined(JSON_HEDLEY_FLAGS) + #undef JSON_HEDLEY_FLAGS +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(flag_enum) && (!defined(__cplusplus) || JSON_HEDLEY_HAS_WARNING("-Wbitfield-enum-conversion")) + #define JSON_HEDLEY_FLAGS __attribute__((__flag_enum__)) +#else + #define JSON_HEDLEY_FLAGS +#endif + +#if defined(JSON_HEDLEY_FLAGS_CAST) + #undef JSON_HEDLEY_FLAGS_CAST +#endif +#if JSON_HEDLEY_INTEL_VERSION_CHECK(19,0,0) +# define JSON_HEDLEY_FLAGS_CAST(T, expr) (__extension__ ({ \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("warning(disable:188)") \ + ((T) (expr)); \ + JSON_HEDLEY_DIAGNOSTIC_POP \ + })) +#else +# define JSON_HEDLEY_FLAGS_CAST(T, expr) JSON_HEDLEY_STATIC_CAST(T, expr) +#endif + +#if defined(JSON_HEDLEY_EMPTY_BASES) + #undef JSON_HEDLEY_EMPTY_BASES +#endif +#if \ + (JSON_HEDLEY_MSVC_VERSION_CHECK(19,0,23918) && !JSON_HEDLEY_MSVC_VERSION_CHECK(20,0,0)) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_EMPTY_BASES __declspec(empty_bases) +#else + #define JSON_HEDLEY_EMPTY_BASES +#endif + +/* Remaining macros are deprecated. */ + +#if defined(JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK) + #undef JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK +#endif +#if defined(__clang__) + #define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major,minor,patch) (0) +#else + #define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_CLANG_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_CLANG_HAS_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_BUILTIN) + #undef JSON_HEDLEY_CLANG_HAS_BUILTIN +#endif +#define JSON_HEDLEY_CLANG_HAS_BUILTIN(builtin) JSON_HEDLEY_HAS_BUILTIN(builtin) + +#if defined(JSON_HEDLEY_CLANG_HAS_FEATURE) + #undef JSON_HEDLEY_CLANG_HAS_FEATURE +#endif +#define JSON_HEDLEY_CLANG_HAS_FEATURE(feature) JSON_HEDLEY_HAS_FEATURE(feature) + +#if defined(JSON_HEDLEY_CLANG_HAS_EXTENSION) + #undef JSON_HEDLEY_CLANG_HAS_EXTENSION +#endif +#define JSON_HEDLEY_CLANG_HAS_EXTENSION(extension) JSON_HEDLEY_HAS_EXTENSION(extension) + +#if defined(JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_DECLSPEC_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_WARNING) + #undef JSON_HEDLEY_CLANG_HAS_WARNING +#endif +#define JSON_HEDLEY_CLANG_HAS_WARNING(warning) JSON_HEDLEY_HAS_WARNING(warning) + +#endif /* !defined(JSON_HEDLEY_VERSION) || (JSON_HEDLEY_VERSION < X) */ + +// #include + + +#include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template struct make_void +{ + using type = void; +}; +template using void_t = typename make_void::type; +} // namespace detail +} // namespace nlohmann + + +// https://en.cppreference.com/w/cpp/experimental/is_detected +namespace nlohmann +{ +namespace detail +{ +struct nonesuch +{ + nonesuch() = delete; + ~nonesuch() = delete; + nonesuch(nonesuch const&) = delete; + nonesuch(nonesuch const&&) = delete; + void operator=(nonesuch const&) = delete; + void operator=(nonesuch&&) = delete; +}; + +template class Op, + class... Args> +struct detector +{ + using value_t = std::false_type; + using type = Default; +}; + +template class Op, class... Args> +struct detector>, Op, Args...> +{ + using value_t = std::true_type; + using type = Op; +}; + +template class Op, class... Args> +using is_detected = typename detector::value_t; + +template class Op, class... Args> +struct is_detected_lazy : is_detected { }; + +template class Op, class... Args> +using detected_t = typename detector::type; + +template class Op, class... Args> +using detected_or = detector; + +template class Op, class... Args> +using detected_or_t = typename detected_or::type; + +template class Op, class... Args> +using is_detected_exact = std::is_same>; + +template class Op, class... Args> +using is_detected_convertible = + std::is_convertible, To>; +} // namespace detail +} // namespace nlohmann + + +// This file contains all internal macro definitions +// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them + +// exclude unsupported compilers +#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) + #if defined(__clang__) + #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 + #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) + #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 + #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #endif +#endif + +// C++ language standard detection +// if the user manually specified the used c++ version this is skipped +#if !defined(JSON_HAS_CPP_20) && !defined(JSON_HAS_CPP_17) && !defined(JSON_HAS_CPP_14) && !defined(JSON_HAS_CPP_11) + #if (defined(__cplusplus) && __cplusplus >= 202002L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L) + #define JSON_HAS_CPP_20 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 + #elif (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 + #elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1) + #define JSON_HAS_CPP_14 + #endif + // the cpp 11 flag is always specified because it is the minimal required version + #define JSON_HAS_CPP_11 +#endif + +#if !defined(JSON_HAS_FILESYSTEM) && !defined(JSON_HAS_EXPERIMENTAL_FILESYSTEM) + #ifdef JSON_HAS_CPP_17 + #if defined(__cpp_lib_filesystem) + #define JSON_HAS_FILESYSTEM 1 + #elif defined(__cpp_lib_experimental_filesystem) + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1 + #elif !defined(__has_include) + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1 + #elif __has_include() + #define JSON_HAS_FILESYSTEM 1 + #elif __has_include() + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1 + #endif + + // std::filesystem does not work on MinGW GCC 8: https://sourceforge.net/p/mingw-w64/bugs/737/ + #if defined(__MINGW32__) && defined(__GNUC__) && __GNUC__ == 8 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before GCC 8: https://en.cppreference.com/w/cpp/compiler_support + #if defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 8 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before Clang 7: https://en.cppreference.com/w/cpp/compiler_support + #if defined(__clang_major__) && __clang_major__ < 7 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before MSVC 19.14: https://en.cppreference.com/w/cpp/compiler_support + #if defined(_MSC_VER) && _MSC_VER < 1914 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before iOS 13 + #if defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && __IPHONE_OS_VERSION_MIN_REQUIRED < 130000 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before macOS Catalina + #if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED < 101500 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + #endif +#endif + +#ifndef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 0 +#endif + +#ifndef JSON_HAS_FILESYSTEM + #define JSON_HAS_FILESYSTEM 0 +#endif + +// disable documentation warnings on clang +#if defined(__clang__) + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wdocumentation" + #pragma clang diagnostic ignored "-Wdocumentation-unknown-command" +#endif + +// allow disabling exceptions +#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) + #define JSON_THROW(exception) throw exception + #define JSON_TRY try + #define JSON_CATCH(exception) catch(exception) + #define JSON_INTERNAL_CATCH(exception) catch(exception) +#else + #include + #define JSON_THROW(exception) std::abort() + #define JSON_TRY if(true) + #define JSON_CATCH(exception) if(false) + #define JSON_INTERNAL_CATCH(exception) if(false) +#endif + +// override exception macros +#if defined(JSON_THROW_USER) + #undef JSON_THROW + #define JSON_THROW JSON_THROW_USER +#endif +#if defined(JSON_TRY_USER) + #undef JSON_TRY + #define JSON_TRY JSON_TRY_USER +#endif +#if defined(JSON_CATCH_USER) + #undef JSON_CATCH + #define JSON_CATCH JSON_CATCH_USER + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_CATCH_USER +#endif +#if defined(JSON_INTERNAL_CATCH_USER) + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER +#endif + +// allow overriding assert +#if !defined(JSON_ASSERT) + #include // assert + #define JSON_ASSERT(x) assert(x) +#endif + +// allow to access some private functions (needed by the test suite) +#if defined(JSON_TESTS_PRIVATE) + #define JSON_PRIVATE_UNLESS_TESTED public +#else + #define JSON_PRIVATE_UNLESS_TESTED private +#endif + +/*! +@brief macro to briefly define a mapping between an enum and JSON +@def NLOHMANN_JSON_SERIALIZE_ENUM +@since version 3.4.0 +*/ +#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ + template \ + inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [e](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.first == e; \ + }); \ + j = ((it != std::end(m)) ? it : std::begin(m))->second; \ + } \ + template \ + inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [&j](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.second == j; \ + }); \ + e = ((it != std::end(m)) ? it : std::begin(m))->first; \ + } + +// Ugly macros to avoid uglier copy-paste when specializing basic_json. They +// may be removed in the future once the class is split. + +#define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ + template class ObjectType, \ + template class ArrayType, \ + class StringType, class BooleanType, class NumberIntegerType, \ + class NumberUnsignedType, class NumberFloatType, \ + template class AllocatorType, \ + template class JSONSerializer, \ + class BinaryType> + +#define NLOHMANN_BASIC_JSON_TPL \ + basic_json + +// Macros to simplify conversion from/to types + +#define NLOHMANN_JSON_EXPAND( x ) x +#define NLOHMANN_JSON_GET_MACRO(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, _25, _26, _27, _28, _29, _30, _31, _32, _33, _34, _35, _36, _37, _38, _39, _40, _41, _42, _43, _44, _45, _46, _47, _48, _49, _50, _51, _52, _53, _54, _55, _56, _57, _58, _59, _60, _61, _62, _63, _64, NAME,...) NAME +#define NLOHMANN_JSON_PASTE(...) NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_GET_MACRO(__VA_ARGS__, \ + NLOHMANN_JSON_PASTE64, \ + NLOHMANN_JSON_PASTE63, \ + NLOHMANN_JSON_PASTE62, \ + NLOHMANN_JSON_PASTE61, \ + NLOHMANN_JSON_PASTE60, \ + NLOHMANN_JSON_PASTE59, \ + NLOHMANN_JSON_PASTE58, \ + NLOHMANN_JSON_PASTE57, \ + NLOHMANN_JSON_PASTE56, \ + NLOHMANN_JSON_PASTE55, \ + NLOHMANN_JSON_PASTE54, \ + NLOHMANN_JSON_PASTE53, \ + NLOHMANN_JSON_PASTE52, \ + NLOHMANN_JSON_PASTE51, \ + NLOHMANN_JSON_PASTE50, \ + NLOHMANN_JSON_PASTE49, \ + NLOHMANN_JSON_PASTE48, \ + NLOHMANN_JSON_PASTE47, \ + NLOHMANN_JSON_PASTE46, \ + NLOHMANN_JSON_PASTE45, \ + NLOHMANN_JSON_PASTE44, \ + NLOHMANN_JSON_PASTE43, \ + NLOHMANN_JSON_PASTE42, \ + NLOHMANN_JSON_PASTE41, \ + NLOHMANN_JSON_PASTE40, \ + NLOHMANN_JSON_PASTE39, \ + NLOHMANN_JSON_PASTE38, \ + NLOHMANN_JSON_PASTE37, \ + NLOHMANN_JSON_PASTE36, \ + NLOHMANN_JSON_PASTE35, \ + NLOHMANN_JSON_PASTE34, \ + NLOHMANN_JSON_PASTE33, \ + NLOHMANN_JSON_PASTE32, \ + NLOHMANN_JSON_PASTE31, \ + NLOHMANN_JSON_PASTE30, \ + NLOHMANN_JSON_PASTE29, \ + NLOHMANN_JSON_PASTE28, \ + NLOHMANN_JSON_PASTE27, \ + NLOHMANN_JSON_PASTE26, \ + NLOHMANN_JSON_PASTE25, \ + NLOHMANN_JSON_PASTE24, \ + NLOHMANN_JSON_PASTE23, \ + NLOHMANN_JSON_PASTE22, \ + NLOHMANN_JSON_PASTE21, \ + NLOHMANN_JSON_PASTE20, \ + NLOHMANN_JSON_PASTE19, \ + NLOHMANN_JSON_PASTE18, \ + NLOHMANN_JSON_PASTE17, \ + NLOHMANN_JSON_PASTE16, \ + NLOHMANN_JSON_PASTE15, \ + NLOHMANN_JSON_PASTE14, \ + NLOHMANN_JSON_PASTE13, \ + NLOHMANN_JSON_PASTE12, \ + NLOHMANN_JSON_PASTE11, \ + NLOHMANN_JSON_PASTE10, \ + NLOHMANN_JSON_PASTE9, \ + NLOHMANN_JSON_PASTE8, \ + NLOHMANN_JSON_PASTE7, \ + NLOHMANN_JSON_PASTE6, \ + NLOHMANN_JSON_PASTE5, \ + NLOHMANN_JSON_PASTE4, \ + NLOHMANN_JSON_PASTE3, \ + NLOHMANN_JSON_PASTE2, \ + NLOHMANN_JSON_PASTE1)(__VA_ARGS__)) +#define NLOHMANN_JSON_PASTE2(func, v1) func(v1) +#define NLOHMANN_JSON_PASTE3(func, v1, v2) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE2(func, v2) +#define NLOHMANN_JSON_PASTE4(func, v1, v2, v3) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE3(func, v2, v3) +#define NLOHMANN_JSON_PASTE5(func, v1, v2, v3, v4) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE4(func, v2, v3, v4) +#define NLOHMANN_JSON_PASTE6(func, v1, v2, v3, v4, v5) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE5(func, v2, v3, v4, v5) +#define NLOHMANN_JSON_PASTE7(func, v1, v2, v3, v4, v5, v6) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE6(func, v2, v3, v4, v5, v6) +#define NLOHMANN_JSON_PASTE8(func, v1, v2, v3, v4, v5, v6, v7) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE7(func, v2, v3, v4, v5, v6, v7) +#define NLOHMANN_JSON_PASTE9(func, v1, v2, v3, v4, v5, v6, v7, v8) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE8(func, v2, v3, v4, v5, v6, v7, v8) +#define NLOHMANN_JSON_PASTE10(func, v1, v2, v3, v4, v5, v6, v7, v8, v9) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE9(func, v2, v3, v4, v5, v6, v7, v8, v9) +#define NLOHMANN_JSON_PASTE11(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE10(func, v2, v3, v4, v5, v6, v7, v8, v9, v10) +#define NLOHMANN_JSON_PASTE12(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE11(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11) +#define NLOHMANN_JSON_PASTE13(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE12(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12) +#define NLOHMANN_JSON_PASTE14(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE13(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13) +#define NLOHMANN_JSON_PASTE15(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE14(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14) +#define NLOHMANN_JSON_PASTE16(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE15(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15) +#define NLOHMANN_JSON_PASTE17(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE16(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16) +#define NLOHMANN_JSON_PASTE18(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE17(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17) +#define NLOHMANN_JSON_PASTE19(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE18(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18) +#define NLOHMANN_JSON_PASTE20(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE19(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19) +#define NLOHMANN_JSON_PASTE21(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE20(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20) +#define NLOHMANN_JSON_PASTE22(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE21(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21) +#define NLOHMANN_JSON_PASTE23(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE22(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22) +#define NLOHMANN_JSON_PASTE24(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE23(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23) +#define NLOHMANN_JSON_PASTE25(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE24(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24) +#define NLOHMANN_JSON_PASTE26(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE25(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25) +#define NLOHMANN_JSON_PASTE27(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE26(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26) +#define NLOHMANN_JSON_PASTE28(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE27(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27) +#define NLOHMANN_JSON_PASTE29(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE28(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28) +#define NLOHMANN_JSON_PASTE30(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE29(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29) +#define NLOHMANN_JSON_PASTE31(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE30(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30) +#define NLOHMANN_JSON_PASTE32(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE31(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31) +#define NLOHMANN_JSON_PASTE33(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE32(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32) +#define NLOHMANN_JSON_PASTE34(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE33(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33) +#define NLOHMANN_JSON_PASTE35(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE34(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34) +#define NLOHMANN_JSON_PASTE36(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE35(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35) +#define NLOHMANN_JSON_PASTE37(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE36(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36) +#define NLOHMANN_JSON_PASTE38(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE37(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37) +#define NLOHMANN_JSON_PASTE39(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE38(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38) +#define NLOHMANN_JSON_PASTE40(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE39(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39) +#define NLOHMANN_JSON_PASTE41(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE40(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40) +#define NLOHMANN_JSON_PASTE42(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE41(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41) +#define NLOHMANN_JSON_PASTE43(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE42(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42) +#define NLOHMANN_JSON_PASTE44(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE43(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43) +#define NLOHMANN_JSON_PASTE45(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE44(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44) +#define NLOHMANN_JSON_PASTE46(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE45(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45) +#define NLOHMANN_JSON_PASTE47(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE46(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46) +#define NLOHMANN_JSON_PASTE48(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE47(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47) +#define NLOHMANN_JSON_PASTE49(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE48(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48) +#define NLOHMANN_JSON_PASTE50(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE49(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49) +#define NLOHMANN_JSON_PASTE51(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE50(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50) +#define NLOHMANN_JSON_PASTE52(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE51(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51) +#define NLOHMANN_JSON_PASTE53(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE52(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52) +#define NLOHMANN_JSON_PASTE54(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE53(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53) +#define NLOHMANN_JSON_PASTE55(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE54(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54) +#define NLOHMANN_JSON_PASTE56(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE55(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55) +#define NLOHMANN_JSON_PASTE57(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE56(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56) +#define NLOHMANN_JSON_PASTE58(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE57(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57) +#define NLOHMANN_JSON_PASTE59(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE58(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58) +#define NLOHMANN_JSON_PASTE60(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE59(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59) +#define NLOHMANN_JSON_PASTE61(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE60(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60) +#define NLOHMANN_JSON_PASTE62(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE61(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61) +#define NLOHMANN_JSON_PASTE63(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE62(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62) +#define NLOHMANN_JSON_PASTE64(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62, v63) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE63(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62, v63) + +#define NLOHMANN_JSON_TO(v1) nlohmann_json_j[#v1] = nlohmann_json_t.v1; +#define NLOHMANN_JSON_FROM(v1) nlohmann_json_j.at(#v1).get_to(nlohmann_json_t.v1); +#define NLOHMANN_JSON_FROM_WITH_DEFAULT(v1) nlohmann_json_t.v1 = nlohmann_json_j.value(#v1, nlohmann_json_default_obj.v1); + +/*! +@brief macro +@def NLOHMANN_DEFINE_TYPE_INTRUSIVE +@since version 3.9.0 +*/ +#define NLOHMANN_DEFINE_TYPE_INTRUSIVE(Type, ...) \ + friend void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + friend void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) } + +#define NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(Type, ...) \ + friend void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + friend void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { Type nlohmann_json_default_obj; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE +@since version 3.9.0 +*/ +#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Type, ...) \ + inline void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + inline void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) } + +#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(Type, ...) \ + inline void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + inline void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { Type nlohmann_json_default_obj; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) } + + +// inspired from https://stackoverflow.com/a/26745591 +// allows to call any std function as if (e.g. with begin): +// using std::begin; begin(x); +// +// it allows using the detected idiom to retrieve the return type +// of such an expression +#define NLOHMANN_CAN_CALL_STD_FUNC_IMPL(std_name) \ + namespace detail { \ + using std::std_name; \ + \ + template \ + using result_of_##std_name = decltype(std_name(std::declval()...)); \ + } \ + \ + namespace detail2 { \ + struct std_name##_tag \ + { \ + }; \ + \ + template \ + std_name##_tag std_name(T&&...); \ + \ + template \ + using result_of_##std_name = decltype(std_name(std::declval()...)); \ + \ + template \ + struct would_call_std_##std_name \ + { \ + static constexpr auto const value = ::nlohmann::detail:: \ + is_detected_exact::value; \ + }; \ + } /* namespace detail2 */ \ + \ + template \ + struct would_call_std_##std_name : detail2::would_call_std_##std_name \ + { \ + } + +#ifndef JSON_USE_IMPLICIT_CONVERSIONS + #define JSON_USE_IMPLICIT_CONVERSIONS 1 +#endif + +#if JSON_USE_IMPLICIT_CONVERSIONS + #define JSON_EXPLICIT +#else + #define JSON_EXPLICIT explicit +#endif + +#ifndef JSON_DIAGNOSTICS + #define JSON_DIAGNOSTICS 0 +#endif + + +namespace nlohmann +{ +namespace detail +{ + +/*! +@brief replace all occurrences of a substring by another string + +@param[in,out] s the string to manipulate; changed so that all + occurrences of @a f are replaced with @a t +@param[in] f the substring to replace with @a t +@param[in] t the string to replace @a f + +@pre The search string @a f must not be empty. **This precondition is +enforced with an assertion.** + +@since version 2.0.0 +*/ +inline void replace_substring(std::string& s, const std::string& f, + const std::string& t) +{ + JSON_ASSERT(!f.empty()); + for (auto pos = s.find(f); // find first occurrence of f + pos != std::string::npos; // make sure f was found + s.replace(pos, f.size(), t), // replace with t, and + pos = s.find(f, pos + t.size())) // find next occurrence of f + {} +} + +/*! + * @brief string escaping as described in RFC 6901 (Sect. 4) + * @param[in] s string to escape + * @return escaped string + * + * Note the order of escaping "~" to "~0" and "/" to "~1" is important. + */ +inline std::string escape(std::string s) +{ + replace_substring(s, "~", "~0"); + replace_substring(s, "/", "~1"); + return s; +} + +/*! + * @brief string unescaping as described in RFC 6901 (Sect. 4) + * @param[in] s string to unescape + * @return unescaped string + * + * Note the order of escaping "~1" to "/" and "~0" to "~" is important. + */ +static void unescape(std::string& s) +{ + replace_substring(s, "~1", "/"); + replace_substring(s, "~0", "~"); +} + +} // namespace detail +} // namespace nlohmann + +// #include + + +#include // size_t + +namespace nlohmann +{ +namespace detail +{ +/// struct to capture the start position of the current token +struct position_t +{ + /// the total number of characters read + std::size_t chars_read_total = 0; + /// the number of characters read in the current line + std::size_t chars_read_current_line = 0; + /// the number of lines read + std::size_t lines_read = 0; + + /// conversion to size_t to preserve SAX interface + constexpr operator size_t() const + { + return chars_read_total; + } +}; + +} // namespace detail +} // namespace nlohmann + +// #include + + +namespace nlohmann +{ +namespace detail +{ +//////////////// +// exceptions // +//////////////// + +/// @brief general exception of the @ref basic_json class +/// @sa https://json.nlohmann.me/api/basic_json/exception/ +class exception : public std::exception +{ + public: + /// returns the explanatory string + const char* what() const noexcept override + { + return m.what(); + } + + /// the id of the exception + const int id; // NOLINT(cppcoreguidelines-non-private-member-variables-in-classes) + + protected: + JSON_HEDLEY_NON_NULL(3) + exception(int id_, const char* what_arg) : id(id_), m(what_arg) {} // NOLINT(bugprone-throw-keyword-missing) + + static std::string name(const std::string& ename, int id_) + { + return "[json.exception." + ename + "." + std::to_string(id_) + "] "; + } + + template + static std::string diagnostics(const BasicJsonType& leaf_element) + { +#if JSON_DIAGNOSTICS + std::vector tokens; + for (const auto* current = &leaf_element; current->m_parent != nullptr; current = current->m_parent) + { + switch (current->m_parent->type()) + { + case value_t::array: + { + for (std::size_t i = 0; i < current->m_parent->m_value.array->size(); ++i) + { + if (¤t->m_parent->m_value.array->operator[](i) == current) + { + tokens.emplace_back(std::to_string(i)); + break; + } + } + break; + } + + case value_t::object: + { + for (const auto& element : *current->m_parent->m_value.object) + { + if (&element.second == current) + { + tokens.emplace_back(element.first.c_str()); + break; + } + } + break; + } + + case value_t::null: // LCOV_EXCL_LINE + case value_t::string: // LCOV_EXCL_LINE + case value_t::boolean: // LCOV_EXCL_LINE + case value_t::number_integer: // LCOV_EXCL_LINE + case value_t::number_unsigned: // LCOV_EXCL_LINE + case value_t::number_float: // LCOV_EXCL_LINE + case value_t::binary: // LCOV_EXCL_LINE + case value_t::discarded: // LCOV_EXCL_LINE + default: // LCOV_EXCL_LINE + break; // LCOV_EXCL_LINE + } + } + + if (tokens.empty()) + { + return ""; + } + + return "(" + std::accumulate(tokens.rbegin(), tokens.rend(), std::string{}, + [](const std::string & a, const std::string & b) + { + return a + "/" + detail::escape(b); + }) + ") "; +#else + static_cast(leaf_element); + return ""; +#endif + } + + private: + /// an exception object as storage for error messages + std::runtime_error m; +}; + +/// @brief exception indicating a parse error +/// @sa https://json.nlohmann.me/api/basic_json/parse_error/ +class parse_error : public exception +{ + public: + /*! + @brief create a parse error exception + @param[in] id_ the id of the exception + @param[in] pos the position where the error occurred (or with + chars_read_total=0 if the position cannot be + determined) + @param[in] what_arg the explanatory string + @return parse_error object + */ + template + static parse_error create(int id_, const position_t& pos, const std::string& what_arg, const BasicJsonType& context) + { + std::string w = exception::name("parse_error", id_) + "parse error" + + position_string(pos) + ": " + exception::diagnostics(context) + what_arg; + return {id_, pos.chars_read_total, w.c_str()}; + } + + template + static parse_error create(int id_, std::size_t byte_, const std::string& what_arg, const BasicJsonType& context) + { + std::string w = exception::name("parse_error", id_) + "parse error" + + (byte_ != 0 ? (" at byte " + std::to_string(byte_)) : "") + + ": " + exception::diagnostics(context) + what_arg; + return {id_, byte_, w.c_str()}; + } + + /*! + @brief byte index of the parse error + + The byte index of the last read character in the input file. + + @note For an input with n bytes, 1 is the index of the first character and + n+1 is the index of the terminating null byte or the end of file. + This also holds true when reading a byte vector (CBOR or MessagePack). + */ + const std::size_t byte; + + private: + parse_error(int id_, std::size_t byte_, const char* what_arg) + : exception(id_, what_arg), byte(byte_) {} + + static std::string position_string(const position_t& pos) + { + return " at line " + std::to_string(pos.lines_read + 1) + + ", column " + std::to_string(pos.chars_read_current_line); + } +}; + +/// @brief exception indicating errors with iterators +/// @sa https://json.nlohmann.me/api/basic_json/invalid_iterator/ +class invalid_iterator : public exception +{ + public: + template + static invalid_iterator create(int id_, const std::string& what_arg, const BasicJsonType& context) + { + std::string w = exception::name("invalid_iterator", id_) + exception::diagnostics(context) + what_arg; + return {id_, w.c_str()}; + } + + private: + JSON_HEDLEY_NON_NULL(3) + invalid_iterator(int id_, const char* what_arg) + : exception(id_, what_arg) {} +}; + +/// @brief exception indicating executing a member function with a wrong type +/// @sa https://json.nlohmann.me/api/basic_json/type_error/ +class type_error : public exception +{ + public: + template + static type_error create(int id_, const std::string& what_arg, const BasicJsonType& context) + { + std::string w = exception::name("type_error", id_) + exception::diagnostics(context) + what_arg; + return {id_, w.c_str()}; + } + + private: + JSON_HEDLEY_NON_NULL(3) + type_error(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; + +/// @brief exception indicating access out of the defined range +/// @sa https://json.nlohmann.me/api/basic_json/out_of_range/ +class out_of_range : public exception +{ + public: + template + static out_of_range create(int id_, const std::string& what_arg, const BasicJsonType& context) + { + std::string w = exception::name("out_of_range", id_) + exception::diagnostics(context) + what_arg; + return {id_, w.c_str()}; + } + + private: + JSON_HEDLEY_NON_NULL(3) + out_of_range(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; + +/// @brief exception indicating other library errors +/// @sa https://json.nlohmann.me/api/basic_json/other_error/ +class other_error : public exception +{ + public: + template + static other_error create(int id_, const std::string& what_arg, const BasicJsonType& context) + { + std::string w = exception::name("other_error", id_) + exception::diagnostics(context) + what_arg; + return {id_, w.c_str()}; + } + + private: + JSON_HEDLEY_NON_NULL(3) + other_error(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; + +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + + +#include // size_t +#include // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type +#include // index_sequence, make_index_sequence, index_sequence_for + +// #include + + +namespace nlohmann +{ +namespace detail +{ + +template +using uncvref_t = typename std::remove_cv::type>::type; + +#ifdef JSON_HAS_CPP_14 + +// the following utilities are natively available in C++14 +using std::enable_if_t; +using std::index_sequence; +using std::make_index_sequence; +using std::index_sequence_for; + +#else + +// alias templates to reduce boilerplate +template +using enable_if_t = typename std::enable_if::type; + +// The following code is taken from https://github.com/abseil/abseil-cpp/blob/10cb35e459f5ecca5b2ff107635da0bfa41011b4/absl/utility/utility.h +// which is part of Google Abseil (https://github.com/abseil/abseil-cpp), licensed under the Apache License 2.0. + +//// START OF CODE FROM GOOGLE ABSEIL + +// integer_sequence +// +// Class template representing a compile-time integer sequence. An instantiation +// of `integer_sequence` has a sequence of integers encoded in its +// type through its template arguments (which is a common need when +// working with C++11 variadic templates). `absl::integer_sequence` is designed +// to be a drop-in replacement for C++14's `std::integer_sequence`. +// +// Example: +// +// template< class T, T... Ints > +// void user_function(integer_sequence); +// +// int main() +// { +// // user_function's `T` will be deduced to `int` and `Ints...` +// // will be deduced to `0, 1, 2, 3, 4`. +// user_function(make_integer_sequence()); +// } +template +struct integer_sequence +{ + using value_type = T; + static constexpr std::size_t size() noexcept + { + return sizeof...(Ints); + } +}; + +// index_sequence +// +// A helper template for an `integer_sequence` of `size_t`, +// `absl::index_sequence` is designed to be a drop-in replacement for C++14's +// `std::index_sequence`. +template +using index_sequence = integer_sequence; + +namespace utility_internal +{ + +template +struct Extend; + +// Note that SeqSize == sizeof...(Ints). It's passed explicitly for efficiency. +template +struct Extend, SeqSize, 0> +{ + using type = integer_sequence < T, Ints..., (Ints + SeqSize)... >; +}; + +template +struct Extend, SeqSize, 1> +{ + using type = integer_sequence < T, Ints..., (Ints + SeqSize)..., 2 * SeqSize >; +}; + +// Recursion helper for 'make_integer_sequence'. +// 'Gen::type' is an alias for 'integer_sequence'. +template +struct Gen +{ + using type = + typename Extend < typename Gen < T, N / 2 >::type, N / 2, N % 2 >::type; +}; + +template +struct Gen +{ + using type = integer_sequence; +}; + +} // namespace utility_internal + +// Compile-time sequences of integers + +// make_integer_sequence +// +// This template alias is equivalent to +// `integer_sequence`, and is designed to be a drop-in +// replacement for C++14's `std::make_integer_sequence`. +template +using make_integer_sequence = typename utility_internal::Gen::type; + +// make_index_sequence +// +// This template alias is equivalent to `index_sequence<0, 1, ..., N-1>`, +// and is designed to be a drop-in replacement for C++14's +// `std::make_index_sequence`. +template +using make_index_sequence = make_integer_sequence; + +// index_sequence_for +// +// Converts a typename pack into an index sequence of the same length, and +// is designed to be a drop-in replacement for C++14's +// `std::index_sequence_for()` +template +using index_sequence_for = make_index_sequence; + +//// END OF CODE FROM GOOGLE ABSEIL + +#endif + +// dispatch utility (taken from ranges-v3) +template struct priority_tag : priority_tag < N - 1 > {}; +template<> struct priority_tag<0> {}; + +// taken from ranges-v3 +template +struct static_const +{ + static constexpr T value{}; +}; + +template +constexpr T static_const::value; // NOLINT(readability-redundant-declaration) + +} // namespace detail +} // namespace nlohmann + +// #include + + +namespace nlohmann +{ +namespace detail +{ +// dispatching helper struct +template struct identity_tag {}; +} // namespace detail +} // namespace nlohmann + +// #include + + +#include // numeric_limits +#include // false_type, is_constructible, is_integral, is_same, true_type +#include // declval +#include // tuple + +// #include + + +// #include + + +#include // random_access_iterator_tag + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template +struct iterator_types {}; + +template +struct iterator_types < + It, + void_t> +{ + using difference_type = typename It::difference_type; + using value_type = typename It::value_type; + using pointer = typename It::pointer; + using reference = typename It::reference; + using iterator_category = typename It::iterator_category; +}; + +// This is required as some compilers implement std::iterator_traits in a way that +// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341. +template +struct iterator_traits +{ +}; + +template +struct iterator_traits < T, enable_if_t < !std::is_pointer::value >> + : iterator_types +{ +}; + +template +struct iterator_traits::value>> +{ + using iterator_category = std::random_access_iterator_tag; + using value_type = T; + using difference_type = ptrdiff_t; + using pointer = T*; + using reference = T&; +}; +} // namespace detail +} // namespace nlohmann + +// #include + + +// #include + + +namespace nlohmann +{ +NLOHMANN_CAN_CALL_STD_FUNC_IMPL(begin); +} // namespace nlohmann + +// #include + + +// #include + + +namespace nlohmann +{ +NLOHMANN_CAN_CALL_STD_FUNC_IMPL(end); +} // namespace nlohmann + +// #include + +// #include + +// #include +#ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_ +#define INCLUDE_NLOHMANN_JSON_FWD_HPP_ + +#include // int64_t, uint64_t +#include // map +#include // allocator +#include // string +#include // vector + +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ +/*! +@brief default JSONSerializer template argument + +This serializer ignores the template arguments and uses ADL +([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) +for serialization. +*/ +template +struct adl_serializer; + +/// a class to store JSON values +/// @sa https://json.nlohmann.me/api/basic_json/ +template class ObjectType = + std::map, + template class ArrayType = std::vector, + class StringType = std::string, class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template class AllocatorType = std::allocator, + template class JSONSerializer = + adl_serializer, + class BinaryType = std::vector> +class basic_json; + +/// @brief JSON Pointer defines a string syntax for identifying a specific value within a JSON document +/// @sa https://json.nlohmann.me/api/json_pointer/ +template +class json_pointer; + +/*! +@brief default specialization +@sa https://json.nlohmann.me/api/json/ +*/ +using json = basic_json<>; + +/// @brief a minimal map-like container that preserves insertion order +/// @sa https://json.nlohmann.me/api/ordered_map/ +template +struct ordered_map; + +/// @brief specialization that maintains the insertion order of object keys +/// @sa https://json.nlohmann.me/api/ordered_json/ +using ordered_json = basic_json; + +} // namespace nlohmann + +#endif // INCLUDE_NLOHMANN_JSON_FWD_HPP_ + + +namespace nlohmann +{ +/*! +@brief detail namespace with internal helper functions + +This namespace collects functions that should not be exposed, +implementations of some @ref basic_json methods, and meta-programming helpers. + +@since version 2.1.0 +*/ +namespace detail +{ +///////////// +// helpers // +///////////// + +// Note to maintainers: +// +// Every trait in this file expects a non CV-qualified type. +// The only exceptions are in the 'aliases for detected' section +// (i.e. those of the form: decltype(T::member_function(std::declval()))) +// +// In this case, T has to be properly CV-qualified to constraint the function arguments +// (e.g. to_json(BasicJsonType&, const T&)) + +template struct is_basic_json : std::false_type {}; + +NLOHMANN_BASIC_JSON_TPL_DECLARATION +struct is_basic_json : std::true_type {}; + +////////////////////// +// json_ref helpers // +////////////////////// + +template +class json_ref; + +template +struct is_json_ref : std::false_type {}; + +template +struct is_json_ref> : std::true_type {}; + +////////////////////////// +// aliases for detected // +////////////////////////// + +template +using mapped_type_t = typename T::mapped_type; + +template +using key_type_t = typename T::key_type; + +template +using value_type_t = typename T::value_type; + +template +using difference_type_t = typename T::difference_type; + +template +using pointer_t = typename T::pointer; + +template +using reference_t = typename T::reference; + +template +using iterator_category_t = typename T::iterator_category; + +template +using to_json_function = decltype(T::to_json(std::declval()...)); + +template +using from_json_function = decltype(T::from_json(std::declval()...)); + +template +using get_template_function = decltype(std::declval().template get()); + +// trait checking if JSONSerializer::from_json(json const&, udt&) exists +template +struct has_from_json : std::false_type {}; + +// trait checking if j.get is valid +// use this trait instead of std::is_constructible or std::is_convertible, +// both rely on, or make use of implicit conversions, and thus fail when T +// has several constructors/operator= (see https://github.com/nlohmann/json/issues/958) +template +struct is_getable +{ + static constexpr bool value = is_detected::value; +}; + +template +struct has_from_json < BasicJsonType, T, enable_if_t < !is_basic_json::value >> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + +// This trait checks if JSONSerializer::from_json(json const&) exists +// this overload is used for non-default-constructible user-defined-types +template +struct has_non_default_from_json : std::false_type {}; + +template +struct has_non_default_from_json < BasicJsonType, T, enable_if_t < !is_basic_json::value >> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + +// This trait checks if BasicJsonType::json_serializer::to_json exists +// Do not evaluate the trait when T is a basic_json type, to avoid template instantiation infinite recursion. +template +struct has_to_json : std::false_type {}; + +template +struct has_to_json < BasicJsonType, T, enable_if_t < !is_basic_json::value >> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + + +/////////////////// +// is_ functions // +/////////////////// + +// https://en.cppreference.com/w/cpp/types/conjunction +template struct conjunction : std::true_type { }; +template struct conjunction : B1 { }; +template +struct conjunction +: std::conditional, B1>::type {}; + +// https://en.cppreference.com/w/cpp/types/negation +template struct negation : std::integral_constant < bool, !B::value > { }; + +// Reimplementation of is_constructible and is_default_constructible, due to them being broken for +// std::pair and std::tuple until LWG 2367 fix (see https://cplusplus.github.io/LWG/lwg-defects.html#2367). +// This causes compile errors in e.g. clang 3.5 or gcc 4.9. +template +struct is_default_constructible : std::is_default_constructible {}; + +template +struct is_default_constructible> + : conjunction, is_default_constructible> {}; + +template +struct is_default_constructible> + : conjunction, is_default_constructible> {}; + +template +struct is_default_constructible> + : conjunction...> {}; + +template +struct is_default_constructible> + : conjunction...> {}; + + +template +struct is_constructible : std::is_constructible {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_constructible> : is_default_constructible> {}; + + +template +struct is_iterator_traits : std::false_type {}; + +template +struct is_iterator_traits> +{ + private: + using traits = iterator_traits; + + public: + static constexpr auto value = + is_detected::value && + is_detected::value && + is_detected::value && + is_detected::value && + is_detected::value; +}; + +template +struct is_range +{ + private: + using t_ref = typename std::add_lvalue_reference::type; + + using iterator = detected_t; + using sentinel = detected_t; + + // to be 100% correct, it should use https://en.cppreference.com/w/cpp/iterator/input_or_output_iterator + // and https://en.cppreference.com/w/cpp/iterator/sentinel_for + // but reimplementing these would be too much work, as a lot of other concepts are used underneath + static constexpr auto is_iterator_begin = + is_iterator_traits>::value; + + public: + static constexpr bool value = !std::is_same::value && !std::is_same::value && is_iterator_begin; +}; + +template +using iterator_t = enable_if_t::value, result_of_begin())>>; + +template +using range_value_t = value_type_t>>; + +// The following implementation of is_complete_type is taken from +// https://blogs.msdn.microsoft.com/vcblog/2015/12/02/partial-support-for-expression-sfinae-in-vs-2015-update-1/ +// and is written by Xiang Fan who agreed to using it in this library. + +template +struct is_complete_type : std::false_type {}; + +template +struct is_complete_type : std::true_type {}; + +template +struct is_compatible_object_type_impl : std::false_type {}; + +template +struct is_compatible_object_type_impl < + BasicJsonType, CompatibleObjectType, + enable_if_t < is_detected::value&& + is_detected::value >> +{ + using object_t = typename BasicJsonType::object_t; + + // macOS's is_constructible does not play well with nonesuch... + static constexpr bool value = + is_constructible::value && + is_constructible::value; +}; + +template +struct is_compatible_object_type + : is_compatible_object_type_impl {}; + +template +struct is_constructible_object_type_impl : std::false_type {}; + +template +struct is_constructible_object_type_impl < + BasicJsonType, ConstructibleObjectType, + enable_if_t < is_detected::value&& + is_detected::value >> +{ + using object_t = typename BasicJsonType::object_t; + + static constexpr bool value = + (is_default_constructible::value && + (std::is_move_assignable::value || + std::is_copy_assignable::value) && + (is_constructible::value && + std::is_same < + typename object_t::mapped_type, + typename ConstructibleObjectType::mapped_type >::value)) || + (has_from_json::value || + has_non_default_from_json < + BasicJsonType, + typename ConstructibleObjectType::mapped_type >::value); +}; + +template +struct is_constructible_object_type + : is_constructible_object_type_impl {}; + +template +struct is_compatible_string_type +{ + static constexpr auto value = + is_constructible::value; +}; + +template +struct is_constructible_string_type +{ + static constexpr auto value = + is_constructible::value; +}; + +template +struct is_compatible_array_type_impl : std::false_type {}; + +template +struct is_compatible_array_type_impl < + BasicJsonType, CompatibleArrayType, + enable_if_t < + is_detected::value&& + is_iterator_traits>>::value&& +// special case for types like std::filesystem::path whose iterator's value_type are themselves +// c.f. https://github.com/nlohmann/json/pull/3073 + !std::is_same>::value >> +{ + static constexpr bool value = + is_constructible>::value; +}; + +template +struct is_compatible_array_type + : is_compatible_array_type_impl {}; + +template +struct is_constructible_array_type_impl : std::false_type {}; + +template +struct is_constructible_array_type_impl < + BasicJsonType, ConstructibleArrayType, + enable_if_t::value >> + : std::true_type {}; + +template +struct is_constructible_array_type_impl < + BasicJsonType, ConstructibleArrayType, + enable_if_t < !std::is_same::value&& + !is_compatible_string_type::value&& + is_default_constructible::value&& +(std::is_move_assignable::value || + std::is_copy_assignable::value)&& +is_detected::value&& +is_iterator_traits>>::value&& +is_detected::value&& +// special case for types like std::filesystem::path whose iterator's value_type are themselves +// c.f. https://github.com/nlohmann/json/pull/3073 +!std::is_same>::value&& + is_complete_type < + detected_t>::value >> +{ + using value_type = range_value_t; + + static constexpr bool value = + std::is_same::value || + has_from_json::value || + has_non_default_from_json < + BasicJsonType, + value_type >::value; +}; + +template +struct is_constructible_array_type + : is_constructible_array_type_impl {}; + +template +struct is_compatible_integer_type_impl : std::false_type {}; + +template +struct is_compatible_integer_type_impl < + RealIntegerType, CompatibleNumberIntegerType, + enable_if_t < std::is_integral::value&& + std::is_integral::value&& + !std::is_same::value >> +{ + // is there an assert somewhere on overflows? + using RealLimits = std::numeric_limits; + using CompatibleLimits = std::numeric_limits; + + static constexpr auto value = + is_constructible::value && + CompatibleLimits::is_integer && + RealLimits::is_signed == CompatibleLimits::is_signed; +}; + +template +struct is_compatible_integer_type + : is_compatible_integer_type_impl {}; + +template +struct is_compatible_type_impl: std::false_type {}; + +template +struct is_compatible_type_impl < + BasicJsonType, CompatibleType, + enable_if_t::value >> +{ + static constexpr bool value = + has_to_json::value; +}; + +template +struct is_compatible_type + : is_compatible_type_impl {}; + +template +struct is_constructible_tuple : std::false_type {}; + +template +struct is_constructible_tuple> : conjunction...> {}; + +// a naive helper to check if a type is an ordered_map (exploits the fact that +// ordered_map inherits capacity() from std::vector) +template +struct is_ordered_map +{ + using one = char; + + struct two + { + char x[2]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + }; + + template static one test( decltype(&C::capacity) ) ; + template static two test(...); + + enum { value = sizeof(test(nullptr)) == sizeof(char) }; // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) +}; + +// to avoid useless casts (see https://github.com/nlohmann/json/issues/2893#issuecomment-889152324) +template < typename T, typename U, enable_if_t < !std::is_same::value, int > = 0 > +T conditional_static_cast(U value) +{ + return static_cast(value); +} + +template::value, int> = 0> +T conditional_static_cast(U value) +{ + return value; +} + +} // namespace detail +} // namespace nlohmann + +// #include + + +#if JSON_HAS_EXPERIMENTAL_FILESYSTEM +#include +namespace nlohmann::detail +{ +namespace std_fs = std::experimental::filesystem; +} // namespace nlohmann::detail +#elif JSON_HAS_FILESYSTEM +#include +namespace nlohmann::detail +{ +namespace std_fs = std::filesystem; +} // namespace nlohmann::detail +#endif + +namespace nlohmann +{ +namespace detail +{ +template +void from_json(const BasicJsonType& j, typename std::nullptr_t& n) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_null())) + { + JSON_THROW(type_error::create(302, "type must be null, but is " + std::string(j.type_name()), j)); + } + n = nullptr; +} + +// overloads for basic_json template parameters +template < typename BasicJsonType, typename ArithmeticType, + enable_if_t < std::is_arithmetic::value&& + !std::is_same::value, + int > = 0 > +void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val) +{ + switch (static_cast(j)) + { + case value_t::number_unsigned: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::number_integer: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::number_float: + { + val = static_cast(*j.template get_ptr()); + break; + } + + case value_t::null: + case value_t::object: + case value_t::array: + case value_t::string: + case value_t::boolean: + case value_t::binary: + case value_t::discarded: + default: + JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name()), j)); + } +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_boolean())) + { + JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(j.type_name()), j)); + } + b = *j.template get_ptr(); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_string())) + { + JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name()), j)); + } + s = *j.template get_ptr(); +} + +template < + typename BasicJsonType, typename ConstructibleStringType, + enable_if_t < + is_constructible_string_type::value&& + !std::is_same::value, + int > = 0 > +void from_json(const BasicJsonType& j, ConstructibleStringType& s) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_string())) + { + JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name()), j)); + } + + s = *j.template get_ptr(); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::number_float_t& val) +{ + get_arithmetic_value(j, val); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val) +{ + get_arithmetic_value(j, val); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::number_integer_t& val) +{ + get_arithmetic_value(j, val); +} + +template::value, int> = 0> +void from_json(const BasicJsonType& j, EnumType& e) +{ + typename std::underlying_type::type val; + get_arithmetic_value(j, val); + e = static_cast(val); +} + +// forward_list doesn't have an insert method +template::value, int> = 0> +void from_json(const BasicJsonType& j, std::forward_list& l) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()), j)); + } + l.clear(); + std::transform(j.rbegin(), j.rend(), + std::front_inserter(l), [](const BasicJsonType & i) + { + return i.template get(); + }); +} + +// valarray doesn't have an insert method +template::value, int> = 0> +void from_json(const BasicJsonType& j, std::valarray& l) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()), j)); + } + l.resize(j.size()); + std::transform(j.begin(), j.end(), std::begin(l), + [](const BasicJsonType & elem) + { + return elem.template get(); + }); +} + +template +auto from_json(const BasicJsonType& j, T (&arr)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) +-> decltype(j.template get(), void()) +{ + for (std::size_t i = 0; i < N; ++i) + { + arr[i] = j.at(i).template get(); + } +} + +template +void from_json_array_impl(const BasicJsonType& j, typename BasicJsonType::array_t& arr, priority_tag<3> /*unused*/) +{ + arr = *j.template get_ptr(); +} + +template +auto from_json_array_impl(const BasicJsonType& j, std::array& arr, + priority_tag<2> /*unused*/) +-> decltype(j.template get(), void()) +{ + for (std::size_t i = 0; i < N; ++i) + { + arr[i] = j.at(i).template get(); + } +} + +template::value, + int> = 0> +auto from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, priority_tag<1> /*unused*/) +-> decltype( + arr.reserve(std::declval()), + j.template get(), + void()) +{ + using std::end; + + ConstructibleArrayType ret; + ret.reserve(j.size()); + std::transform(j.begin(), j.end(), + std::inserter(ret, end(ret)), [](const BasicJsonType & i) + { + // get() returns *this, this won't call a from_json + // method when value_type is BasicJsonType + return i.template get(); + }); + arr = std::move(ret); +} + +template::value, + int> = 0> +void from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, + priority_tag<0> /*unused*/) +{ + using std::end; + + ConstructibleArrayType ret; + std::transform( + j.begin(), j.end(), std::inserter(ret, end(ret)), + [](const BasicJsonType & i) + { + // get() returns *this, this won't call a from_json + // method when value_type is BasicJsonType + return i.template get(); + }); + arr = std::move(ret); +} + +template < typename BasicJsonType, typename ConstructibleArrayType, + enable_if_t < + is_constructible_array_type::value&& + !is_constructible_object_type::value&& + !is_constructible_string_type::value&& + !std::is_same::value&& + !is_basic_json::value, + int > = 0 > +auto from_json(const BasicJsonType& j, ConstructibleArrayType& arr) +-> decltype(from_json_array_impl(j, arr, priority_tag<3> {}), +j.template get(), +void()) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()), j)); + } + + from_json_array_impl(j, arr, priority_tag<3> {}); +} + +template < typename BasicJsonType, typename T, std::size_t... Idx > +std::array from_json_inplace_array_impl(BasicJsonType&& j, + identity_tag> /*unused*/, index_sequence /*unused*/) +{ + return { { std::forward(j).at(Idx).template get()... } }; +} + +template < typename BasicJsonType, typename T, std::size_t N > +auto from_json(BasicJsonType&& j, identity_tag> tag) +-> decltype(from_json_inplace_array_impl(std::forward(j), tag, make_index_sequence {})) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()), j)); + } + + return from_json_inplace_array_impl(std::forward(j), tag, make_index_sequence {}); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::binary_t& bin) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_binary())) + { + JSON_THROW(type_error::create(302, "type must be binary, but is " + std::string(j.type_name()), j)); + } + + bin = *j.template get_ptr(); +} + +template::value, int> = 0> +void from_json(const BasicJsonType& j, ConstructibleObjectType& obj) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_object())) + { + JSON_THROW(type_error::create(302, "type must be object, but is " + std::string(j.type_name()), j)); + } + + ConstructibleObjectType ret; + const auto* inner_object = j.template get_ptr(); + using value_type = typename ConstructibleObjectType::value_type; + std::transform( + inner_object->begin(), inner_object->end(), + std::inserter(ret, ret.begin()), + [](typename BasicJsonType::object_t::value_type const & p) + { + return value_type(p.first, p.second.template get()); + }); + obj = std::move(ret); +} + +// overload for arithmetic types, not chosen for basic_json template arguments +// (BooleanType, etc..); note: Is it really necessary to provide explicit +// overloads for boolean_t etc. in case of a custom BooleanType which is not +// an arithmetic type? +template < typename BasicJsonType, typename ArithmeticType, + enable_if_t < + std::is_arithmetic::value&& + !std::is_same::value&& + !std::is_same::value&& + !std::is_same::value&& + !std::is_same::value, + int > = 0 > +void from_json(const BasicJsonType& j, ArithmeticType& val) +{ + switch (static_cast(j)) + { + case value_t::number_unsigned: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::number_integer: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::number_float: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::boolean: + { + val = static_cast(*j.template get_ptr()); + break; + } + + case value_t::null: + case value_t::object: + case value_t::array: + case value_t::string: + case value_t::binary: + case value_t::discarded: + default: + JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name()), j)); + } +} + +template +std::tuple from_json_tuple_impl_base(BasicJsonType&& j, index_sequence /*unused*/) +{ + return std::make_tuple(std::forward(j).at(Idx).template get()...); +} + +template < typename BasicJsonType, class A1, class A2 > +std::pair from_json_tuple_impl(BasicJsonType&& j, identity_tag> /*unused*/, priority_tag<0> /*unused*/) +{ + return {std::forward(j).at(0).template get(), + std::forward(j).at(1).template get()}; +} + +template +void from_json_tuple_impl(BasicJsonType&& j, std::pair& p, priority_tag<1> /*unused*/) +{ + p = from_json_tuple_impl(std::forward(j), identity_tag> {}, priority_tag<0> {}); +} + +template +std::tuple from_json_tuple_impl(BasicJsonType&& j, identity_tag> /*unused*/, priority_tag<2> /*unused*/) +{ + return from_json_tuple_impl_base(std::forward(j), index_sequence_for {}); +} + +template +void from_json_tuple_impl(BasicJsonType&& j, std::tuple& t, priority_tag<3> /*unused*/) +{ + t = from_json_tuple_impl_base(std::forward(j), index_sequence_for {}); +} + +template +auto from_json(BasicJsonType&& j, TupleRelated&& t) +-> decltype(from_json_tuple_impl(std::forward(j), std::forward(t), priority_tag<3> {})) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()), j)); + } + + return from_json_tuple_impl(std::forward(j), std::forward(t), priority_tag<3> {}); +} + +template < typename BasicJsonType, typename Key, typename Value, typename Compare, typename Allocator, + typename = enable_if_t < !std::is_constructible < + typename BasicJsonType::string_t, Key >::value >> +void from_json(const BasicJsonType& j, std::map& m) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()), j)); + } + m.clear(); + for (const auto& p : j) + { + if (JSON_HEDLEY_UNLIKELY(!p.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(p.type_name()), j)); + } + m.emplace(p.at(0).template get(), p.at(1).template get()); + } +} + +template < typename BasicJsonType, typename Key, typename Value, typename Hash, typename KeyEqual, typename Allocator, + typename = enable_if_t < !std::is_constructible < + typename BasicJsonType::string_t, Key >::value >> +void from_json(const BasicJsonType& j, std::unordered_map& m) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()), j)); + } + m.clear(); + for (const auto& p : j) + { + if (JSON_HEDLEY_UNLIKELY(!p.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(p.type_name()), j)); + } + m.emplace(p.at(0).template get(), p.at(1).template get()); + } +} + +#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM +template +void from_json(const BasicJsonType& j, std_fs::path& p) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_string())) + { + JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name()), j)); + } + p = *j.template get_ptr(); +} +#endif + +struct from_json_fn +{ + template + auto operator()(const BasicJsonType& j, T&& val) const + noexcept(noexcept(from_json(j, std::forward(val)))) + -> decltype(from_json(j, std::forward(val))) + { + return from_json(j, std::forward(val)); + } +}; +} // namespace detail + +/// namespace to hold default `from_json` function +/// to see why this is required: +/// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/n4381.html +namespace // NOLINT(cert-dcl59-cpp,fuchsia-header-anon-namespaces,google-build-namespaces) +{ +constexpr const auto& from_json = detail::static_const::value; // NOLINT(misc-definitions-in-headers) +} // namespace +} // namespace nlohmann + +// #include + + +#include // copy +#include // begin, end +#include // string +#include // tuple, get +#include // is_same, is_constructible, is_floating_point, is_enum, underlying_type +#include // move, forward, declval, pair +#include // valarray +#include // vector + +// #include + +// #include + + +#include // size_t +#include // input_iterator_tag +#include // string, to_string +#include // tuple_size, get, tuple_element +#include // move + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template +void int_to_string( string_type& target, std::size_t value ) +{ + // For ADL + using std::to_string; + target = to_string(value); +} +template class iteration_proxy_value +{ + public: + using difference_type = std::ptrdiff_t; + using value_type = iteration_proxy_value; + using pointer = value_type * ; + using reference = value_type & ; + using iterator_category = std::input_iterator_tag; + using string_type = typename std::remove_cv< typename std::remove_reference().key() ) >::type >::type; + + private: + /// the iterator + IteratorType anchor; + /// an index for arrays (used to create key names) + std::size_t array_index = 0; + /// last stringified array index + mutable std::size_t array_index_last = 0; + /// a string representation of the array index + mutable string_type array_index_str = "0"; + /// an empty string (to return a reference for primitive values) + const string_type empty_str{}; + + public: + explicit iteration_proxy_value(IteratorType it) noexcept + : anchor(std::move(it)) + {} + + /// dereference operator (needed for range-based for) + iteration_proxy_value& operator*() + { + return *this; + } + + /// increment operator (needed for range-based for) + iteration_proxy_value& operator++() + { + ++anchor; + ++array_index; + + return *this; + } + + /// equality operator (needed for InputIterator) + bool operator==(const iteration_proxy_value& o) const + { + return anchor == o.anchor; + } + + /// inequality operator (needed for range-based for) + bool operator!=(const iteration_proxy_value& o) const + { + return anchor != o.anchor; + } + + /// return key of the iterator + const string_type& key() const + { + JSON_ASSERT(anchor.m_object != nullptr); + + switch (anchor.m_object->type()) + { + // use integer array index as key + case value_t::array: + { + if (array_index != array_index_last) + { + int_to_string( array_index_str, array_index ); + array_index_last = array_index; + } + return array_index_str; + } + + // use key from the object + case value_t::object: + return anchor.key(); + + // use an empty key for all primitive types + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + return empty_str; + } + } + + /// return value of the iterator + typename IteratorType::reference value() const + { + return anchor.value(); + } +}; + +/// proxy class for the items() function +template class iteration_proxy +{ + private: + /// the container to iterate + typename IteratorType::reference container; + + public: + /// construct iteration proxy from a container + explicit iteration_proxy(typename IteratorType::reference cont) noexcept + : container(cont) {} + + /// return iterator begin (needed for range-based for) + iteration_proxy_value begin() noexcept + { + return iteration_proxy_value(container.begin()); + } + + /// return iterator end (needed for range-based for) + iteration_proxy_value end() noexcept + { + return iteration_proxy_value(container.end()); + } +}; +// Structured Bindings Support +// For further reference see https://blog.tartanllama.xyz/structured-bindings/ +// And see https://github.com/nlohmann/json/pull/1391 +template = 0> +auto get(const nlohmann::detail::iteration_proxy_value& i) -> decltype(i.key()) +{ + return i.key(); +} +// Structured Bindings Support +// For further reference see https://blog.tartanllama.xyz/structured-bindings/ +// And see https://github.com/nlohmann/json/pull/1391 +template = 0> +auto get(const nlohmann::detail::iteration_proxy_value& i) -> decltype(i.value()) +{ + return i.value(); +} +} // namespace detail +} // namespace nlohmann + +// The Addition to the STD Namespace is required to add +// Structured Bindings Support to the iteration_proxy_value class +// For further reference see https://blog.tartanllama.xyz/structured-bindings/ +// And see https://github.com/nlohmann/json/pull/1391 +namespace std +{ +#if defined(__clang__) + // Fix: https://github.com/nlohmann/json/issues/1401 + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wmismatched-tags" +#endif +template +class tuple_size<::nlohmann::detail::iteration_proxy_value> + : public std::integral_constant {}; + +template +class tuple_element> +{ + public: + using type = decltype( + get(std::declval < + ::nlohmann::detail::iteration_proxy_value> ())); +}; +#if defined(__clang__) + #pragma clang diagnostic pop +#endif +} // namespace std + +// #include + +// #include + +// #include + + +#if JSON_HAS_EXPERIMENTAL_FILESYSTEM +#include +namespace nlohmann::detail +{ +namespace std_fs = std::experimental::filesystem; +} // namespace nlohmann::detail +#elif JSON_HAS_FILESYSTEM +#include +namespace nlohmann::detail +{ +namespace std_fs = std::filesystem; +} // namespace nlohmann::detail +#endif + +namespace nlohmann +{ +namespace detail +{ +////////////////// +// constructors // +////////////////// + +/* + * Note all external_constructor<>::construct functions need to call + * j.m_value.destroy(j.m_type) to avoid a memory leak in case j contains an + * allocated value (e.g., a string). See bug issue + * https://github.com/nlohmann/json/issues/2865 for more information. + */ + +template struct external_constructor; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::boolean_t b) noexcept + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::boolean; + j.m_value = b; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::string_t& s) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::string; + j.m_value = s; + j.assert_invariant(); + } + + template + static void construct(BasicJsonType& j, typename BasicJsonType::string_t&& s) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::string; + j.m_value = std::move(s); + j.assert_invariant(); + } + + template < typename BasicJsonType, typename CompatibleStringType, + enable_if_t < !std::is_same::value, + int > = 0 > + static void construct(BasicJsonType& j, const CompatibleStringType& str) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::string; + j.m_value.string = j.template create(str); + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::binary_t& b) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::binary; + j.m_value = typename BasicJsonType::binary_t(b); + j.assert_invariant(); + } + + template + static void construct(BasicJsonType& j, typename BasicJsonType::binary_t&& b) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::binary; + j.m_value = typename BasicJsonType::binary_t(std::move(b)); + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::number_float_t val) noexcept + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::number_float; + j.m_value = val; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::number_unsigned_t val) noexcept + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::number_unsigned; + j.m_value = val; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::number_integer_t val) noexcept + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::number_integer; + j.m_value = val; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::array_t& arr) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::array; + j.m_value = arr; + j.set_parents(); + j.assert_invariant(); + } + + template + static void construct(BasicJsonType& j, typename BasicJsonType::array_t&& arr) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::array; + j.m_value = std::move(arr); + j.set_parents(); + j.assert_invariant(); + } + + template < typename BasicJsonType, typename CompatibleArrayType, + enable_if_t < !std::is_same::value, + int > = 0 > + static void construct(BasicJsonType& j, const CompatibleArrayType& arr) + { + using std::begin; + using std::end; + + j.m_value.destroy(j.m_type); + j.m_type = value_t::array; + j.m_value.array = j.template create(begin(arr), end(arr)); + j.set_parents(); + j.assert_invariant(); + } + + template + static void construct(BasicJsonType& j, const std::vector& arr) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::array; + j.m_value = value_t::array; + j.m_value.array->reserve(arr.size()); + for (const bool x : arr) + { + j.m_value.array->push_back(x); + j.set_parent(j.m_value.array->back()); + } + j.assert_invariant(); + } + + template::value, int> = 0> + static void construct(BasicJsonType& j, const std::valarray& arr) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::array; + j.m_value = value_t::array; + j.m_value.array->resize(arr.size()); + if (arr.size() > 0) + { + std::copy(std::begin(arr), std::end(arr), j.m_value.array->begin()); + } + j.set_parents(); + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::object_t& obj) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::object; + j.m_value = obj; + j.set_parents(); + j.assert_invariant(); + } + + template + static void construct(BasicJsonType& j, typename BasicJsonType::object_t&& obj) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::object; + j.m_value = std::move(obj); + j.set_parents(); + j.assert_invariant(); + } + + template < typename BasicJsonType, typename CompatibleObjectType, + enable_if_t < !std::is_same::value, int > = 0 > + static void construct(BasicJsonType& j, const CompatibleObjectType& obj) + { + using std::begin; + using std::end; + + j.m_value.destroy(j.m_type); + j.m_type = value_t::object; + j.m_value.object = j.template create(begin(obj), end(obj)); + j.set_parents(); + j.assert_invariant(); + } +}; + +///////////// +// to_json // +///////////// + +template::value, int> = 0> +void to_json(BasicJsonType& j, T b) noexcept +{ + external_constructor::construct(j, b); +} + +template::value, int> = 0> +void to_json(BasicJsonType& j, const CompatibleString& s) +{ + external_constructor::construct(j, s); +} + +template +void to_json(BasicJsonType& j, typename BasicJsonType::string_t&& s) +{ + external_constructor::construct(j, std::move(s)); +} + +template::value, int> = 0> +void to_json(BasicJsonType& j, FloatType val) noexcept +{ + external_constructor::construct(j, static_cast(val)); +} + +template::value, int> = 0> +void to_json(BasicJsonType& j, CompatibleNumberUnsignedType val) noexcept +{ + external_constructor::construct(j, static_cast(val)); +} + +template::value, int> = 0> +void to_json(BasicJsonType& j, CompatibleNumberIntegerType val) noexcept +{ + external_constructor::construct(j, static_cast(val)); +} + +template::value, int> = 0> +void to_json(BasicJsonType& j, EnumType e) noexcept +{ + using underlying_type = typename std::underlying_type::type; + external_constructor::construct(j, static_cast(e)); +} + +template +void to_json(BasicJsonType& j, const std::vector& e) +{ + external_constructor::construct(j, e); +} + +template < typename BasicJsonType, typename CompatibleArrayType, + enable_if_t < is_compatible_array_type::value&& + !is_compatible_object_type::value&& + !is_compatible_string_type::value&& + !std::is_same::value&& + !is_basic_json::value, + int > = 0 > +void to_json(BasicJsonType& j, const CompatibleArrayType& arr) +{ + external_constructor::construct(j, arr); +} + +template +void to_json(BasicJsonType& j, const typename BasicJsonType::binary_t& bin) +{ + external_constructor::construct(j, bin); +} + +template::value, int> = 0> +void to_json(BasicJsonType& j, const std::valarray& arr) +{ + external_constructor::construct(j, std::move(arr)); +} + +template +void to_json(BasicJsonType& j, typename BasicJsonType::array_t&& arr) +{ + external_constructor::construct(j, std::move(arr)); +} + +template < typename BasicJsonType, typename CompatibleObjectType, + enable_if_t < is_compatible_object_type::value&& !is_basic_json::value, int > = 0 > +void to_json(BasicJsonType& j, const CompatibleObjectType& obj) +{ + external_constructor::construct(j, obj); +} + +template +void to_json(BasicJsonType& j, typename BasicJsonType::object_t&& obj) +{ + external_constructor::construct(j, std::move(obj)); +} + +template < + typename BasicJsonType, typename T, std::size_t N, + enable_if_t < !std::is_constructible::value, // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + int > = 0 > +void to_json(BasicJsonType& j, const T(&arr)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) +{ + external_constructor::construct(j, arr); +} + +template < typename BasicJsonType, typename T1, typename T2, enable_if_t < std::is_constructible::value&& std::is_constructible::value, int > = 0 > +void to_json(BasicJsonType& j, const std::pair& p) +{ + j = { p.first, p.second }; +} + +// for https://github.com/nlohmann/json/pull/1134 +template>::value, int> = 0> +void to_json(BasicJsonType& j, const T& b) +{ + j = { {b.key(), b.value()} }; +} + +template +void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence /*unused*/) +{ + j = { std::get(t)... }; +} + +template::value, int > = 0> +void to_json(BasicJsonType& j, const T& t) +{ + to_json_tuple_impl(j, t, make_index_sequence::value> {}); +} + +#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM +template +void to_json(BasicJsonType& j, const std_fs::path& p) +{ + j = p.string(); +} +#endif + +struct to_json_fn +{ + template + auto operator()(BasicJsonType& j, T&& val) const noexcept(noexcept(to_json(j, std::forward(val)))) + -> decltype(to_json(j, std::forward(val)), void()) + { + return to_json(j, std::forward(val)); + } +}; +} // namespace detail + +/// namespace to hold default `to_json` function +/// to see why this is required: +/// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/n4381.html +namespace // NOLINT(cert-dcl59-cpp,fuchsia-header-anon-namespaces,google-build-namespaces) +{ +constexpr const auto& to_json = detail::static_const::value; // NOLINT(misc-definitions-in-headers) +} // namespace +} // namespace nlohmann + +// #include + +// #include + + +namespace nlohmann +{ + +/// @sa https://json.nlohmann.me/api/adl_serializer/ +template +struct adl_serializer +{ + /// @brief convert a JSON value to any value type + /// @sa https://json.nlohmann.me/api/adl_serializer/from_json/ + template + static auto from_json(BasicJsonType && j, TargetType& val) noexcept( + noexcept(::nlohmann::from_json(std::forward(j), val))) + -> decltype(::nlohmann::from_json(std::forward(j), val), void()) + { + ::nlohmann::from_json(std::forward(j), val); + } + + /// @brief convert a JSON value to any value type + /// @sa https://json.nlohmann.me/api/adl_serializer/from_json/ + template + static auto from_json(BasicJsonType && j) noexcept( + noexcept(::nlohmann::from_json(std::forward(j), detail::identity_tag {}))) + -> decltype(::nlohmann::from_json(std::forward(j), detail::identity_tag {})) + { + return ::nlohmann::from_json(std::forward(j), detail::identity_tag {}); + } + + /// @brief convert any value type to a JSON value + /// @sa https://json.nlohmann.me/api/adl_serializer/to_json/ + template + static auto to_json(BasicJsonType& j, TargetType && val) noexcept( + noexcept(::nlohmann::to_json(j, std::forward(val)))) + -> decltype(::nlohmann::to_json(j, std::forward(val)), void()) + { + ::nlohmann::to_json(j, std::forward(val)); + } +}; +} // namespace nlohmann + +// #include + + +#include // uint8_t, uint64_t +#include // tie +#include // move + +namespace nlohmann +{ + +/// @brief an internal type for a backed binary type +/// @sa https://json.nlohmann.me/api/byte_container_with_subtype/ +template +class byte_container_with_subtype : public BinaryType +{ + public: + using container_type = BinaryType; + using subtype_type = std::uint64_t; + + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/ + byte_container_with_subtype() noexcept(noexcept(container_type())) + : container_type() + {} + + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/ + byte_container_with_subtype(const container_type& b) noexcept(noexcept(container_type(b))) + : container_type(b) + {} + + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/ + byte_container_with_subtype(container_type&& b) noexcept(noexcept(container_type(std::move(b)))) + : container_type(std::move(b)) + {} + + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/ + byte_container_with_subtype(const container_type& b, subtype_type subtype_) noexcept(noexcept(container_type(b))) + : container_type(b) + , m_subtype(subtype_) + , m_has_subtype(true) + {} + + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/ + byte_container_with_subtype(container_type&& b, subtype_type subtype_) noexcept(noexcept(container_type(std::move(b)))) + : container_type(std::move(b)) + , m_subtype(subtype_) + , m_has_subtype(true) + {} + + bool operator==(const byte_container_with_subtype& rhs) const + { + return std::tie(static_cast(*this), m_subtype, m_has_subtype) == + std::tie(static_cast(rhs), rhs.m_subtype, rhs.m_has_subtype); + } + + bool operator!=(const byte_container_with_subtype& rhs) const + { + return !(rhs == *this); + } + + /// @brief sets the binary subtype + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/set_subtype/ + void set_subtype(subtype_type subtype_) noexcept + { + m_subtype = subtype_; + m_has_subtype = true; + } + + /// @brief return the binary subtype + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/subtype/ + constexpr subtype_type subtype() const noexcept + { + return m_has_subtype ? m_subtype : static_cast(-1); + } + + /// @brief return whether the value has a subtype + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/has_subtype/ + constexpr bool has_subtype() const noexcept + { + return m_has_subtype; + } + + /// @brief clears the binary subtype + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/clear_subtype/ + void clear_subtype() noexcept + { + m_subtype = 0; + m_has_subtype = false; + } + + private: + subtype_type m_subtype = 0; + bool m_has_subtype = false; +}; + +} // namespace nlohmann + +// #include + +// #include + +// #include + +// #include + + +#include // uint8_t +#include // size_t +#include // hash + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ + +// boost::hash_combine +inline std::size_t combine(std::size_t seed, std::size_t h) noexcept +{ + seed ^= h + 0x9e3779b9 + (seed << 6U) + (seed >> 2U); + return seed; +} + +/*! +@brief hash a JSON value + +The hash function tries to rely on std::hash where possible. Furthermore, the +type of the JSON value is taken into account to have different hash values for +null, 0, 0U, and false, etc. + +@tparam BasicJsonType basic_json specialization +@param j JSON value to hash +@return hash value of j +*/ +template +std::size_t hash(const BasicJsonType& j) +{ + using string_t = typename BasicJsonType::string_t; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + + const auto type = static_cast(j.type()); + switch (j.type()) + { + case BasicJsonType::value_t::null: + case BasicJsonType::value_t::discarded: + { + return combine(type, 0); + } + + case BasicJsonType::value_t::object: + { + auto seed = combine(type, j.size()); + for (const auto& element : j.items()) + { + const auto h = std::hash {}(element.key()); + seed = combine(seed, h); + seed = combine(seed, hash(element.value())); + } + return seed; + } + + case BasicJsonType::value_t::array: + { + auto seed = combine(type, j.size()); + for (const auto& element : j) + { + seed = combine(seed, hash(element)); + } + return seed; + } + + case BasicJsonType::value_t::string: + { + const auto h = std::hash {}(j.template get_ref()); + return combine(type, h); + } + + case BasicJsonType::value_t::boolean: + { + const auto h = std::hash {}(j.template get()); + return combine(type, h); + } + + case BasicJsonType::value_t::number_integer: + { + const auto h = std::hash {}(j.template get()); + return combine(type, h); + } + + case BasicJsonType::value_t::number_unsigned: + { + const auto h = std::hash {}(j.template get()); + return combine(type, h); + } + + case BasicJsonType::value_t::number_float: + { + const auto h = std::hash {}(j.template get()); + return combine(type, h); + } + + case BasicJsonType::value_t::binary: + { + auto seed = combine(type, j.get_binary().size()); + const auto h = std::hash {}(j.get_binary().has_subtype()); + seed = combine(seed, h); + seed = combine(seed, static_cast(j.get_binary().subtype())); + for (const auto byte : j.get_binary()) + { + seed = combine(seed, std::hash {}(byte)); + } + return seed; + } + + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + return 0; // LCOV_EXCL_LINE + } +} + +} // namespace detail +} // namespace nlohmann + +// #include + + +#include // generate_n +#include // array +#include // ldexp +#include // size_t +#include // uint8_t, uint16_t, uint32_t, uint64_t +#include // snprintf +#include // memcpy +#include // back_inserter +#include // numeric_limits +#include // char_traits, string +#include // make_pair, move +#include // vector + +// #include + +// #include + + +#include // array +#include // size_t +#include // strlen +#include // begin, end, iterator_traits, random_access_iterator_tag, distance, next +#include // shared_ptr, make_shared, addressof +#include // accumulate +#include // string, char_traits +#include // enable_if, is_base_of, is_pointer, is_integral, remove_pointer +#include // pair, declval + +#ifndef JSON_NO_IO + #include // FILE * + #include // istream +#endif // JSON_NO_IO + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +/// the supported input formats +enum class input_format_t { json, cbor, msgpack, ubjson, bson }; + +//////////////////// +// input adapters // +//////////////////// + +#ifndef JSON_NO_IO +/*! +Input adapter for stdio file access. This adapter read only 1 byte and do not use any + buffer. This adapter is a very low level adapter. +*/ +class file_input_adapter +{ + public: + using char_type = char; + + JSON_HEDLEY_NON_NULL(2) + explicit file_input_adapter(std::FILE* f) noexcept + : m_file(f) + {} + + // make class move-only + file_input_adapter(const file_input_adapter&) = delete; + file_input_adapter(file_input_adapter&&) noexcept = default; + file_input_adapter& operator=(const file_input_adapter&) = delete; + file_input_adapter& operator=(file_input_adapter&&) = delete; + ~file_input_adapter() = default; + + std::char_traits::int_type get_character() noexcept + { + return std::fgetc(m_file); + } + + private: + /// the file pointer to read from + std::FILE* m_file; +}; + + +/*! +Input adapter for a (caching) istream. Ignores a UFT Byte Order Mark at +beginning of input. Does not support changing the underlying std::streambuf +in mid-input. Maintains underlying std::istream and std::streambuf to support +subsequent use of standard std::istream operations to process any input +characters following those used in parsing the JSON input. Clears the +std::istream flags; any input errors (e.g., EOF) will be detected by the first +subsequent call for input from the std::istream. +*/ +class input_stream_adapter +{ + public: + using char_type = char; + + ~input_stream_adapter() + { + // clear stream flags; we use underlying streambuf I/O, do not + // maintain ifstream flags, except eof + if (is != nullptr) + { + is->clear(is->rdstate() & std::ios::eofbit); + } + } + + explicit input_stream_adapter(std::istream& i) + : is(&i), sb(i.rdbuf()) + {} + + // delete because of pointer members + input_stream_adapter(const input_stream_adapter&) = delete; + input_stream_adapter& operator=(input_stream_adapter&) = delete; + input_stream_adapter& operator=(input_stream_adapter&&) = delete; + + input_stream_adapter(input_stream_adapter&& rhs) noexcept + : is(rhs.is), sb(rhs.sb) + { + rhs.is = nullptr; + rhs.sb = nullptr; + } + + // std::istream/std::streambuf use std::char_traits::to_int_type, to + // ensure that std::char_traits::eof() and the character 0xFF do not + // end up as the same value, e.g. 0xFFFFFFFF. + std::char_traits::int_type get_character() + { + auto res = sb->sbumpc(); + // set eof manually, as we don't use the istream interface. + if (JSON_HEDLEY_UNLIKELY(res == std::char_traits::eof())) + { + is->clear(is->rdstate() | std::ios::eofbit); + } + return res; + } + + private: + /// the associated input stream + std::istream* is = nullptr; + std::streambuf* sb = nullptr; +}; +#endif // JSON_NO_IO + +// General-purpose iterator-based adapter. It might not be as fast as +// theoretically possible for some containers, but it is extremely versatile. +template +class iterator_input_adapter +{ + public: + using char_type = typename std::iterator_traits::value_type; + + iterator_input_adapter(IteratorType first, IteratorType last) + : current(std::move(first)), end(std::move(last)) + {} + + typename std::char_traits::int_type get_character() + { + if (JSON_HEDLEY_LIKELY(current != end)) + { + auto result = std::char_traits::to_int_type(*current); + std::advance(current, 1); + return result; + } + + return std::char_traits::eof(); + } + + private: + IteratorType current; + IteratorType end; + + template + friend struct wide_string_input_helper; + + bool empty() const + { + return current == end; + } +}; + + +template +struct wide_string_input_helper; + +template +struct wide_string_input_helper +{ + // UTF-32 + static void fill_buffer(BaseInputAdapter& input, + std::array::int_type, 4>& utf8_bytes, + size_t& utf8_bytes_index, + size_t& utf8_bytes_filled) + { + utf8_bytes_index = 0; + + if (JSON_HEDLEY_UNLIKELY(input.empty())) + { + utf8_bytes[0] = std::char_traits::eof(); + utf8_bytes_filled = 1; + } + else + { + // get the current character + const auto wc = input.get_character(); + + // UTF-32 to UTF-8 encoding + if (wc < 0x80) + { + utf8_bytes[0] = static_cast::int_type>(wc); + utf8_bytes_filled = 1; + } + else if (wc <= 0x7FF) + { + utf8_bytes[0] = static_cast::int_type>(0xC0u | ((static_cast(wc) >> 6u) & 0x1Fu)); + utf8_bytes[1] = static_cast::int_type>(0x80u | (static_cast(wc) & 0x3Fu)); + utf8_bytes_filled = 2; + } + else if (wc <= 0xFFFF) + { + utf8_bytes[0] = static_cast::int_type>(0xE0u | ((static_cast(wc) >> 12u) & 0x0Fu)); + utf8_bytes[1] = static_cast::int_type>(0x80u | ((static_cast(wc) >> 6u) & 0x3Fu)); + utf8_bytes[2] = static_cast::int_type>(0x80u | (static_cast(wc) & 0x3Fu)); + utf8_bytes_filled = 3; + } + else if (wc <= 0x10FFFF) + { + utf8_bytes[0] = static_cast::int_type>(0xF0u | ((static_cast(wc) >> 18u) & 0x07u)); + utf8_bytes[1] = static_cast::int_type>(0x80u | ((static_cast(wc) >> 12u) & 0x3Fu)); + utf8_bytes[2] = static_cast::int_type>(0x80u | ((static_cast(wc) >> 6u) & 0x3Fu)); + utf8_bytes[3] = static_cast::int_type>(0x80u | (static_cast(wc) & 0x3Fu)); + utf8_bytes_filled = 4; + } + else + { + // unknown character + utf8_bytes[0] = static_cast::int_type>(wc); + utf8_bytes_filled = 1; + } + } + } +}; + +template +struct wide_string_input_helper +{ + // UTF-16 + static void fill_buffer(BaseInputAdapter& input, + std::array::int_type, 4>& utf8_bytes, + size_t& utf8_bytes_index, + size_t& utf8_bytes_filled) + { + utf8_bytes_index = 0; + + if (JSON_HEDLEY_UNLIKELY(input.empty())) + { + utf8_bytes[0] = std::char_traits::eof(); + utf8_bytes_filled = 1; + } + else + { + // get the current character + const auto wc = input.get_character(); + + // UTF-16 to UTF-8 encoding + if (wc < 0x80) + { + utf8_bytes[0] = static_cast::int_type>(wc); + utf8_bytes_filled = 1; + } + else if (wc <= 0x7FF) + { + utf8_bytes[0] = static_cast::int_type>(0xC0u | ((static_cast(wc) >> 6u))); + utf8_bytes[1] = static_cast::int_type>(0x80u | (static_cast(wc) & 0x3Fu)); + utf8_bytes_filled = 2; + } + else if (0xD800 > wc || wc >= 0xE000) + { + utf8_bytes[0] = static_cast::int_type>(0xE0u | ((static_cast(wc) >> 12u))); + utf8_bytes[1] = static_cast::int_type>(0x80u | ((static_cast(wc) >> 6u) & 0x3Fu)); + utf8_bytes[2] = static_cast::int_type>(0x80u | (static_cast(wc) & 0x3Fu)); + utf8_bytes_filled = 3; + } + else + { + if (JSON_HEDLEY_UNLIKELY(!input.empty())) + { + const auto wc2 = static_cast(input.get_character()); + const auto charcode = 0x10000u + (((static_cast(wc) & 0x3FFu) << 10u) | (wc2 & 0x3FFu)); + utf8_bytes[0] = static_cast::int_type>(0xF0u | (charcode >> 18u)); + utf8_bytes[1] = static_cast::int_type>(0x80u | ((charcode >> 12u) & 0x3Fu)); + utf8_bytes[2] = static_cast::int_type>(0x80u | ((charcode >> 6u) & 0x3Fu)); + utf8_bytes[3] = static_cast::int_type>(0x80u | (charcode & 0x3Fu)); + utf8_bytes_filled = 4; + } + else + { + utf8_bytes[0] = static_cast::int_type>(wc); + utf8_bytes_filled = 1; + } + } + } + } +}; + +// Wraps another input apdater to convert wide character types into individual bytes. +template +class wide_string_input_adapter +{ + public: + using char_type = char; + + wide_string_input_adapter(BaseInputAdapter base) + : base_adapter(base) {} + + typename std::char_traits::int_type get_character() noexcept + { + // check if buffer needs to be filled + if (utf8_bytes_index == utf8_bytes_filled) + { + fill_buffer(); + + JSON_ASSERT(utf8_bytes_filled > 0); + JSON_ASSERT(utf8_bytes_index == 0); + } + + // use buffer + JSON_ASSERT(utf8_bytes_filled > 0); + JSON_ASSERT(utf8_bytes_index < utf8_bytes_filled); + return utf8_bytes[utf8_bytes_index++]; + } + + private: + BaseInputAdapter base_adapter; + + template + void fill_buffer() + { + wide_string_input_helper::fill_buffer(base_adapter, utf8_bytes, utf8_bytes_index, utf8_bytes_filled); + } + + /// a buffer for UTF-8 bytes + std::array::int_type, 4> utf8_bytes = {{0, 0, 0, 0}}; + + /// index to the utf8_codes array for the next valid byte + std::size_t utf8_bytes_index = 0; + /// number of valid bytes in the utf8_codes array + std::size_t utf8_bytes_filled = 0; +}; + + +template +struct iterator_input_adapter_factory +{ + using iterator_type = IteratorType; + using char_type = typename std::iterator_traits::value_type; + using adapter_type = iterator_input_adapter; + + static adapter_type create(IteratorType first, IteratorType last) + { + return adapter_type(std::move(first), std::move(last)); + } +}; + +template +struct is_iterator_of_multibyte +{ + using value_type = typename std::iterator_traits::value_type; + enum + { + value = sizeof(value_type) > 1 + }; +}; + +template +struct iterator_input_adapter_factory::value>> +{ + using iterator_type = IteratorType; + using char_type = typename std::iterator_traits::value_type; + using base_adapter_type = iterator_input_adapter; + using adapter_type = wide_string_input_adapter; + + static adapter_type create(IteratorType first, IteratorType last) + { + return adapter_type(base_adapter_type(std::move(first), std::move(last))); + } +}; + +// General purpose iterator-based input +template +typename iterator_input_adapter_factory::adapter_type input_adapter(IteratorType first, IteratorType last) +{ + using factory_type = iterator_input_adapter_factory; + return factory_type::create(first, last); +} + +// Convenience shorthand from container to iterator +// Enables ADL on begin(container) and end(container) +// Encloses the using declarations in namespace for not to leak them to outside scope + +namespace container_input_adapter_factory_impl +{ + +using std::begin; +using std::end; + +template +struct container_input_adapter_factory {}; + +template +struct container_input_adapter_factory< ContainerType, + void_t()), end(std::declval()))>> + { + using adapter_type = decltype(input_adapter(begin(std::declval()), end(std::declval()))); + + static adapter_type create(const ContainerType& container) +{ + return input_adapter(begin(container), end(container)); +} + }; + +} // namespace container_input_adapter_factory_impl + +template +typename container_input_adapter_factory_impl::container_input_adapter_factory::adapter_type input_adapter(const ContainerType& container) +{ + return container_input_adapter_factory_impl::container_input_adapter_factory::create(container); +} + +#ifndef JSON_NO_IO +// Special cases with fast paths +inline file_input_adapter input_adapter(std::FILE* file) +{ + return file_input_adapter(file); +} + +inline input_stream_adapter input_adapter(std::istream& stream) +{ + return input_stream_adapter(stream); +} + +inline input_stream_adapter input_adapter(std::istream&& stream) +{ + return input_stream_adapter(stream); +} +#endif // JSON_NO_IO + +using contiguous_bytes_input_adapter = decltype(input_adapter(std::declval(), std::declval())); + +// Null-delimited strings, and the like. +template < typename CharT, + typename std::enable_if < + std::is_pointer::value&& + !std::is_array::value&& + std::is_integral::type>::value&& + sizeof(typename std::remove_pointer::type) == 1, + int >::type = 0 > +contiguous_bytes_input_adapter input_adapter(CharT b) +{ + auto length = std::strlen(reinterpret_cast(b)); + const auto* ptr = reinterpret_cast(b); + return input_adapter(ptr, ptr + length); +} + +template +auto input_adapter(T (&array)[N]) -> decltype(input_adapter(array, array + N)) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) +{ + return input_adapter(array, array + N); +} + +// This class only handles inputs of input_buffer_adapter type. +// It's required so that expressions like {ptr, len} can be implicitly cast +// to the correct adapter. +class span_input_adapter +{ + public: + template < typename CharT, + typename std::enable_if < + std::is_pointer::value&& + std::is_integral::type>::value&& + sizeof(typename std::remove_pointer::type) == 1, + int >::type = 0 > + span_input_adapter(CharT b, std::size_t l) + : ia(reinterpret_cast(b), reinterpret_cast(b) + l) {} + + template::iterator_category, std::random_access_iterator_tag>::value, + int>::type = 0> + span_input_adapter(IteratorType first, IteratorType last) + : ia(input_adapter(first, last)) {} + + contiguous_bytes_input_adapter&& get() + { + return std::move(ia); // NOLINT(hicpp-move-const-arg,performance-move-const-arg) + } + + private: + contiguous_bytes_input_adapter ia; +}; +} // namespace detail +} // namespace nlohmann + +// #include + + +#include +#include // string +#include // move +#include // vector + +// #include + +// #include + + +namespace nlohmann +{ + +/*! +@brief SAX interface + +This class describes the SAX interface used by @ref nlohmann::json::sax_parse. +Each function is called in different situations while the input is parsed. The +boolean return value informs the parser whether to continue processing the +input. +*/ +template +struct json_sax +{ + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + + /*! + @brief a null value was read + @return whether parsing should proceed + */ + virtual bool null() = 0; + + /*! + @brief a boolean value was read + @param[in] val boolean value + @return whether parsing should proceed + */ + virtual bool boolean(bool val) = 0; + + /*! + @brief an integer number was read + @param[in] val integer value + @return whether parsing should proceed + */ + virtual bool number_integer(number_integer_t val) = 0; + + /*! + @brief an unsigned integer number was read + @param[in] val unsigned integer value + @return whether parsing should proceed + */ + virtual bool number_unsigned(number_unsigned_t val) = 0; + + /*! + @brief a floating-point number was read + @param[in] val floating-point value + @param[in] s raw token value + @return whether parsing should proceed + */ + virtual bool number_float(number_float_t val, const string_t& s) = 0; + + /*! + @brief a string value was read + @param[in] val string value + @return whether parsing should proceed + @note It is safe to move the passed string value. + */ + virtual bool string(string_t& val) = 0; + + /*! + @brief a binary value was read + @param[in] val binary value + @return whether parsing should proceed + @note It is safe to move the passed binary value. + */ + virtual bool binary(binary_t& val) = 0; + + /*! + @brief the beginning of an object was read + @param[in] elements number of object elements or -1 if unknown + @return whether parsing should proceed + @note binary formats may report the number of elements + */ + virtual bool start_object(std::size_t elements) = 0; + + /*! + @brief an object key was read + @param[in] val object key + @return whether parsing should proceed + @note It is safe to move the passed string. + */ + virtual bool key(string_t& val) = 0; + + /*! + @brief the end of an object was read + @return whether parsing should proceed + */ + virtual bool end_object() = 0; + + /*! + @brief the beginning of an array was read + @param[in] elements number of array elements or -1 if unknown + @return whether parsing should proceed + @note binary formats may report the number of elements + */ + virtual bool start_array(std::size_t elements) = 0; + + /*! + @brief the end of an array was read + @return whether parsing should proceed + */ + virtual bool end_array() = 0; + + /*! + @brief a parse error occurred + @param[in] position the position in the input where the error occurs + @param[in] last_token the last read token + @param[in] ex an exception object describing the error + @return whether parsing should proceed (must return false) + */ + virtual bool parse_error(std::size_t position, + const std::string& last_token, + const detail::exception& ex) = 0; + + json_sax() = default; + json_sax(const json_sax&) = default; + json_sax(json_sax&&) noexcept = default; + json_sax& operator=(const json_sax&) = default; + json_sax& operator=(json_sax&&) noexcept = default; + virtual ~json_sax() = default; +}; + + +namespace detail +{ +/*! +@brief SAX implementation to create a JSON value from SAX events + +This class implements the @ref json_sax interface and processes the SAX events +to create a JSON value which makes it basically a DOM parser. The structure or +hierarchy of the JSON value is managed by the stack `ref_stack` which contains +a pointer to the respective array or object for each recursion depth. + +After successful parsing, the value that is passed by reference to the +constructor contains the parsed value. + +@tparam BasicJsonType the JSON type +*/ +template +class json_sax_dom_parser +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + + /*! + @param[in,out] r reference to a JSON value that is manipulated while + parsing + @param[in] allow_exceptions_ whether parse errors yield exceptions + */ + explicit json_sax_dom_parser(BasicJsonType& r, const bool allow_exceptions_ = true) + : root(r), allow_exceptions(allow_exceptions_) + {} + + // make class move-only + json_sax_dom_parser(const json_sax_dom_parser&) = delete; + json_sax_dom_parser(json_sax_dom_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + json_sax_dom_parser& operator=(const json_sax_dom_parser&) = delete; + json_sax_dom_parser& operator=(json_sax_dom_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + ~json_sax_dom_parser() = default; + + bool null() + { + handle_value(nullptr); + return true; + } + + bool boolean(bool val) + { + handle_value(val); + return true; + } + + bool number_integer(number_integer_t val) + { + handle_value(val); + return true; + } + + bool number_unsigned(number_unsigned_t val) + { + handle_value(val); + return true; + } + + bool number_float(number_float_t val, const string_t& /*unused*/) + { + handle_value(val); + return true; + } + + bool string(string_t& val) + { + handle_value(val); + return true; + } + + bool binary(binary_t& val) + { + handle_value(std::move(val)); + return true; + } + + bool start_object(std::size_t len) + { + ref_stack.push_back(handle_value(BasicJsonType::value_t::object)); + + if (JSON_HEDLEY_UNLIKELY(len != static_cast(-1) && len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, "excessive object size: " + std::to_string(len), *ref_stack.back())); + } + + return true; + } + + bool key(string_t& val) + { + // add null at given key and store the reference for later + object_element = &(ref_stack.back()->m_value.object->operator[](val)); + return true; + } + + bool end_object() + { + ref_stack.back()->set_parents(); + ref_stack.pop_back(); + return true; + } + + bool start_array(std::size_t len) + { + ref_stack.push_back(handle_value(BasicJsonType::value_t::array)); + + if (JSON_HEDLEY_UNLIKELY(len != static_cast(-1) && len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, "excessive array size: " + std::to_string(len), *ref_stack.back())); + } + + return true; + } + + bool end_array() + { + ref_stack.back()->set_parents(); + ref_stack.pop_back(); + return true; + } + + template + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, + const Exception& ex) + { + errored = true; + static_cast(ex); + if (allow_exceptions) + { + JSON_THROW(ex); + } + return false; + } + + constexpr bool is_errored() const + { + return errored; + } + + private: + /*! + @invariant If the ref stack is empty, then the passed value will be the new + root. + @invariant If the ref stack contains a value, then it is an array or an + object to which we can add elements + */ + template + JSON_HEDLEY_RETURNS_NON_NULL + BasicJsonType* handle_value(Value&& v) + { + if (ref_stack.empty()) + { + root = BasicJsonType(std::forward(v)); + return &root; + } + + JSON_ASSERT(ref_stack.back()->is_array() || ref_stack.back()->is_object()); + + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->emplace_back(std::forward(v)); + return &(ref_stack.back()->m_value.array->back()); + } + + JSON_ASSERT(ref_stack.back()->is_object()); + JSON_ASSERT(object_element); + *object_element = BasicJsonType(std::forward(v)); + return object_element; + } + + /// the parsed JSON value + BasicJsonType& root; + /// stack to model hierarchy of values + std::vector ref_stack {}; + /// helper to hold the reference for the next object element + BasicJsonType* object_element = nullptr; + /// whether a syntax error occurred + bool errored = false; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; +}; + +template +class json_sax_dom_callback_parser +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + using parser_callback_t = typename BasicJsonType::parser_callback_t; + using parse_event_t = typename BasicJsonType::parse_event_t; + + json_sax_dom_callback_parser(BasicJsonType& r, + const parser_callback_t cb, + const bool allow_exceptions_ = true) + : root(r), callback(cb), allow_exceptions(allow_exceptions_) + { + keep_stack.push_back(true); + } + + // make class move-only + json_sax_dom_callback_parser(const json_sax_dom_callback_parser&) = delete; + json_sax_dom_callback_parser(json_sax_dom_callback_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + json_sax_dom_callback_parser& operator=(const json_sax_dom_callback_parser&) = delete; + json_sax_dom_callback_parser& operator=(json_sax_dom_callback_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + ~json_sax_dom_callback_parser() = default; + + bool null() + { + handle_value(nullptr); + return true; + } + + bool boolean(bool val) + { + handle_value(val); + return true; + } + + bool number_integer(number_integer_t val) + { + handle_value(val); + return true; + } + + bool number_unsigned(number_unsigned_t val) + { + handle_value(val); + return true; + } + + bool number_float(number_float_t val, const string_t& /*unused*/) + { + handle_value(val); + return true; + } + + bool string(string_t& val) + { + handle_value(val); + return true; + } + + bool binary(binary_t& val) + { + handle_value(std::move(val)); + return true; + } + + bool start_object(std::size_t len) + { + // check callback for object start + const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::object_start, discarded); + keep_stack.push_back(keep); + + auto val = handle_value(BasicJsonType::value_t::object, true); + ref_stack.push_back(val.second); + + // check object limit + if (ref_stack.back() && JSON_HEDLEY_UNLIKELY(len != static_cast(-1) && len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, "excessive object size: " + std::to_string(len), *ref_stack.back())); + } + + return true; + } + + bool key(string_t& val) + { + BasicJsonType k = BasicJsonType(val); + + // check callback for key + const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::key, k); + key_keep_stack.push_back(keep); + + // add discarded value at given key and store the reference for later + if (keep && ref_stack.back()) + { + object_element = &(ref_stack.back()->m_value.object->operator[](val) = discarded); + } + + return true; + } + + bool end_object() + { + if (ref_stack.back()) + { + if (!callback(static_cast(ref_stack.size()) - 1, parse_event_t::object_end, *ref_stack.back())) + { + // discard object + *ref_stack.back() = discarded; + } + else + { + ref_stack.back()->set_parents(); + } + } + + JSON_ASSERT(!ref_stack.empty()); + JSON_ASSERT(!keep_stack.empty()); + ref_stack.pop_back(); + keep_stack.pop_back(); + + if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_structured()) + { + // remove discarded value + for (auto it = ref_stack.back()->begin(); it != ref_stack.back()->end(); ++it) + { + if (it->is_discarded()) + { + ref_stack.back()->erase(it); + break; + } + } + } + + return true; + } + + bool start_array(std::size_t len) + { + const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::array_start, discarded); + keep_stack.push_back(keep); + + auto val = handle_value(BasicJsonType::value_t::array, true); + ref_stack.push_back(val.second); + + // check array limit + if (ref_stack.back() && JSON_HEDLEY_UNLIKELY(len != static_cast(-1) && len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, "excessive array size: " + std::to_string(len), *ref_stack.back())); + } + + return true; + } + + bool end_array() + { + bool keep = true; + + if (ref_stack.back()) + { + keep = callback(static_cast(ref_stack.size()) - 1, parse_event_t::array_end, *ref_stack.back()); + if (keep) + { + ref_stack.back()->set_parents(); + } + else + { + // discard array + *ref_stack.back() = discarded; + } + } + + JSON_ASSERT(!ref_stack.empty()); + JSON_ASSERT(!keep_stack.empty()); + ref_stack.pop_back(); + keep_stack.pop_back(); + + // remove discarded value + if (!keep && !ref_stack.empty() && ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->pop_back(); + } + + return true; + } + + template + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, + const Exception& ex) + { + errored = true; + static_cast(ex); + if (allow_exceptions) + { + JSON_THROW(ex); + } + return false; + } + + constexpr bool is_errored() const + { + return errored; + } + + private: + /*! + @param[in] v value to add to the JSON value we build during parsing + @param[in] skip_callback whether we should skip calling the callback + function; this is required after start_array() and + start_object() SAX events, because otherwise we would call the + callback function with an empty array or object, respectively. + + @invariant If the ref stack is empty, then the passed value will be the new + root. + @invariant If the ref stack contains a value, then it is an array or an + object to which we can add elements + + @return pair of boolean (whether value should be kept) and pointer (to the + passed value in the ref_stack hierarchy; nullptr if not kept) + */ + template + std::pair handle_value(Value&& v, const bool skip_callback = false) + { + JSON_ASSERT(!keep_stack.empty()); + + // do not handle this value if we know it would be added to a discarded + // container + if (!keep_stack.back()) + { + return {false, nullptr}; + } + + // create value + auto value = BasicJsonType(std::forward(v)); + + // check callback + const bool keep = skip_callback || callback(static_cast(ref_stack.size()), parse_event_t::value, value); + + // do not handle this value if we just learnt it shall be discarded + if (!keep) + { + return {false, nullptr}; + } + + if (ref_stack.empty()) + { + root = std::move(value); + return {true, &root}; + } + + // skip this value if we already decided to skip the parent + // (https://github.com/nlohmann/json/issues/971#issuecomment-413678360) + if (!ref_stack.back()) + { + return {false, nullptr}; + } + + // we now only expect arrays and objects + JSON_ASSERT(ref_stack.back()->is_array() || ref_stack.back()->is_object()); + + // array + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->emplace_back(std::move(value)); + return {true, &(ref_stack.back()->m_value.array->back())}; + } + + // object + JSON_ASSERT(ref_stack.back()->is_object()); + // check if we should store an element for the current key + JSON_ASSERT(!key_keep_stack.empty()); + const bool store_element = key_keep_stack.back(); + key_keep_stack.pop_back(); + + if (!store_element) + { + return {false, nullptr}; + } + + JSON_ASSERT(object_element); + *object_element = std::move(value); + return {true, object_element}; + } + + /// the parsed JSON value + BasicJsonType& root; + /// stack to model hierarchy of values + std::vector ref_stack {}; + /// stack to manage which values to keep + std::vector keep_stack {}; + /// stack to manage which object keys to keep + std::vector key_keep_stack {}; + /// helper to hold the reference for the next object element + BasicJsonType* object_element = nullptr; + /// whether a syntax error occurred + bool errored = false; + /// callback function + const parser_callback_t callback = nullptr; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; + /// a discarded value for the callback + BasicJsonType discarded = BasicJsonType::value_t::discarded; +}; + +template +class json_sax_acceptor +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + + bool null() + { + return true; + } + + bool boolean(bool /*unused*/) + { + return true; + } + + bool number_integer(number_integer_t /*unused*/) + { + return true; + } + + bool number_unsigned(number_unsigned_t /*unused*/) + { + return true; + } + + bool number_float(number_float_t /*unused*/, const string_t& /*unused*/) + { + return true; + } + + bool string(string_t& /*unused*/) + { + return true; + } + + bool binary(binary_t& /*unused*/) + { + return true; + } + + bool start_object(std::size_t /*unused*/ = static_cast(-1)) + { + return true; + } + + bool key(string_t& /*unused*/) + { + return true; + } + + bool end_object() + { + return true; + } + + bool start_array(std::size_t /*unused*/ = static_cast(-1)) + { + return true; + } + + bool end_array() + { + return true; + } + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const detail::exception& /*unused*/) + { + return false; + } +}; +} // namespace detail + +} // namespace nlohmann + +// #include + + +#include // array +#include // localeconv +#include // size_t +#include // snprintf +#include // strtof, strtod, strtold, strtoll, strtoull +#include // initializer_list +#include // char_traits, string +#include // move +#include // vector + +// #include + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +/////////// +// lexer // +/////////// + +template +class lexer_base +{ + public: + /// token types for the parser + enum class token_type + { + uninitialized, ///< indicating the scanner is uninitialized + literal_true, ///< the `true` literal + literal_false, ///< the `false` literal + literal_null, ///< the `null` literal + value_string, ///< a string -- use get_string() for actual value + value_unsigned, ///< an unsigned integer -- use get_number_unsigned() for actual value + value_integer, ///< a signed integer -- use get_number_integer() for actual value + value_float, ///< an floating point number -- use get_number_float() for actual value + begin_array, ///< the character for array begin `[` + begin_object, ///< the character for object begin `{` + end_array, ///< the character for array end `]` + end_object, ///< the character for object end `}` + name_separator, ///< the name separator `:` + value_separator, ///< the value separator `,` + parse_error, ///< indicating a parse error + end_of_input, ///< indicating the end of the input buffer + literal_or_value ///< a literal or the begin of a value (only for diagnostics) + }; + + /// return name of values of type token_type (only used for errors) + JSON_HEDLEY_RETURNS_NON_NULL + JSON_HEDLEY_CONST + static const char* token_type_name(const token_type t) noexcept + { + switch (t) + { + case token_type::uninitialized: + return ""; + case token_type::literal_true: + return "true literal"; + case token_type::literal_false: + return "false literal"; + case token_type::literal_null: + return "null literal"; + case token_type::value_string: + return "string literal"; + case token_type::value_unsigned: + case token_type::value_integer: + case token_type::value_float: + return "number literal"; + case token_type::begin_array: + return "'['"; + case token_type::begin_object: + return "'{'"; + case token_type::end_array: + return "']'"; + case token_type::end_object: + return "'}'"; + case token_type::name_separator: + return "':'"; + case token_type::value_separator: + return "','"; + case token_type::parse_error: + return ""; + case token_type::end_of_input: + return "end of input"; + case token_type::literal_or_value: + return "'[', '{', or a literal"; + // LCOV_EXCL_START + default: // catch non-enum values + return "unknown token"; + // LCOV_EXCL_STOP + } + } +}; +/*! +@brief lexical analysis + +This class organizes the lexical analysis during JSON deserialization. +*/ +template +class lexer : public lexer_base +{ + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using char_type = typename InputAdapterType::char_type; + using char_int_type = typename std::char_traits::int_type; + + public: + using token_type = typename lexer_base::token_type; + + explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false) noexcept + : ia(std::move(adapter)) + , ignore_comments(ignore_comments_) + , decimal_point_char(static_cast(get_decimal_point())) + {} + + // delete because of pointer members + lexer(const lexer&) = delete; + lexer(lexer&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + lexer& operator=(lexer&) = delete; + lexer& operator=(lexer&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + ~lexer() = default; + + private: + ///////////////////// + // locales + ///////////////////// + + /// return the locale-dependent decimal point + JSON_HEDLEY_PURE + static char get_decimal_point() noexcept + { + const auto* loc = localeconv(); + JSON_ASSERT(loc != nullptr); + return (loc->decimal_point == nullptr) ? '.' : *(loc->decimal_point); + } + + ///////////////////// + // scan functions + ///////////////////// + + /*! + @brief get codepoint from 4 hex characters following `\u` + + For input "\u c1 c2 c3 c4" the codepoint is: + (c1 * 0x1000) + (c2 * 0x0100) + (c3 * 0x0010) + c4 + = (c1 << 12) + (c2 << 8) + (c3 << 4) + (c4 << 0) + + Furthermore, the possible characters '0'..'9', 'A'..'F', and 'a'..'f' + must be converted to the integers 0x0..0x9, 0xA..0xF, 0xA..0xF, resp. The + conversion is done by subtracting the offset (0x30, 0x37, and 0x57) + between the ASCII value of the character and the desired integer value. + + @return codepoint (0x0000..0xFFFF) or -1 in case of an error (e.g. EOF or + non-hex character) + */ + int get_codepoint() + { + // this function only makes sense after reading `\u` + JSON_ASSERT(current == 'u'); + int codepoint = 0; + + const auto factors = { 12u, 8u, 4u, 0u }; + for (const auto factor : factors) + { + get(); + + if (current >= '0' && current <= '9') + { + codepoint += static_cast((static_cast(current) - 0x30u) << factor); + } + else if (current >= 'A' && current <= 'F') + { + codepoint += static_cast((static_cast(current) - 0x37u) << factor); + } + else if (current >= 'a' && current <= 'f') + { + codepoint += static_cast((static_cast(current) - 0x57u) << factor); + } + else + { + return -1; + } + } + + JSON_ASSERT(0x0000 <= codepoint && codepoint <= 0xFFFF); + return codepoint; + } + + /*! + @brief check if the next byte(s) are inside a given range + + Adds the current byte and, for each passed range, reads a new byte and + checks if it is inside the range. If a violation was detected, set up an + error message and return false. Otherwise, return true. + + @param[in] ranges list of integers; interpreted as list of pairs of + inclusive lower and upper bound, respectively + + @pre The passed list @a ranges must have 2, 4, or 6 elements; that is, + 1, 2, or 3 pairs. This precondition is enforced by an assertion. + + @return true if and only if no range violation was detected + */ + bool next_byte_in_range(std::initializer_list ranges) + { + JSON_ASSERT(ranges.size() == 2 || ranges.size() == 4 || ranges.size() == 6); + add(current); + + for (auto range = ranges.begin(); range != ranges.end(); ++range) + { + get(); + if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) + { + add(current); + } + else + { + error_message = "invalid string: ill-formed UTF-8 byte"; + return false; + } + } + + return true; + } + + /*! + @brief scan a string literal + + This function scans a string according to Sect. 7 of RFC 8259. While + scanning, bytes are escaped and copied into buffer token_buffer. Then the + function returns successfully, token_buffer is *not* null-terminated (as it + may contain \0 bytes), and token_buffer.size() is the number of bytes in the + string. + + @return token_type::value_string if string could be successfully scanned, + token_type::parse_error otherwise + + @note In case of errors, variable error_message contains a textual + description. + */ + token_type scan_string() + { + // reset token_buffer (ignore opening quote) + reset(); + + // we entered the function by reading an open quote + JSON_ASSERT(current == '\"'); + + while (true) + { + // get next character + switch (get()) + { + // end of file while parsing string + case std::char_traits::eof(): + { + error_message = "invalid string: missing closing quote"; + return token_type::parse_error; + } + + // closing quote + case '\"': + { + return token_type::value_string; + } + + // escapes + case '\\': + { + switch (get()) + { + // quotation mark + case '\"': + add('\"'); + break; + // reverse solidus + case '\\': + add('\\'); + break; + // solidus + case '/': + add('/'); + break; + // backspace + case 'b': + add('\b'); + break; + // form feed + case 'f': + add('\f'); + break; + // line feed + case 'n': + add('\n'); + break; + // carriage return + case 'r': + add('\r'); + break; + // tab + case 't': + add('\t'); + break; + + // unicode escapes + case 'u': + { + const int codepoint1 = get_codepoint(); + int codepoint = codepoint1; // start with codepoint1 + + if (JSON_HEDLEY_UNLIKELY(codepoint1 == -1)) + { + error_message = "invalid string: '\\u' must be followed by 4 hex digits"; + return token_type::parse_error; + } + + // check if code point is a high surrogate + if (0xD800 <= codepoint1 && codepoint1 <= 0xDBFF) + { + // expect next \uxxxx entry + if (JSON_HEDLEY_LIKELY(get() == '\\' && get() == 'u')) + { + const int codepoint2 = get_codepoint(); + + if (JSON_HEDLEY_UNLIKELY(codepoint2 == -1)) + { + error_message = "invalid string: '\\u' must be followed by 4 hex digits"; + return token_type::parse_error; + } + + // check if codepoint2 is a low surrogate + if (JSON_HEDLEY_LIKELY(0xDC00 <= codepoint2 && codepoint2 <= 0xDFFF)) + { + // overwrite codepoint + codepoint = static_cast( + // high surrogate occupies the most significant 22 bits + (static_cast(codepoint1) << 10u) + // low surrogate occupies the least significant 15 bits + + static_cast(codepoint2) + // there is still the 0xD800, 0xDC00 and 0x10000 noise + // in the result, so we have to subtract with: + // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00 + - 0x35FDC00u); + } + else + { + error_message = "invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF"; + return token_type::parse_error; + } + } + else + { + error_message = "invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF"; + return token_type::parse_error; + } + } + else + { + if (JSON_HEDLEY_UNLIKELY(0xDC00 <= codepoint1 && codepoint1 <= 0xDFFF)) + { + error_message = "invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF"; + return token_type::parse_error; + } + } + + // result of the above calculation yields a proper codepoint + JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF); + + // translate codepoint into bytes + if (codepoint < 0x80) + { + // 1-byte characters: 0xxxxxxx (ASCII) + add(static_cast(codepoint)); + } + else if (codepoint <= 0x7FF) + { + // 2-byte characters: 110xxxxx 10xxxxxx + add(static_cast(0xC0u | (static_cast(codepoint) >> 6u))); + add(static_cast(0x80u | (static_cast(codepoint) & 0x3Fu))); + } + else if (codepoint <= 0xFFFF) + { + // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx + add(static_cast(0xE0u | (static_cast(codepoint) >> 12u))); + add(static_cast(0x80u | ((static_cast(codepoint) >> 6u) & 0x3Fu))); + add(static_cast(0x80u | (static_cast(codepoint) & 0x3Fu))); + } + else + { + // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + add(static_cast(0xF0u | (static_cast(codepoint) >> 18u))); + add(static_cast(0x80u | ((static_cast(codepoint) >> 12u) & 0x3Fu))); + add(static_cast(0x80u | ((static_cast(codepoint) >> 6u) & 0x3Fu))); + add(static_cast(0x80u | (static_cast(codepoint) & 0x3Fu))); + } + + break; + } + + // other characters after escape + default: + error_message = "invalid string: forbidden character after backslash"; + return token_type::parse_error; + } + + break; + } + + // invalid control characters + case 0x00: + { + error_message = "invalid string: control character U+0000 (NUL) must be escaped to \\u0000"; + return token_type::parse_error; + } + + case 0x01: + { + error_message = "invalid string: control character U+0001 (SOH) must be escaped to \\u0001"; + return token_type::parse_error; + } + + case 0x02: + { + error_message = "invalid string: control character U+0002 (STX) must be escaped to \\u0002"; + return token_type::parse_error; + } + + case 0x03: + { + error_message = "invalid string: control character U+0003 (ETX) must be escaped to \\u0003"; + return token_type::parse_error; + } + + case 0x04: + { + error_message = "invalid string: control character U+0004 (EOT) must be escaped to \\u0004"; + return token_type::parse_error; + } + + case 0x05: + { + error_message = "invalid string: control character U+0005 (ENQ) must be escaped to \\u0005"; + return token_type::parse_error; + } + + case 0x06: + { + error_message = "invalid string: control character U+0006 (ACK) must be escaped to \\u0006"; + return token_type::parse_error; + } + + case 0x07: + { + error_message = "invalid string: control character U+0007 (BEL) must be escaped to \\u0007"; + return token_type::parse_error; + } + + case 0x08: + { + error_message = "invalid string: control character U+0008 (BS) must be escaped to \\u0008 or \\b"; + return token_type::parse_error; + } + + case 0x09: + { + error_message = "invalid string: control character U+0009 (HT) must be escaped to \\u0009 or \\t"; + return token_type::parse_error; + } + + case 0x0A: + { + error_message = "invalid string: control character U+000A (LF) must be escaped to \\u000A or \\n"; + return token_type::parse_error; + } + + case 0x0B: + { + error_message = "invalid string: control character U+000B (VT) must be escaped to \\u000B"; + return token_type::parse_error; + } + + case 0x0C: + { + error_message = "invalid string: control character U+000C (FF) must be escaped to \\u000C or \\f"; + return token_type::parse_error; + } + + case 0x0D: + { + error_message = "invalid string: control character U+000D (CR) must be escaped to \\u000D or \\r"; + return token_type::parse_error; + } + + case 0x0E: + { + error_message = "invalid string: control character U+000E (SO) must be escaped to \\u000E"; + return token_type::parse_error; + } + + case 0x0F: + { + error_message = "invalid string: control character U+000F (SI) must be escaped to \\u000F"; + return token_type::parse_error; + } + + case 0x10: + { + error_message = "invalid string: control character U+0010 (DLE) must be escaped to \\u0010"; + return token_type::parse_error; + } + + case 0x11: + { + error_message = "invalid string: control character U+0011 (DC1) must be escaped to \\u0011"; + return token_type::parse_error; + } + + case 0x12: + { + error_message = "invalid string: control character U+0012 (DC2) must be escaped to \\u0012"; + return token_type::parse_error; + } + + case 0x13: + { + error_message = "invalid string: control character U+0013 (DC3) must be escaped to \\u0013"; + return token_type::parse_error; + } + + case 0x14: + { + error_message = "invalid string: control character U+0014 (DC4) must be escaped to \\u0014"; + return token_type::parse_error; + } + + case 0x15: + { + error_message = "invalid string: control character U+0015 (NAK) must be escaped to \\u0015"; + return token_type::parse_error; + } + + case 0x16: + { + error_message = "invalid string: control character U+0016 (SYN) must be escaped to \\u0016"; + return token_type::parse_error; + } + + case 0x17: + { + error_message = "invalid string: control character U+0017 (ETB) must be escaped to \\u0017"; + return token_type::parse_error; + } + + case 0x18: + { + error_message = "invalid string: control character U+0018 (CAN) must be escaped to \\u0018"; + return token_type::parse_error; + } + + case 0x19: + { + error_message = "invalid string: control character U+0019 (EM) must be escaped to \\u0019"; + return token_type::parse_error; + } + + case 0x1A: + { + error_message = "invalid string: control character U+001A (SUB) must be escaped to \\u001A"; + return token_type::parse_error; + } + + case 0x1B: + { + error_message = "invalid string: control character U+001B (ESC) must be escaped to \\u001B"; + return token_type::parse_error; + } + + case 0x1C: + { + error_message = "invalid string: control character U+001C (FS) must be escaped to \\u001C"; + return token_type::parse_error; + } + + case 0x1D: + { + error_message = "invalid string: control character U+001D (GS) must be escaped to \\u001D"; + return token_type::parse_error; + } + + case 0x1E: + { + error_message = "invalid string: control character U+001E (RS) must be escaped to \\u001E"; + return token_type::parse_error; + } + + case 0x1F: + { + error_message = "invalid string: control character U+001F (US) must be escaped to \\u001F"; + return token_type::parse_error; + } + + // U+0020..U+007F (except U+0022 (quote) and U+005C (backspace)) + case 0x20: + case 0x21: + case 0x23: + case 0x24: + case 0x25: + case 0x26: + case 0x27: + case 0x28: + case 0x29: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: + case 0x30: + case 0x31: + case 0x32: + case 0x33: + case 0x34: + case 0x35: + case 0x36: + case 0x37: + case 0x38: + case 0x39: + case 0x3A: + case 0x3B: + case 0x3C: + case 0x3D: + case 0x3E: + case 0x3F: + case 0x40: + case 0x41: + case 0x42: + case 0x43: + case 0x44: + case 0x45: + case 0x46: + case 0x47: + case 0x48: + case 0x49: + case 0x4A: + case 0x4B: + case 0x4C: + case 0x4D: + case 0x4E: + case 0x4F: + case 0x50: + case 0x51: + case 0x52: + case 0x53: + case 0x54: + case 0x55: + case 0x56: + case 0x57: + case 0x58: + case 0x59: + case 0x5A: + case 0x5B: + case 0x5D: + case 0x5E: + case 0x5F: + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + case 0x78: + case 0x79: + case 0x7A: + case 0x7B: + case 0x7C: + case 0x7D: + case 0x7E: + case 0x7F: + { + add(current); + break; + } + + // U+0080..U+07FF: bytes C2..DF 80..BF + case 0xC2: + case 0xC3: + case 0xC4: + case 0xC5: + case 0xC6: + case 0xC7: + case 0xC8: + case 0xC9: + case 0xCA: + case 0xCB: + case 0xCC: + case 0xCD: + case 0xCE: + case 0xCF: + case 0xD0: + case 0xD1: + case 0xD2: + case 0xD3: + case 0xD4: + case 0xD5: + case 0xD6: + case 0xD7: + case 0xD8: + case 0xD9: + case 0xDA: + case 0xDB: + case 0xDC: + case 0xDD: + case 0xDE: + case 0xDF: + { + if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range({0x80, 0xBF}))) + { + return token_type::parse_error; + } + break; + } + + // U+0800..U+0FFF: bytes E0 A0..BF 80..BF + case 0xE0: + { + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // U+1000..U+CFFF: bytes E1..EC 80..BF 80..BF + // U+E000..U+FFFF: bytes EE..EF 80..BF 80..BF + case 0xE1: + case 0xE2: + case 0xE3: + case 0xE4: + case 0xE5: + case 0xE6: + case 0xE7: + case 0xE8: + case 0xE9: + case 0xEA: + case 0xEB: + case 0xEC: + case 0xEE: + case 0xEF: + { + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // U+D000..U+D7FF: bytes ED 80..9F 80..BF + case 0xED: + { + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x9F, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF + case 0xF0: + { + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF + case 0xF1: + case 0xF2: + case 0xF3: + { + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF + case 0xF4: + { + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // remaining bytes (80..C1 and F5..FF) are ill-formed + default: + { + error_message = "invalid string: ill-formed UTF-8 byte"; + return token_type::parse_error; + } + } + } + } + + /*! + * @brief scan a comment + * @return whether comment could be scanned successfully + */ + bool scan_comment() + { + switch (get()) + { + // single-line comments skip input until a newline or EOF is read + case '/': + { + while (true) + { + switch (get()) + { + case '\n': + case '\r': + case std::char_traits::eof(): + case '\0': + return true; + + default: + break; + } + } + } + + // multi-line comments skip input until */ is read + case '*': + { + while (true) + { + switch (get()) + { + case std::char_traits::eof(): + case '\0': + { + error_message = "invalid comment; missing closing '*/'"; + return false; + } + + case '*': + { + switch (get()) + { + case '/': + return true; + + default: + { + unget(); + continue; + } + } + } + + default: + continue; + } + } + } + + // unexpected character after reading '/' + default: + { + error_message = "invalid comment; expecting '/' or '*' after '/'"; + return false; + } + } + } + + JSON_HEDLEY_NON_NULL(2) + static void strtof(float& f, const char* str, char** endptr) noexcept + { + f = std::strtof(str, endptr); + } + + JSON_HEDLEY_NON_NULL(2) + static void strtof(double& f, const char* str, char** endptr) noexcept + { + f = std::strtod(str, endptr); + } + + JSON_HEDLEY_NON_NULL(2) + static void strtof(long double& f, const char* str, char** endptr) noexcept + { + f = std::strtold(str, endptr); + } + + /*! + @brief scan a number literal + + This function scans a string according to Sect. 6 of RFC 8259. + + The function is realized with a deterministic finite state machine derived + from the grammar described in RFC 8259. Starting in state "init", the + input is read and used to determined the next state. Only state "done" + accepts the number. State "error" is a trap state to model errors. In the + table below, "anything" means any character but the ones listed before. + + state | 0 | 1-9 | e E | + | - | . | anything + ---------|----------|----------|----------|---------|---------|----------|----------- + init | zero | any1 | [error] | [error] | minus | [error] | [error] + minus | zero | any1 | [error] | [error] | [error] | [error] | [error] + zero | done | done | exponent | done | done | decimal1 | done + any1 | any1 | any1 | exponent | done | done | decimal1 | done + decimal1 | decimal2 | decimal2 | [error] | [error] | [error] | [error] | [error] + decimal2 | decimal2 | decimal2 | exponent | done | done | done | done + exponent | any2 | any2 | [error] | sign | sign | [error] | [error] + sign | any2 | any2 | [error] | [error] | [error] | [error] | [error] + any2 | any2 | any2 | done | done | done | done | done + + The state machine is realized with one label per state (prefixed with + "scan_number_") and `goto` statements between them. The state machine + contains cycles, but any cycle can be left when EOF is read. Therefore, + the function is guaranteed to terminate. + + During scanning, the read bytes are stored in token_buffer. This string is + then converted to a signed integer, an unsigned integer, or a + floating-point number. + + @return token_type::value_unsigned, token_type::value_integer, or + token_type::value_float if number could be successfully scanned, + token_type::parse_error otherwise + + @note The scanner is independent of the current locale. Internally, the + locale's decimal point is used instead of `.` to work with the + locale-dependent converters. + */ + token_type scan_number() // lgtm [cpp/use-of-goto] + { + // reset token_buffer to store the number's bytes + reset(); + + // the type of the parsed number; initially set to unsigned; will be + // changed if minus sign, decimal point or exponent is read + token_type number_type = token_type::value_unsigned; + + // state (init): we just found out we need to scan a number + switch (current) + { + case '-': + { + add(current); + goto scan_number_minus; + } + + case '0': + { + add(current); + goto scan_number_zero; + } + + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any1; + } + + // all other characters are rejected outside scan_number() + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + +scan_number_minus: + // state: we just parsed a leading minus sign + number_type = token_type::value_integer; + switch (get()) + { + case '0': + { + add(current); + goto scan_number_zero; + } + + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any1; + } + + default: + { + error_message = "invalid number; expected digit after '-'"; + return token_type::parse_error; + } + } + +scan_number_zero: + // state: we just parse a zero (maybe with a leading minus sign) + switch (get()) + { + case '.': + { + add(decimal_point_char); + goto scan_number_decimal1; + } + + case 'e': + case 'E': + { + add(current); + goto scan_number_exponent; + } + + default: + goto scan_number_done; + } + +scan_number_any1: + // state: we just parsed a number 0-9 (maybe with a leading minus sign) + switch (get()) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any1; + } + + case '.': + { + add(decimal_point_char); + goto scan_number_decimal1; + } + + case 'e': + case 'E': + { + add(current); + goto scan_number_exponent; + } + + default: + goto scan_number_done; + } + +scan_number_decimal1: + // state: we just parsed a decimal point + number_type = token_type::value_float; + switch (get()) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_decimal2; + } + + default: + { + error_message = "invalid number; expected digit after '.'"; + return token_type::parse_error; + } + } + +scan_number_decimal2: + // we just parsed at least one number after a decimal point + switch (get()) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_decimal2; + } + + case 'e': + case 'E': + { + add(current); + goto scan_number_exponent; + } + + default: + goto scan_number_done; + } + +scan_number_exponent: + // we just parsed an exponent + number_type = token_type::value_float; + switch (get()) + { + case '+': + case '-': + { + add(current); + goto scan_number_sign; + } + + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any2; + } + + default: + { + error_message = + "invalid number; expected '+', '-', or digit after exponent"; + return token_type::parse_error; + } + } + +scan_number_sign: + // we just parsed an exponent sign + switch (get()) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any2; + } + + default: + { + error_message = "invalid number; expected digit after exponent sign"; + return token_type::parse_error; + } + } + +scan_number_any2: + // we just parsed a number after the exponent or exponent sign + switch (get()) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any2; + } + + default: + goto scan_number_done; + } + +scan_number_done: + // unget the character after the number (we only read it to know that + // we are done scanning a number) + unget(); + + char* endptr = nullptr; // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + errno = 0; + + // try to parse integers first and fall back to floats + if (number_type == token_type::value_unsigned) + { + const auto x = std::strtoull(token_buffer.data(), &endptr, 10); + + // we checked the number format before + JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size()); + + if (errno == 0) + { + value_unsigned = static_cast(x); + if (value_unsigned == x) + { + return token_type::value_unsigned; + } + } + } + else if (number_type == token_type::value_integer) + { + const auto x = std::strtoll(token_buffer.data(), &endptr, 10); + + // we checked the number format before + JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size()); + + if (errno == 0) + { + value_integer = static_cast(x); + if (value_integer == x) + { + return token_type::value_integer; + } + } + } + + // this code is reached if we parse a floating-point number or if an + // integer conversion above failed + strtof(value_float, token_buffer.data(), &endptr); + + // we checked the number format before + JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size()); + + return token_type::value_float; + } + + /*! + @param[in] literal_text the literal text to expect + @param[in] length the length of the passed literal text + @param[in] return_type the token type to return on success + */ + JSON_HEDLEY_NON_NULL(2) + token_type scan_literal(const char_type* literal_text, const std::size_t length, + token_type return_type) + { + JSON_ASSERT(std::char_traits::to_char_type(current) == literal_text[0]); + for (std::size_t i = 1; i < length; ++i) + { + if (JSON_HEDLEY_UNLIKELY(std::char_traits::to_char_type(get()) != literal_text[i])) + { + error_message = "invalid literal"; + return token_type::parse_error; + } + } + return return_type; + } + + ///////////////////// + // input management + ///////////////////// + + /// reset token_buffer; current character is beginning of token + void reset() noexcept + { + token_buffer.clear(); + token_string.clear(); + token_string.push_back(std::char_traits::to_char_type(current)); + } + + /* + @brief get next character from the input + + This function provides the interface to the used input adapter. It does + not throw in case the input reached EOF, but returns a + `std::char_traits::eof()` in that case. Stores the scanned characters + for use in error messages. + + @return character read from the input + */ + char_int_type get() + { + ++position.chars_read_total; + ++position.chars_read_current_line; + + if (next_unget) + { + // just reset the next_unget variable and work with current + next_unget = false; + } + else + { + current = ia.get_character(); + } + + if (JSON_HEDLEY_LIKELY(current != std::char_traits::eof())) + { + token_string.push_back(std::char_traits::to_char_type(current)); + } + + if (current == '\n') + { + ++position.lines_read; + position.chars_read_current_line = 0; + } + + return current; + } + + /*! + @brief unget current character (read it again on next get) + + We implement unget by setting variable next_unget to true. The input is not + changed - we just simulate ungetting by modifying chars_read_total, + chars_read_current_line, and token_string. The next call to get() will + behave as if the unget character is read again. + */ + void unget() + { + next_unget = true; + + --position.chars_read_total; + + // in case we "unget" a newline, we have to also decrement the lines_read + if (position.chars_read_current_line == 0) + { + if (position.lines_read > 0) + { + --position.lines_read; + } + } + else + { + --position.chars_read_current_line; + } + + if (JSON_HEDLEY_LIKELY(current != std::char_traits::eof())) + { + JSON_ASSERT(!token_string.empty()); + token_string.pop_back(); + } + } + + /// add a character to token_buffer + void add(char_int_type c) + { + token_buffer.push_back(static_cast(c)); + } + + public: + ///////////////////// + // value getters + ///////////////////// + + /// return integer value + constexpr number_integer_t get_number_integer() const noexcept + { + return value_integer; + } + + /// return unsigned integer value + constexpr number_unsigned_t get_number_unsigned() const noexcept + { + return value_unsigned; + } + + /// return floating-point value + constexpr number_float_t get_number_float() const noexcept + { + return value_float; + } + + /// return current string value (implicitly resets the token; useful only once) + string_t& get_string() + { + return token_buffer; + } + + ///////////////////// + // diagnostics + ///////////////////// + + /// return position of last read token + constexpr position_t get_position() const noexcept + { + return position; + } + + /// return the last read token (for errors only). Will never contain EOF + /// (an arbitrary value that is not a valid char value, often -1), because + /// 255 may legitimately occur. May contain NUL, which should be escaped. + std::string get_token_string() const + { + // escape control characters + std::string result; + for (const auto c : token_string) + { + if (static_cast(c) <= '\x1F') + { + // escape control characters + std::array cs{{}}; + static_cast((std::snprintf)(cs.data(), cs.size(), "", static_cast(c))); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + result += cs.data(); + } + else + { + // add character as is + result.push_back(static_cast(c)); + } + } + + return result; + } + + /// return syntax error message + JSON_HEDLEY_RETURNS_NON_NULL + constexpr const char* get_error_message() const noexcept + { + return error_message; + } + + ///////////////////// + // actual scanner + ///////////////////// + + /*! + @brief skip the UTF-8 byte order mark + @return true iff there is no BOM or the correct BOM has been skipped + */ + bool skip_bom() + { + if (get() == 0xEF) + { + // check if we completely parse the BOM + return get() == 0xBB && get() == 0xBF; + } + + // the first character is not the beginning of the BOM; unget it to + // process is later + unget(); + return true; + } + + void skip_whitespace() + { + do + { + get(); + } + while (current == ' ' || current == '\t' || current == '\n' || current == '\r'); + } + + token_type scan() + { + // initially, skip the BOM + if (position.chars_read_total == 0 && !skip_bom()) + { + error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given"; + return token_type::parse_error; + } + + // read next character and ignore whitespace + skip_whitespace(); + + // ignore comments + while (ignore_comments && current == '/') + { + if (!scan_comment()) + { + return token_type::parse_error; + } + + // skip following whitespace + skip_whitespace(); + } + + switch (current) + { + // structural characters + case '[': + return token_type::begin_array; + case ']': + return token_type::end_array; + case '{': + return token_type::begin_object; + case '}': + return token_type::end_object; + case ':': + return token_type::name_separator; + case ',': + return token_type::value_separator; + + // literals + case 't': + { + std::array true_literal = {{static_cast('t'), static_cast('r'), static_cast('u'), static_cast('e')}}; + return scan_literal(true_literal.data(), true_literal.size(), token_type::literal_true); + } + case 'f': + { + std::array false_literal = {{static_cast('f'), static_cast('a'), static_cast('l'), static_cast('s'), static_cast('e')}}; + return scan_literal(false_literal.data(), false_literal.size(), token_type::literal_false); + } + case 'n': + { + std::array null_literal = {{static_cast('n'), static_cast('u'), static_cast('l'), static_cast('l')}}; + return scan_literal(null_literal.data(), null_literal.size(), token_type::literal_null); + } + + // string + case '\"': + return scan_string(); + + // number + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + return scan_number(); + + // end of input (the null byte is needed when parsing from + // string literals) + case '\0': + case std::char_traits::eof(): + return token_type::end_of_input; + + // error + default: + error_message = "invalid literal"; + return token_type::parse_error; + } + } + + private: + /// input adapter + InputAdapterType ia; + + /// whether comments should be ignored (true) or signaled as errors (false) + const bool ignore_comments = false; + + /// the current character + char_int_type current = std::char_traits::eof(); + + /// whether the next get() call should just return current + bool next_unget = false; + + /// the start position of the current token + position_t position {}; + + /// raw input token string (for error messages) + std::vector token_string {}; + + /// buffer for variable-length tokens (numbers, strings) + string_t token_buffer {}; + + /// a description of occurred lexer errors + const char* error_message = ""; + + // number values + number_integer_t value_integer = 0; + number_unsigned_t value_unsigned = 0; + number_float_t value_float = 0; + + /// the decimal point + const char_int_type decimal_point_char = '.'; +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + + +#include // size_t +#include // declval +#include // string + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template +using null_function_t = decltype(std::declval().null()); + +template +using boolean_function_t = + decltype(std::declval().boolean(std::declval())); + +template +using number_integer_function_t = + decltype(std::declval().number_integer(std::declval())); + +template +using number_unsigned_function_t = + decltype(std::declval().number_unsigned(std::declval())); + +template +using number_float_function_t = decltype(std::declval().number_float( + std::declval(), std::declval())); + +template +using string_function_t = + decltype(std::declval().string(std::declval())); + +template +using binary_function_t = + decltype(std::declval().binary(std::declval())); + +template +using start_object_function_t = + decltype(std::declval().start_object(std::declval())); + +template +using key_function_t = + decltype(std::declval().key(std::declval())); + +template +using end_object_function_t = decltype(std::declval().end_object()); + +template +using start_array_function_t = + decltype(std::declval().start_array(std::declval())); + +template +using end_array_function_t = decltype(std::declval().end_array()); + +template +using parse_error_function_t = decltype(std::declval().parse_error( + std::declval(), std::declval(), + std::declval())); + +template +struct is_sax +{ + private: + static_assert(is_basic_json::value, + "BasicJsonType must be of type basic_json<...>"); + + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + using exception_t = typename BasicJsonType::exception; + + public: + static constexpr bool value = + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value; +}; + +template +struct is_sax_static_asserts +{ + private: + static_assert(is_basic_json::value, + "BasicJsonType must be of type basic_json<...>"); + + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + using exception_t = typename BasicJsonType::exception; + + public: + static_assert(is_detected_exact::value, + "Missing/invalid function: bool null()"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool boolean(bool)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool boolean(bool)"); + static_assert( + is_detected_exact::value, + "Missing/invalid function: bool number_integer(number_integer_t)"); + static_assert( + is_detected_exact::value, + "Missing/invalid function: bool number_unsigned(number_unsigned_t)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool number_float(number_float_t, const string_t&)"); + static_assert( + is_detected_exact::value, + "Missing/invalid function: bool string(string_t&)"); + static_assert( + is_detected_exact::value, + "Missing/invalid function: bool binary(binary_t&)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool start_object(std::size_t)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool key(string_t&)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool end_object()"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool start_array(std::size_t)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool end_array()"); + static_assert( + is_detected_exact::value, + "Missing/invalid function: bool parse_error(std::size_t, const " + "std::string&, const exception&)"); +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ + +/// how to treat CBOR tags +enum class cbor_tag_handler_t +{ + error, ///< throw a parse_error exception in case of a tag + ignore, ///< ignore tags + store ///< store tags as binary type +}; + +/*! +@brief determine system byte order + +@return true if and only if system's byte order is little endian + +@note from https://stackoverflow.com/a/1001328/266378 +*/ +static inline bool little_endianness(int num = 1) noexcept +{ + return *reinterpret_cast(&num) == 1; +} + + +/////////////////// +// binary reader // +/////////////////// + +/*! +@brief deserialization of CBOR, MessagePack, and UBJSON values +*/ +template> +class binary_reader +{ + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + using json_sax_t = SAX; + using char_type = typename InputAdapterType::char_type; + using char_int_type = typename std::char_traits::int_type; + + public: + /*! + @brief create a binary reader + + @param[in] adapter input adapter to read from + */ + explicit binary_reader(InputAdapterType&& adapter) noexcept : ia(std::move(adapter)) + { + (void)detail::is_sax_static_asserts {}; + } + + // make class move-only + binary_reader(const binary_reader&) = delete; + binary_reader(binary_reader&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + binary_reader& operator=(const binary_reader&) = delete; + binary_reader& operator=(binary_reader&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + ~binary_reader() = default; + + /*! + @param[in] format the binary format to parse + @param[in] sax_ a SAX event processor + @param[in] strict whether to expect the input to be consumed completed + @param[in] tag_handler how to treat CBOR tags + + @return whether parsing was successful + */ + JSON_HEDLEY_NON_NULL(3) + bool sax_parse(const input_format_t format, + json_sax_t* sax_, + const bool strict = true, + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) + { + sax = sax_; + bool result = false; + + switch (format) + { + case input_format_t::bson: + result = parse_bson_internal(); + break; + + case input_format_t::cbor: + result = parse_cbor_internal(true, tag_handler); + break; + + case input_format_t::msgpack: + result = parse_msgpack_internal(); + break; + + case input_format_t::ubjson: + result = parse_ubjson_internal(); + break; + + case input_format_t::json: // LCOV_EXCL_LINE + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + + // strict mode: next byte must be EOF + if (result && strict) + { + if (format == input_format_t::ubjson) + { + get_ignore_noop(); + } + else + { + get(); + } + + if (JSON_HEDLEY_UNLIKELY(current != std::char_traits::eof())) + { + return sax->parse_error(chars_read, get_token_string(), + parse_error::create(110, chars_read, exception_message(format, "expected end of input; last byte: 0x" + get_token_string(), "value"), BasicJsonType())); + } + } + + return result; + } + + private: + ////////// + // BSON // + ////////// + + /*! + @brief Reads in a BSON-object and passes it to the SAX-parser. + @return whether a valid BSON-value was passed to the SAX parser + */ + bool parse_bson_internal() + { + std::int32_t document_size{}; + get_number(input_format_t::bson, document_size); + + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(static_cast(-1)))) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/false))) + { + return false; + } + + return sax->end_object(); + } + + /*! + @brief Parses a C-style string from the BSON input. + @param[in,out] result A reference to the string variable where the read + string is to be stored. + @return `true` if the \x00-byte indicating the end of the string was + encountered before the EOF; false` indicates an unexpected EOF. + */ + bool get_bson_cstr(string_t& result) + { + auto out = std::back_inserter(result); + while (true) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "cstring"))) + { + return false; + } + if (current == 0x00) + { + return true; + } + *out++ = static_cast(current); + } + } + + /*! + @brief Parses a zero-terminated string of length @a len from the BSON + input. + @param[in] len The length (including the zero-byte at the end) of the + string to be read. + @param[in,out] result A reference to the string variable where the read + string is to be stored. + @tparam NumberType The type of the length @a len + @pre len >= 1 + @return `true` if the string was successfully parsed + */ + template + bool get_bson_string(const NumberType len, string_t& result) + { + if (JSON_HEDLEY_UNLIKELY(len < 1)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bson, "string length must be at least 1, is " + std::to_string(len), "string"), BasicJsonType())); + } + + return get_string(input_format_t::bson, len - static_cast(1), result) && get() != std::char_traits::eof(); + } + + /*! + @brief Parses a byte array input of length @a len from the BSON input. + @param[in] len The length of the byte array to be read. + @param[in,out] result A reference to the binary variable where the read + array is to be stored. + @tparam NumberType The type of the length @a len + @pre len >= 0 + @return `true` if the byte array was successfully parsed + */ + template + bool get_bson_binary(const NumberType len, binary_t& result) + { + if (JSON_HEDLEY_UNLIKELY(len < 0)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bson, "byte array length cannot be negative, is " + std::to_string(len), "binary"), BasicJsonType())); + } + + // All BSON binary values have a subtype + std::uint8_t subtype{}; + get_number(input_format_t::bson, subtype); + result.set_subtype(subtype); + + return get_binary(input_format_t::bson, len, result); + } + + /*! + @brief Read a BSON document element of the given @a element_type. + @param[in] element_type The BSON element type, c.f. http://bsonspec.org/spec.html + @param[in] element_type_parse_position The position in the input stream, + where the `element_type` was read. + @warning Not all BSON element types are supported yet. An unsupported + @a element_type will give rise to a parse_error.114: + Unsupported BSON record type 0x... + @return whether a valid BSON-object/array was passed to the SAX parser + */ + bool parse_bson_element_internal(const char_int_type element_type, + const std::size_t element_type_parse_position) + { + switch (element_type) + { + case 0x01: // double + { + double number{}; + return get_number(input_format_t::bson, number) && sax->number_float(static_cast(number), ""); + } + + case 0x02: // string + { + std::int32_t len{}; + string_t value; + return get_number(input_format_t::bson, len) && get_bson_string(len, value) && sax->string(value); + } + + case 0x03: // object + { + return parse_bson_internal(); + } + + case 0x04: // array + { + return parse_bson_array(); + } + + case 0x05: // binary + { + std::int32_t len{}; + binary_t value; + return get_number(input_format_t::bson, len) && get_bson_binary(len, value) && sax->binary(value); + } + + case 0x08: // boolean + { + return sax->boolean(get() != 0); + } + + case 0x0A: // null + { + return sax->null(); + } + + case 0x10: // int32 + { + std::int32_t value{}; + return get_number(input_format_t::bson, value) && sax->number_integer(value); + } + + case 0x12: // int64 + { + std::int64_t value{}; + return get_number(input_format_t::bson, value) && sax->number_integer(value); + } + + default: // anything else not supported (yet) + { + std::array cr{{}}; + static_cast((std::snprintf)(cr.data(), cr.size(), "%.2hhX", static_cast(element_type))); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + return sax->parse_error(element_type_parse_position, std::string(cr.data()), parse_error::create(114, element_type_parse_position, "Unsupported BSON record type 0x" + std::string(cr.data()), BasicJsonType())); + } + } + } + + /*! + @brief Read a BSON element list (as specified in the BSON-spec) + + The same binary layout is used for objects and arrays, hence it must be + indicated with the argument @a is_array which one is expected + (true --> array, false --> object). + + @param[in] is_array Determines if the element list being read is to be + treated as an object (@a is_array == false), or as an + array (@a is_array == true). + @return whether a valid BSON-object/array was passed to the SAX parser + */ + bool parse_bson_element_list(const bool is_array) + { + string_t key; + + while (auto element_type = get()) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list"))) + { + return false; + } + + const std::size_t element_type_parse_position = chars_read; + if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key))) + { + return false; + } + + if (!is_array && !sax->key(key)) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position))) + { + return false; + } + + // get_bson_cstr only appends + key.clear(); + } + + return true; + } + + /*! + @brief Reads an array from the BSON input and passes it to the SAX-parser. + @return whether a valid BSON-array was passed to the SAX parser + */ + bool parse_bson_array() + { + std::int32_t document_size{}; + get_number(input_format_t::bson, document_size); + + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(static_cast(-1)))) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/true))) + { + return false; + } + + return sax->end_array(); + } + + ////////// + // CBOR // + ////////// + + /*! + @param[in] get_char whether a new character should be retrieved from the + input (true) or whether the last read character should + be considered instead (false) + @param[in] tag_handler how CBOR tags should be treated + + @return whether a valid CBOR value was passed to the SAX parser + */ + bool parse_cbor_internal(const bool get_char, + const cbor_tag_handler_t tag_handler) + { + switch (get_char ? get() : current) + { + // EOF + case std::char_traits::eof(): + return unexpect_eof(input_format_t::cbor, "value"); + + // Integer 0x00..0x17 (0..23) + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x08: + case 0x09: + case 0x0A: + case 0x0B: + case 0x0C: + case 0x0D: + case 0x0E: + case 0x0F: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + return sax->number_unsigned(static_cast(current)); + + case 0x18: // Unsigned integer (one-byte uint8_t follows) + { + std::uint8_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + } + + case 0x19: // Unsigned integer (two-byte uint16_t follows) + { + std::uint16_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + } + + case 0x1A: // Unsigned integer (four-byte uint32_t follows) + { + std::uint32_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + } + + case 0x1B: // Unsigned integer (eight-byte uint64_t follows) + { + std::uint64_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + } + + // Negative integer -1-0x00..-1-0x17 (-1..-24) + case 0x20: + case 0x21: + case 0x22: + case 0x23: + case 0x24: + case 0x25: + case 0x26: + case 0x27: + case 0x28: + case 0x29: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: + case 0x30: + case 0x31: + case 0x32: + case 0x33: + case 0x34: + case 0x35: + case 0x36: + case 0x37: + return sax->number_integer(static_cast(0x20 - 1 - current)); + + case 0x38: // Negative integer (one-byte uint8_t follows) + { + std::uint8_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast(-1) - number); + } + + case 0x39: // Negative integer -1-n (two-byte uint16_t follows) + { + std::uint16_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast(-1) - number); + } + + case 0x3A: // Negative integer -1-n (four-byte uint32_t follows) + { + std::uint32_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast(-1) - number); + } + + case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows) + { + std::uint64_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast(-1) + - static_cast(number)); + } + + // Binary data (0x00..0x17 bytes follow) + case 0x40: + case 0x41: + case 0x42: + case 0x43: + case 0x44: + case 0x45: + case 0x46: + case 0x47: + case 0x48: + case 0x49: + case 0x4A: + case 0x4B: + case 0x4C: + case 0x4D: + case 0x4E: + case 0x4F: + case 0x50: + case 0x51: + case 0x52: + case 0x53: + case 0x54: + case 0x55: + case 0x56: + case 0x57: + case 0x58: // Binary data (one-byte uint8_t for n follows) + case 0x59: // Binary data (two-byte uint16_t for n follow) + case 0x5A: // Binary data (four-byte uint32_t for n follow) + case 0x5B: // Binary data (eight-byte uint64_t for n follow) + case 0x5F: // Binary data (indefinite length) + { + binary_t b; + return get_cbor_binary(b) && sax->binary(b); + } + + // UTF-8 string (0x00..0x17 bytes follow) + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + case 0x78: // UTF-8 string (one-byte uint8_t for n follows) + case 0x79: // UTF-8 string (two-byte uint16_t for n follow) + case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) + case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) + case 0x7F: // UTF-8 string (indefinite length) + { + string_t s; + return get_cbor_string(s) && sax->string(s); + } + + // array (0x00..0x17 data items follow) + case 0x80: + case 0x81: + case 0x82: + case 0x83: + case 0x84: + case 0x85: + case 0x86: + case 0x87: + case 0x88: + case 0x89: + case 0x8A: + case 0x8B: + case 0x8C: + case 0x8D: + case 0x8E: + case 0x8F: + case 0x90: + case 0x91: + case 0x92: + case 0x93: + case 0x94: + case 0x95: + case 0x96: + case 0x97: + return get_cbor_array(static_cast(static_cast(current) & 0x1Fu), tag_handler); + + case 0x98: // array (one-byte uint8_t for n follows) + { + std::uint8_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + } + + case 0x99: // array (two-byte uint16_t for n follow) + { + std::uint16_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + } + + case 0x9A: // array (four-byte uint32_t for n follow) + { + std::uint32_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + } + + case 0x9B: // array (eight-byte uint64_t for n follow) + { + std::uint64_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_array(detail::conditional_static_cast(len), tag_handler); + } + + case 0x9F: // array (indefinite length) + return get_cbor_array(static_cast(-1), tag_handler); + + // map (0x00..0x17 pairs of data items follow) + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + return get_cbor_object(static_cast(static_cast(current) & 0x1Fu), tag_handler); + + case 0xB8: // map (one-byte uint8_t for n follows) + { + std::uint8_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); + } + + case 0xB9: // map (two-byte uint16_t for n follow) + { + std::uint16_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); + } + + case 0xBA: // map (four-byte uint32_t for n follow) + { + std::uint32_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); + } + + case 0xBB: // map (eight-byte uint64_t for n follow) + { + std::uint64_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_object(detail::conditional_static_cast(len), tag_handler); + } + + case 0xBF: // map (indefinite length) + return get_cbor_object(static_cast(-1), tag_handler); + + case 0xC6: // tagged item + case 0xC7: + case 0xC8: + case 0xC9: + case 0xCA: + case 0xCB: + case 0xCC: + case 0xCD: + case 0xCE: + case 0xCF: + case 0xD0: + case 0xD1: + case 0xD2: + case 0xD3: + case 0xD4: + case 0xD8: // tagged item (1 bytes follow) + case 0xD9: // tagged item (2 bytes follow) + case 0xDA: // tagged item (4 bytes follow) + case 0xDB: // tagged item (8 bytes follow) + { + switch (tag_handler) + { + case cbor_tag_handler_t::error: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::cbor, "invalid byte: 0x" + last_token, "value"), BasicJsonType())); + } + + case cbor_tag_handler_t::ignore: + { + // ignore binary subtype + switch (current) + { + case 0xD8: + { + std::uint8_t subtype_to_ignore{}; + get_number(input_format_t::cbor, subtype_to_ignore); + break; + } + case 0xD9: + { + std::uint16_t subtype_to_ignore{}; + get_number(input_format_t::cbor, subtype_to_ignore); + break; + } + case 0xDA: + { + std::uint32_t subtype_to_ignore{}; + get_number(input_format_t::cbor, subtype_to_ignore); + break; + } + case 0xDB: + { + std::uint64_t subtype_to_ignore{}; + get_number(input_format_t::cbor, subtype_to_ignore); + break; + } + default: + break; + } + return parse_cbor_internal(true, tag_handler); + } + + case cbor_tag_handler_t::store: + { + binary_t b; + // use binary subtype and store in binary container + switch (current) + { + case 0xD8: + { + std::uint8_t subtype{}; + get_number(input_format_t::cbor, subtype); + b.set_subtype(detail::conditional_static_cast(subtype)); + break; + } + case 0xD9: + { + std::uint16_t subtype{}; + get_number(input_format_t::cbor, subtype); + b.set_subtype(detail::conditional_static_cast(subtype)); + break; + } + case 0xDA: + { + std::uint32_t subtype{}; + get_number(input_format_t::cbor, subtype); + b.set_subtype(detail::conditional_static_cast(subtype)); + break; + } + case 0xDB: + { + std::uint64_t subtype{}; + get_number(input_format_t::cbor, subtype); + b.set_subtype(detail::conditional_static_cast(subtype)); + break; + } + default: + return parse_cbor_internal(true, tag_handler); + } + get(); + return get_cbor_binary(b) && sax->binary(b); + } + + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + return false; // LCOV_EXCL_LINE + } + } + + case 0xF4: // false + return sax->boolean(false); + + case 0xF5: // true + return sax->boolean(true); + + case 0xF6: // null + return sax->null(); + + case 0xF9: // Half-Precision Float (two-byte IEEE 754) + { + const auto byte1_raw = get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "number"))) + { + return false; + } + const auto byte2_raw = get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "number"))) + { + return false; + } + + const auto byte1 = static_cast(byte1_raw); + const auto byte2 = static_cast(byte2_raw); + + // code from RFC 7049, Appendix D, Figure 3: + // As half-precision floating-point numbers were only added + // to IEEE 754 in 2008, today's programming platforms often + // still only have limited support for them. It is very + // easy to include at least decoding support for them even + // without such support. An example of a small decoder for + // half-precision floating-point numbers in the C language + // is shown in Fig. 3. + const auto half = static_cast((byte1 << 8u) + byte2); + const double val = [&half] + { + const int exp = (half >> 10u) & 0x1Fu; + const unsigned int mant = half & 0x3FFu; + JSON_ASSERT(0 <= exp&& exp <= 32); + JSON_ASSERT(mant <= 1024); + switch (exp) + { + case 0: + return std::ldexp(mant, -24); + case 31: + return (mant == 0) + ? std::numeric_limits::infinity() + : std::numeric_limits::quiet_NaN(); + default: + return std::ldexp(mant + 1024, exp - 25); + } + }(); + return sax->number_float((half & 0x8000u) != 0 + ? static_cast(-val) + : static_cast(val), ""); + } + + case 0xFA: // Single-Precision Float (four-byte IEEE 754) + { + float number{}; + return get_number(input_format_t::cbor, number) && sax->number_float(static_cast(number), ""); + } + + case 0xFB: // Double-Precision Float (eight-byte IEEE 754) + { + double number{}; + return get_number(input_format_t::cbor, number) && sax->number_float(static_cast(number), ""); + } + + default: // anything else (0xFF is handled inside the other types) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::cbor, "invalid byte: 0x" + last_token, "value"), BasicJsonType())); + } + } + } + + /*! + @brief reads a CBOR string + + This function first reads starting bytes to determine the expected + string length and then copies this number of bytes into a string. + Additionally, CBOR's strings with indefinite lengths are supported. + + @param[out] result created string + + @return whether string creation completed + */ + bool get_cbor_string(string_t& result) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "string"))) + { + return false; + } + + switch (current) + { + // UTF-8 string (0x00..0x17 bytes follow) + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + { + return get_string(input_format_t::cbor, static_cast(current) & 0x1Fu, result); + } + + case 0x78: // UTF-8 string (one-byte uint8_t for n follows) + { + std::uint8_t len{}; + return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result); + } + + case 0x79: // UTF-8 string (two-byte uint16_t for n follow) + { + std::uint16_t len{}; + return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result); + } + + case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) + { + std::uint32_t len{}; + return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result); + } + + case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) + { + std::uint64_t len{}; + return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result); + } + + case 0x7F: // UTF-8 string (indefinite length) + { + while (get() != 0xFF) + { + string_t chunk; + if (!get_cbor_string(chunk)) + { + return false; + } + result.append(chunk); + } + return true; + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::cbor, "expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x" + last_token, "string"), BasicJsonType())); + } + } + } + + /*! + @brief reads a CBOR byte array + + This function first reads starting bytes to determine the expected + byte array length and then copies this number of bytes into the byte array. + Additionally, CBOR's byte arrays with indefinite lengths are supported. + + @param[out] result created byte array + + @return whether byte array creation completed + */ + bool get_cbor_binary(binary_t& result) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "binary"))) + { + return false; + } + + switch (current) + { + // Binary data (0x00..0x17 bytes follow) + case 0x40: + case 0x41: + case 0x42: + case 0x43: + case 0x44: + case 0x45: + case 0x46: + case 0x47: + case 0x48: + case 0x49: + case 0x4A: + case 0x4B: + case 0x4C: + case 0x4D: + case 0x4E: + case 0x4F: + case 0x50: + case 0x51: + case 0x52: + case 0x53: + case 0x54: + case 0x55: + case 0x56: + case 0x57: + { + return get_binary(input_format_t::cbor, static_cast(current) & 0x1Fu, result); + } + + case 0x58: // Binary data (one-byte uint8_t for n follows) + { + std::uint8_t len{}; + return get_number(input_format_t::cbor, len) && + get_binary(input_format_t::cbor, len, result); + } + + case 0x59: // Binary data (two-byte uint16_t for n follow) + { + std::uint16_t len{}; + return get_number(input_format_t::cbor, len) && + get_binary(input_format_t::cbor, len, result); + } + + case 0x5A: // Binary data (four-byte uint32_t for n follow) + { + std::uint32_t len{}; + return get_number(input_format_t::cbor, len) && + get_binary(input_format_t::cbor, len, result); + } + + case 0x5B: // Binary data (eight-byte uint64_t for n follow) + { + std::uint64_t len{}; + return get_number(input_format_t::cbor, len) && + get_binary(input_format_t::cbor, len, result); + } + + case 0x5F: // Binary data (indefinite length) + { + while (get() != 0xFF) + { + binary_t chunk; + if (!get_cbor_binary(chunk)) + { + return false; + } + result.insert(result.end(), chunk.begin(), chunk.end()); + } + return true; + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::cbor, "expected length specification (0x40-0x5B) or indefinite binary array type (0x5F); last byte: 0x" + last_token, "binary"), BasicJsonType())); + } + } + } + + /*! + @param[in] len the length of the array or static_cast(-1) for an + array of indefinite size + @param[in] tag_handler how CBOR tags should be treated + @return whether array creation completed + */ + bool get_cbor_array(const std::size_t len, + const cbor_tag_handler_t tag_handler) + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) + { + return false; + } + + if (len != static_cast(-1)) + { + for (std::size_t i = 0; i < len; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) + { + return false; + } + } + } + else + { + while (get() != 0xFF) + { + if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(false, tag_handler))) + { + return false; + } + } + } + + return sax->end_array(); + } + + /*! + @param[in] len the length of the object or static_cast(-1) for an + object of indefinite size + @param[in] tag_handler how CBOR tags should be treated + @return whether object creation completed + */ + bool get_cbor_object(const std::size_t len, + const cbor_tag_handler_t tag_handler) + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) + { + return false; + } + + if (len != 0) + { + string_t key; + if (len != static_cast(-1)) + { + for (std::size_t i = 0; i < len; ++i) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) + { + return false; + } + key.clear(); + } + } + else + { + while (get() != 0xFF) + { + if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) + { + return false; + } + key.clear(); + } + } + } + + return sax->end_object(); + } + + ///////////// + // MsgPack // + ///////////// + + /*! + @return whether a valid MessagePack value was passed to the SAX parser + */ + bool parse_msgpack_internal() + { + switch (get()) + { + // EOF + case std::char_traits::eof(): + return unexpect_eof(input_format_t::msgpack, "value"); + + // positive fixint + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x08: + case 0x09: + case 0x0A: + case 0x0B: + case 0x0C: + case 0x0D: + case 0x0E: + case 0x0F: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + case 0x18: + case 0x19: + case 0x1A: + case 0x1B: + case 0x1C: + case 0x1D: + case 0x1E: + case 0x1F: + case 0x20: + case 0x21: + case 0x22: + case 0x23: + case 0x24: + case 0x25: + case 0x26: + case 0x27: + case 0x28: + case 0x29: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: + case 0x30: + case 0x31: + case 0x32: + case 0x33: + case 0x34: + case 0x35: + case 0x36: + case 0x37: + case 0x38: + case 0x39: + case 0x3A: + case 0x3B: + case 0x3C: + case 0x3D: + case 0x3E: + case 0x3F: + case 0x40: + case 0x41: + case 0x42: + case 0x43: + case 0x44: + case 0x45: + case 0x46: + case 0x47: + case 0x48: + case 0x49: + case 0x4A: + case 0x4B: + case 0x4C: + case 0x4D: + case 0x4E: + case 0x4F: + case 0x50: + case 0x51: + case 0x52: + case 0x53: + case 0x54: + case 0x55: + case 0x56: + case 0x57: + case 0x58: + case 0x59: + case 0x5A: + case 0x5B: + case 0x5C: + case 0x5D: + case 0x5E: + case 0x5F: + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + case 0x78: + case 0x79: + case 0x7A: + case 0x7B: + case 0x7C: + case 0x7D: + case 0x7E: + case 0x7F: + return sax->number_unsigned(static_cast(current)); + + // fixmap + case 0x80: + case 0x81: + case 0x82: + case 0x83: + case 0x84: + case 0x85: + case 0x86: + case 0x87: + case 0x88: + case 0x89: + case 0x8A: + case 0x8B: + case 0x8C: + case 0x8D: + case 0x8E: + case 0x8F: + return get_msgpack_object(static_cast(static_cast(current) & 0x0Fu)); + + // fixarray + case 0x90: + case 0x91: + case 0x92: + case 0x93: + case 0x94: + case 0x95: + case 0x96: + case 0x97: + case 0x98: + case 0x99: + case 0x9A: + case 0x9B: + case 0x9C: + case 0x9D: + case 0x9E: + case 0x9F: + return get_msgpack_array(static_cast(static_cast(current) & 0x0Fu)); + + // fixstr + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + case 0xB8: + case 0xB9: + case 0xBA: + case 0xBB: + case 0xBC: + case 0xBD: + case 0xBE: + case 0xBF: + case 0xD9: // str 8 + case 0xDA: // str 16 + case 0xDB: // str 32 + { + string_t s; + return get_msgpack_string(s) && sax->string(s); + } + + case 0xC0: // nil + return sax->null(); + + case 0xC2: // false + return sax->boolean(false); + + case 0xC3: // true + return sax->boolean(true); + + case 0xC4: // bin 8 + case 0xC5: // bin 16 + case 0xC6: // bin 32 + case 0xC7: // ext 8 + case 0xC8: // ext 16 + case 0xC9: // ext 32 + case 0xD4: // fixext 1 + case 0xD5: // fixext 2 + case 0xD6: // fixext 4 + case 0xD7: // fixext 8 + case 0xD8: // fixext 16 + { + binary_t b; + return get_msgpack_binary(b) && sax->binary(b); + } + + case 0xCA: // float 32 + { + float number{}; + return get_number(input_format_t::msgpack, number) && sax->number_float(static_cast(number), ""); + } + + case 0xCB: // float 64 + { + double number{}; + return get_number(input_format_t::msgpack, number) && sax->number_float(static_cast(number), ""); + } + + case 0xCC: // uint 8 + { + std::uint8_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); + } + + case 0xCD: // uint 16 + { + std::uint16_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); + } + + case 0xCE: // uint 32 + { + std::uint32_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); + } + + case 0xCF: // uint 64 + { + std::uint64_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); + } + + case 0xD0: // int 8 + { + std::int8_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_integer(number); + } + + case 0xD1: // int 16 + { + std::int16_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_integer(number); + } + + case 0xD2: // int 32 + { + std::int32_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_integer(number); + } + + case 0xD3: // int 64 + { + std::int64_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_integer(number); + } + + case 0xDC: // array 16 + { + std::uint16_t len{}; + return get_number(input_format_t::msgpack, len) && get_msgpack_array(static_cast(len)); + } + + case 0xDD: // array 32 + { + std::uint32_t len{}; + return get_number(input_format_t::msgpack, len) && get_msgpack_array(static_cast(len)); + } + + case 0xDE: // map 16 + { + std::uint16_t len{}; + return get_number(input_format_t::msgpack, len) && get_msgpack_object(static_cast(len)); + } + + case 0xDF: // map 32 + { + std::uint32_t len{}; + return get_number(input_format_t::msgpack, len) && get_msgpack_object(static_cast(len)); + } + + // negative fixint + case 0xE0: + case 0xE1: + case 0xE2: + case 0xE3: + case 0xE4: + case 0xE5: + case 0xE6: + case 0xE7: + case 0xE8: + case 0xE9: + case 0xEA: + case 0xEB: + case 0xEC: + case 0xED: + case 0xEE: + case 0xEF: + case 0xF0: + case 0xF1: + case 0xF2: + case 0xF3: + case 0xF4: + case 0xF5: + case 0xF6: + case 0xF7: + case 0xF8: + case 0xF9: + case 0xFA: + case 0xFB: + case 0xFC: + case 0xFD: + case 0xFE: + case 0xFF: + return sax->number_integer(static_cast(current)); + + default: // anything else + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::msgpack, "invalid byte: 0x" + last_token, "value"), BasicJsonType())); + } + } + } + + /*! + @brief reads a MessagePack string + + This function first reads starting bytes to determine the expected + string length and then copies this number of bytes into a string. + + @param[out] result created string + + @return whether string creation completed + */ + bool get_msgpack_string(string_t& result) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::msgpack, "string"))) + { + return false; + } + + switch (current) + { + // fixstr + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + case 0xB8: + case 0xB9: + case 0xBA: + case 0xBB: + case 0xBC: + case 0xBD: + case 0xBE: + case 0xBF: + { + return get_string(input_format_t::msgpack, static_cast(current) & 0x1Fu, result); + } + + case 0xD9: // str 8 + { + std::uint8_t len{}; + return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result); + } + + case 0xDA: // str 16 + { + std::uint16_t len{}; + return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result); + } + + case 0xDB: // str 32 + { + std::uint32_t len{}; + return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result); + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::msgpack, "expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x" + last_token, "string"), BasicJsonType())); + } + } + } + + /*! + @brief reads a MessagePack byte array + + This function first reads starting bytes to determine the expected + byte array length and then copies this number of bytes into a byte array. + + @param[out] result created byte array + + @return whether byte array creation completed + */ + bool get_msgpack_binary(binary_t& result) + { + // helper function to set the subtype + auto assign_and_return_true = [&result](std::int8_t subtype) + { + result.set_subtype(static_cast(subtype)); + return true; + }; + + switch (current) + { + case 0xC4: // bin 8 + { + std::uint8_t len{}; + return get_number(input_format_t::msgpack, len) && + get_binary(input_format_t::msgpack, len, result); + } + + case 0xC5: // bin 16 + { + std::uint16_t len{}; + return get_number(input_format_t::msgpack, len) && + get_binary(input_format_t::msgpack, len, result); + } + + case 0xC6: // bin 32 + { + std::uint32_t len{}; + return get_number(input_format_t::msgpack, len) && + get_binary(input_format_t::msgpack, len, result); + } + + case 0xC7: // ext 8 + { + std::uint8_t len{}; + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, len) && + get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, len, result) && + assign_and_return_true(subtype); + } + + case 0xC8: // ext 16 + { + std::uint16_t len{}; + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, len) && + get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, len, result) && + assign_and_return_true(subtype); + } + + case 0xC9: // ext 32 + { + std::uint32_t len{}; + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, len) && + get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, len, result) && + assign_and_return_true(subtype); + } + + case 0xD4: // fixext 1 + { + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, 1, result) && + assign_and_return_true(subtype); + } + + case 0xD5: // fixext 2 + { + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, 2, result) && + assign_and_return_true(subtype); + } + + case 0xD6: // fixext 4 + { + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, 4, result) && + assign_and_return_true(subtype); + } + + case 0xD7: // fixext 8 + { + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, 8, result) && + assign_and_return_true(subtype); + } + + case 0xD8: // fixext 16 + { + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, 16, result) && + assign_and_return_true(subtype); + } + + default: // LCOV_EXCL_LINE + return false; // LCOV_EXCL_LINE + } + } + + /*! + @param[in] len the length of the array + @return whether array creation completed + */ + bool get_msgpack_array(const std::size_t len) + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) + { + return false; + } + + for (std::size_t i = 0; i < len; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_internal())) + { + return false; + } + } + + return sax->end_array(); + } + + /*! + @param[in] len the length of the object + @return whether object creation completed + */ + bool get_msgpack_object(const std::size_t len) + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) + { + return false; + } + + string_t key; + for (std::size_t i = 0; i < len; ++i) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!get_msgpack_string(key) || !sax->key(key))) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_internal())) + { + return false; + } + key.clear(); + } + + return sax->end_object(); + } + + //////////// + // UBJSON // + //////////// + + /*! + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return whether a valid UBJSON value was passed to the SAX parser + */ + bool parse_ubjson_internal(const bool get_char = true) + { + return get_ubjson_value(get_char ? get_ignore_noop() : current); + } + + /*! + @brief reads a UBJSON string + + This function is either called after reading the 'S' byte explicitly + indicating a string, or in case of an object key where the 'S' byte can be + left out. + + @param[out] result created string + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return whether string creation completed + */ + bool get_ubjson_string(string_t& result, const bool get_char = true) + { + if (get_char) + { + get(); // TODO(niels): may we ignore N here? + } + + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "value"))) + { + return false; + } + + switch (current) + { + case 'U': + { + std::uint8_t len{}; + return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + } + + case 'i': + { + std::int8_t len{}; + return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + } + + case 'I': + { + std::int16_t len{}; + return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + } + + case 'l': + { + std::int32_t len{}; + return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + } + + case 'L': + { + std::int64_t len{}; + return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + } + + default: + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L); last byte: 0x" + last_token, "string"), BasicJsonType())); + } + } + + /*! + @param[out] result determined size + @return whether size determination completed + */ + bool get_ubjson_size_value(std::size_t& result) + { + switch (get_ignore_noop()) + { + case 'U': + { + std::uint8_t number{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast(number); + return true; + } + + case 'i': + { + std::int8_t number{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast(number); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char + return true; + } + + case 'I': + { + std::int16_t number{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast(number); + return true; + } + + case 'l': + { + std::int32_t number{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast(number); + return true; + } + + case 'L': + { + std::int64_t number{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast(number); + return true; + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L) after '#'; last byte: 0x" + last_token, "size"), BasicJsonType())); + } + } + } + + /*! + @brief determine the type and size for a container + + In the optimized UBJSON format, a type and a size can be provided to allow + for a more compact representation. + + @param[out] result pair of the size and the type + + @return whether pair creation completed + */ + bool get_ubjson_size_type(std::pair& result) + { + result.first = string_t::npos; // size + result.second = 0; // type + + get_ignore_noop(); + + if (current == '$') + { + result.second = get(); // must not ignore 'N', because 'N' maybe the type + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "type"))) + { + return false; + } + + get_ignore_noop(); + if (JSON_HEDLEY_UNLIKELY(current != '#')) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "value"))) + { + return false; + } + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "expected '#' after type information; last byte: 0x" + last_token, "size"), BasicJsonType())); + } + + return get_ubjson_size_value(result.first); + } + + if (current == '#') + { + return get_ubjson_size_value(result.first); + } + + return true; + } + + /*! + @param prefix the previously read or set type prefix + @return whether value creation completed + */ + bool get_ubjson_value(const char_int_type prefix) + { + switch (prefix) + { + case std::char_traits::eof(): // EOF + return unexpect_eof(input_format_t::ubjson, "value"); + + case 'T': // true + return sax->boolean(true); + case 'F': // false + return sax->boolean(false); + + case 'Z': // null + return sax->null(); + + case 'U': + { + std::uint8_t number{}; + return get_number(input_format_t::ubjson, number) && sax->number_unsigned(number); + } + + case 'i': + { + std::int8_t number{}; + return get_number(input_format_t::ubjson, number) && sax->number_integer(number); + } + + case 'I': + { + std::int16_t number{}; + return get_number(input_format_t::ubjson, number) && sax->number_integer(number); + } + + case 'l': + { + std::int32_t number{}; + return get_number(input_format_t::ubjson, number) && sax->number_integer(number); + } + + case 'L': + { + std::int64_t number{}; + return get_number(input_format_t::ubjson, number) && sax->number_integer(number); + } + + case 'd': + { + float number{}; + return get_number(input_format_t::ubjson, number) && sax->number_float(static_cast(number), ""); + } + + case 'D': + { + double number{}; + return get_number(input_format_t::ubjson, number) && sax->number_float(static_cast(number), ""); + } + + case 'H': + { + return get_ubjson_high_precision_number(); + } + + case 'C': // char + { + get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "char"))) + { + return false; + } + if (JSON_HEDLEY_UNLIKELY(current > 127)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "byte after 'C' must be in range 0x00..0x7F; last byte: 0x" + last_token, "char"), BasicJsonType())); + } + string_t s(1, static_cast(current)); + return sax->string(s); + } + + case 'S': // string + { + string_t s; + return get_ubjson_string(s) && sax->string(s); + } + + case '[': // array + return get_ubjson_array(); + + case '{': // object + return get_ubjson_object(); + + default: // anything else + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "invalid byte: 0x" + last_token, "value"), BasicJsonType())); + } + } + } + + /*! + @return whether array creation completed + */ + bool get_ubjson_array() + { + std::pair size_and_type; + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type))) + { + return false; + } + + if (size_and_type.first != string_t::npos) + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first))) + { + return false; + } + + if (size_and_type.second != 0) + { + if (size_and_type.second != 'N') + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) + { + return false; + } + } + } + } + else + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) + { + return false; + } + } + } + } + else + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(static_cast(-1)))) + { + return false; + } + + while (current != ']') + { + if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal(false))) + { + return false; + } + get_ignore_noop(); + } + } + + return sax->end_array(); + } + + /*! + @return whether object creation completed + */ + bool get_ubjson_object() + { + std::pair size_and_type; + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type))) + { + return false; + } + + string_t key; + if (size_and_type.first != string_t::npos) + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(size_and_type.first))) + { + return false; + } + + if (size_and_type.second != 0) + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) + { + return false; + } + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) + { + return false; + } + key.clear(); + } + } + else + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) + { + return false; + } + if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) + { + return false; + } + key.clear(); + } + } + } + else + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(static_cast(-1)))) + { + return false; + } + + while (current != '}') + { + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key))) + { + return false; + } + if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) + { + return false; + } + get_ignore_noop(); + key.clear(); + } + } + + return sax->end_object(); + } + + // Note, no reader for UBJSON binary types is implemented because they do + // not exist + + bool get_ubjson_high_precision_number() + { + // get size of following number string + std::size_t size{}; + auto res = get_ubjson_size_value(size); + if (JSON_HEDLEY_UNLIKELY(!res)) + { + return res; + } + + // get number string + std::vector number_vector; + for (std::size_t i = 0; i < size; ++i) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "number"))) + { + return false; + } + number_vector.push_back(static_cast(current)); + } + + // parse number string + using ia_type = decltype(detail::input_adapter(number_vector)); + auto number_lexer = detail::lexer(detail::input_adapter(number_vector), false); + const auto result_number = number_lexer.scan(); + const auto number_string = number_lexer.get_token_string(); + const auto result_remainder = number_lexer.scan(); + + using token_type = typename detail::lexer_base::token_type; + + if (JSON_HEDLEY_UNLIKELY(result_remainder != token_type::end_of_input)) + { + return sax->parse_error(chars_read, number_string, parse_error::create(115, chars_read, exception_message(input_format_t::ubjson, "invalid number text: " + number_lexer.get_token_string(), "high-precision number"), BasicJsonType())); + } + + switch (result_number) + { + case token_type::value_integer: + return sax->number_integer(number_lexer.get_number_integer()); + case token_type::value_unsigned: + return sax->number_unsigned(number_lexer.get_number_unsigned()); + case token_type::value_float: + return sax->number_float(number_lexer.get_number_float(), std::move(number_string)); + case token_type::uninitialized: + case token_type::literal_true: + case token_type::literal_false: + case token_type::literal_null: + case token_type::value_string: + case token_type::begin_array: + case token_type::begin_object: + case token_type::end_array: + case token_type::end_object: + case token_type::name_separator: + case token_type::value_separator: + case token_type::parse_error: + case token_type::end_of_input: + case token_type::literal_or_value: + default: + return sax->parse_error(chars_read, number_string, parse_error::create(115, chars_read, exception_message(input_format_t::ubjson, "invalid number text: " + number_lexer.get_token_string(), "high-precision number"), BasicJsonType())); + } + } + + /////////////////////// + // Utility functions // + /////////////////////// + + /*! + @brief get next character from the input + + This function provides the interface to the used input adapter. It does + not throw in case the input reached EOF, but returns a -'ve valued + `std::char_traits::eof()` in that case. + + @return character read from the input + */ + char_int_type get() + { + ++chars_read; + return current = ia.get_character(); + } + + /*! + @return character read from the input after ignoring all 'N' entries + */ + char_int_type get_ignore_noop() + { + do + { + get(); + } + while (current == 'N'); + + return current; + } + + /* + @brief read a number from the input + + @tparam NumberType the type of the number + @param[in] format the current format (for diagnostics) + @param[out] result number of type @a NumberType + + @return whether conversion completed + + @note This function needs to respect the system's endianness, because + bytes in CBOR, MessagePack, and UBJSON are stored in network order + (big endian) and therefore need reordering on little endian systems. + */ + template + bool get_number(const input_format_t format, NumberType& result) + { + // step 1: read input into array with system's byte order + std::array vec{}; + for (std::size_t i = 0; i < sizeof(NumberType); ++i) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "number"))) + { + return false; + } + + // reverse byte order prior to conversion if necessary + if (is_little_endian != InputIsLittleEndian) + { + vec[sizeof(NumberType) - i - 1] = static_cast(current); + } + else + { + vec[i] = static_cast(current); // LCOV_EXCL_LINE + } + } + + // step 2: convert array into number of type T and return + std::memcpy(&result, vec.data(), sizeof(NumberType)); + return true; + } + + /*! + @brief create a string by reading characters from the input + + @tparam NumberType the type of the number + @param[in] format the current format (for diagnostics) + @param[in] len number of characters to read + @param[out] result string created by reading @a len bytes + + @return whether string creation completed + + @note We can not reserve @a len bytes for the result, because @a len + may be too large. Usually, @ref unexpect_eof() detects the end of + the input before we run out of string memory. + */ + template + bool get_string(const input_format_t format, + const NumberType len, + string_t& result) + { + bool success = true; + for (NumberType i = 0; i < len; i++) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "string"))) + { + success = false; + break; + } + result.push_back(static_cast(current)); + } + return success; + } + + /*! + @brief create a byte array by reading bytes from the input + + @tparam NumberType the type of the number + @param[in] format the current format (for diagnostics) + @param[in] len number of bytes to read + @param[out] result byte array created by reading @a len bytes + + @return whether byte array creation completed + + @note We can not reserve @a len bytes for the result, because @a len + may be too large. Usually, @ref unexpect_eof() detects the end of + the input before we run out of memory. + */ + template + bool get_binary(const input_format_t format, + const NumberType len, + binary_t& result) + { + bool success = true; + for (NumberType i = 0; i < len; i++) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "binary"))) + { + success = false; + break; + } + result.push_back(static_cast(current)); + } + return success; + } + + /*! + @param[in] format the current format (for diagnostics) + @param[in] context further context information (for diagnostics) + @return whether the last read character is not EOF + */ + JSON_HEDLEY_NON_NULL(3) + bool unexpect_eof(const input_format_t format, const char* context) const + { + if (JSON_HEDLEY_UNLIKELY(current == std::char_traits::eof())) + { + return sax->parse_error(chars_read, "", + parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context), BasicJsonType())); + } + return true; + } + + /*! + @return a string representation of the last read byte + */ + std::string get_token_string() const + { + std::array cr{{}}; + static_cast((std::snprintf)(cr.data(), cr.size(), "%.2hhX", static_cast(current))); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + return std::string{cr.data()}; + } + + /*! + @param[in] format the current format + @param[in] detail a detailed error message + @param[in] context further context information + @return a message string to use in the parse_error exceptions + */ + std::string exception_message(const input_format_t format, + const std::string& detail, + const std::string& context) const + { + std::string error_msg = "syntax error while parsing "; + + switch (format) + { + case input_format_t::cbor: + error_msg += "CBOR"; + break; + + case input_format_t::msgpack: + error_msg += "MessagePack"; + break; + + case input_format_t::ubjson: + error_msg += "UBJSON"; + break; + + case input_format_t::bson: + error_msg += "BSON"; + break; + + case input_format_t::json: // LCOV_EXCL_LINE + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + + return error_msg + " " + context + ": " + detail; + } + + private: + /// input adapter + InputAdapterType ia; + + /// the current character + char_int_type current = std::char_traits::eof(); + + /// the number of characters read + std::size_t chars_read = 0; + + /// whether we can assume little endianness + const bool is_little_endian = little_endianness(); + + /// the SAX parser + json_sax_t* sax = nullptr; +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + +// #include + + +#include // isfinite +#include // uint8_t +#include // function +#include // string +#include // move +#include // vector + +// #include + +// #include + +// #include + +// #include + +// #include + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +//////////// +// parser // +//////////// + +enum class parse_event_t : std::uint8_t +{ + /// the parser read `{` and started to process a JSON object + object_start, + /// the parser read `}` and finished processing a JSON object + object_end, + /// the parser read `[` and started to process a JSON array + array_start, + /// the parser read `]` and finished processing a JSON array + array_end, + /// the parser read a key of a value in an object + key, + /// the parser finished reading a JSON value + value +}; + +template +using parser_callback_t = + std::function; + +/*! +@brief syntax analysis + +This class implements a recursive descent parser. +*/ +template +class parser +{ + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using lexer_t = lexer; + using token_type = typename lexer_t::token_type; + + public: + /// a parser reading from an input adapter + explicit parser(InputAdapterType&& adapter, + const parser_callback_t cb = nullptr, + const bool allow_exceptions_ = true, + const bool skip_comments = false) + : callback(cb) + , m_lexer(std::move(adapter), skip_comments) + , allow_exceptions(allow_exceptions_) + { + // read first token + get_token(); + } + + /*! + @brief public parser interface + + @param[in] strict whether to expect the last token to be EOF + @param[in,out] result parsed JSON value + + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + */ + void parse(const bool strict, BasicJsonType& result) + { + if (callback) + { + json_sax_dom_callback_parser sdp(result, callback, allow_exceptions); + sax_parse_internal(&sdp); + + // in strict mode, input must be completely read + if (strict && (get_token() != token_type::end_of_input)) + { + sdp.parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_of_input, "value"), BasicJsonType())); + } + + // in case of an error, return discarded value + if (sdp.is_errored()) + { + result = value_t::discarded; + return; + } + + // set top-level value to null if it was discarded by the callback + // function + if (result.is_discarded()) + { + result = nullptr; + } + } + else + { + json_sax_dom_parser sdp(result, allow_exceptions); + sax_parse_internal(&sdp); + + // in strict mode, input must be completely read + if (strict && (get_token() != token_type::end_of_input)) + { + sdp.parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), BasicJsonType())); + } + + // in case of an error, return discarded value + if (sdp.is_errored()) + { + result = value_t::discarded; + return; + } + } + + result.assert_invariant(); + } + + /*! + @brief public accept interface + + @param[in] strict whether to expect the last token to be EOF + @return whether the input is a proper JSON text + */ + bool accept(const bool strict = true) + { + json_sax_acceptor sax_acceptor; + return sax_parse(&sax_acceptor, strict); + } + + template + JSON_HEDLEY_NON_NULL(2) + bool sax_parse(SAX* sax, const bool strict = true) + { + (void)detail::is_sax_static_asserts {}; + const bool result = sax_parse_internal(sax); + + // strict mode: next byte must be EOF + if (result && strict && (get_token() != token_type::end_of_input)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), BasicJsonType())); + } + + return result; + } + + private: + template + JSON_HEDLEY_NON_NULL(2) + bool sax_parse_internal(SAX* sax) + { + // stack to remember the hierarchy of structured values we are parsing + // true = array; false = object + std::vector states; + // value to avoid a goto (see comment where set to true) + bool skip_to_state_evaluation = false; + + while (true) + { + if (!skip_to_state_evaluation) + { + // invariant: get_token() was called before each iteration + switch (last_token) + { + case token_type::begin_object: + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(static_cast(-1)))) + { + return false; + } + + // closing } -> we are done + if (get_token() == token_type::end_object) + { + if (JSON_HEDLEY_UNLIKELY(!sax->end_object())) + { + return false; + } + break; + } + + // parse key + if (JSON_HEDLEY_UNLIKELY(last_token != token_type::value_string)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), BasicJsonType())); + } + if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string()))) + { + return false; + } + + // parse separator (:) + if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), BasicJsonType())); + } + + // remember we are now inside an object + states.push_back(false); + + // parse values + get_token(); + continue; + } + + case token_type::begin_array: + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(static_cast(-1)))) + { + return false; + } + + // closing ] -> we are done + if (get_token() == token_type::end_array) + { + if (JSON_HEDLEY_UNLIKELY(!sax->end_array())) + { + return false; + } + break; + } + + // remember we are now inside an array + states.push_back(true); + + // parse values (no need to call get_token) + continue; + } + + case token_type::value_float: + { + const auto res = m_lexer.get_number_float(); + + if (JSON_HEDLEY_UNLIKELY(!std::isfinite(res))) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + out_of_range::create(406, "number overflow parsing '" + m_lexer.get_token_string() + "'", BasicJsonType())); + } + + if (JSON_HEDLEY_UNLIKELY(!sax->number_float(res, m_lexer.get_string()))) + { + return false; + } + + break; + } + + case token_type::literal_false: + { + if (JSON_HEDLEY_UNLIKELY(!sax->boolean(false))) + { + return false; + } + break; + } + + case token_type::literal_null: + { + if (JSON_HEDLEY_UNLIKELY(!sax->null())) + { + return false; + } + break; + } + + case token_type::literal_true: + { + if (JSON_HEDLEY_UNLIKELY(!sax->boolean(true))) + { + return false; + } + break; + } + + case token_type::value_integer: + { + if (JSON_HEDLEY_UNLIKELY(!sax->number_integer(m_lexer.get_number_integer()))) + { + return false; + } + break; + } + + case token_type::value_string: + { + if (JSON_HEDLEY_UNLIKELY(!sax->string(m_lexer.get_string()))) + { + return false; + } + break; + } + + case token_type::value_unsigned: + { + if (JSON_HEDLEY_UNLIKELY(!sax->number_unsigned(m_lexer.get_number_unsigned()))) + { + return false; + } + break; + } + + case token_type::parse_error: + { + // using "uninitialized" to avoid "expected" message + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::uninitialized, "value"), BasicJsonType())); + } + + case token_type::uninitialized: + case token_type::end_array: + case token_type::end_object: + case token_type::name_separator: + case token_type::value_separator: + case token_type::end_of_input: + case token_type::literal_or_value: + default: // the last token was unexpected + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), BasicJsonType())); + } + } + } + else + { + skip_to_state_evaluation = false; + } + + // we reached this line after we successfully parsed a value + if (states.empty()) + { + // empty stack: we reached the end of the hierarchy: done + return true; + } + + if (states.back()) // array + { + // comma -> next value + if (get_token() == token_type::value_separator) + { + // parse a new value + get_token(); + continue; + } + + // closing ] + if (JSON_HEDLEY_LIKELY(last_token == token_type::end_array)) + { + if (JSON_HEDLEY_UNLIKELY(!sax->end_array())) + { + return false; + } + + // We are done with this array. Before we can parse a + // new value, we need to evaluate the new state first. + // By setting skip_to_state_evaluation to false, we + // are effectively jumping to the beginning of this if. + JSON_ASSERT(!states.empty()); + states.pop_back(); + skip_to_state_evaluation = true; + continue; + } + + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_array, "array"), BasicJsonType())); + } + + // states.back() is false -> object + + // comma -> next value + if (get_token() == token_type::value_separator) + { + // parse key + if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::value_string)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), BasicJsonType())); + } + + if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string()))) + { + return false; + } + + // parse separator (:) + if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), BasicJsonType())); + } + + // parse values + get_token(); + continue; + } + + // closing } + if (JSON_HEDLEY_LIKELY(last_token == token_type::end_object)) + { + if (JSON_HEDLEY_UNLIKELY(!sax->end_object())) + { + return false; + } + + // We are done with this object. Before we can parse a + // new value, we need to evaluate the new state first. + // By setting skip_to_state_evaluation to false, we + // are effectively jumping to the beginning of this if. + JSON_ASSERT(!states.empty()); + states.pop_back(); + skip_to_state_evaluation = true; + continue; + } + + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_object, "object"), BasicJsonType())); + } + } + + /// get next token from lexer + token_type get_token() + { + return last_token = m_lexer.scan(); + } + + std::string exception_message(const token_type expected, const std::string& context) + { + std::string error_msg = "syntax error "; + + if (!context.empty()) + { + error_msg += "while parsing " + context + " "; + } + + error_msg += "- "; + + if (last_token == token_type::parse_error) + { + error_msg += std::string(m_lexer.get_error_message()) + "; last read: '" + + m_lexer.get_token_string() + "'"; + } + else + { + error_msg += "unexpected " + std::string(lexer_t::token_type_name(last_token)); + } + + if (expected != token_type::uninitialized) + { + error_msg += "; expected " + std::string(lexer_t::token_type_name(expected)); + } + + return error_msg; + } + + private: + /// callback function + const parser_callback_t callback = nullptr; + /// the type of the last read token + token_type last_token = token_type::uninitialized; + /// the lexer + lexer_t m_lexer; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; +}; + +} // namespace detail +} // namespace nlohmann + +// #include + + +// #include + + +#include // ptrdiff_t +#include // numeric_limits + +// #include + + +namespace nlohmann +{ +namespace detail +{ +/* +@brief an iterator for primitive JSON types + +This class models an iterator for primitive JSON types (boolean, number, +string). It's only purpose is to allow the iterator/const_iterator classes +to "iterate" over primitive values. Internally, the iterator is modeled by +a `difference_type` variable. Value begin_value (`0`) models the begin, +end_value (`1`) models past the end. +*/ +class primitive_iterator_t +{ + private: + using difference_type = std::ptrdiff_t; + static constexpr difference_type begin_value = 0; + static constexpr difference_type end_value = begin_value + 1; + + JSON_PRIVATE_UNLESS_TESTED: + /// iterator as signed integer type + difference_type m_it = (std::numeric_limits::min)(); + + public: + constexpr difference_type get_value() const noexcept + { + return m_it; + } + + /// set iterator to a defined beginning + void set_begin() noexcept + { + m_it = begin_value; + } + + /// set iterator to a defined past the end + void set_end() noexcept + { + m_it = end_value; + } + + /// return whether the iterator can be dereferenced + constexpr bool is_begin() const noexcept + { + return m_it == begin_value; + } + + /// return whether the iterator is at end + constexpr bool is_end() const noexcept + { + return m_it == end_value; + } + + friend constexpr bool operator==(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it == rhs.m_it; + } + + friend constexpr bool operator<(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it < rhs.m_it; + } + + primitive_iterator_t operator+(difference_type n) noexcept + { + auto result = *this; + result += n; + return result; + } + + friend constexpr difference_type operator-(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it - rhs.m_it; + } + + primitive_iterator_t& operator++() noexcept + { + ++m_it; + return *this; + } + + primitive_iterator_t const operator++(int) noexcept // NOLINT(readability-const-return-type) + { + auto result = *this; + ++m_it; + return result; + } + + primitive_iterator_t& operator--() noexcept + { + --m_it; + return *this; + } + + primitive_iterator_t const operator--(int) noexcept // NOLINT(readability-const-return-type) + { + auto result = *this; + --m_it; + return result; + } + + primitive_iterator_t& operator+=(difference_type n) noexcept + { + m_it += n; + return *this; + } + + primitive_iterator_t& operator-=(difference_type n) noexcept + { + m_it -= n; + return *this; + } +}; +} // namespace detail +} // namespace nlohmann + + +namespace nlohmann +{ +namespace detail +{ +/*! +@brief an iterator value + +@note This structure could easily be a union, but MSVC currently does not allow +unions members with complex constructors, see https://github.com/nlohmann/json/pull/105. +*/ +template struct internal_iterator +{ + /// iterator for JSON objects + typename BasicJsonType::object_t::iterator object_iterator {}; + /// iterator for JSON arrays + typename BasicJsonType::array_t::iterator array_iterator {}; + /// generic iterator for all other types + primitive_iterator_t primitive_iterator {}; +}; +} // namespace detail +} // namespace nlohmann + +// #include + + +#include // iterator, random_access_iterator_tag, bidirectional_iterator_tag, advance, next +#include // conditional, is_const, remove_const + +// #include + +// #include + +// #include + +// #include + +// #include + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +// forward declare, to be able to friend it later on +template class iteration_proxy; +template class iteration_proxy_value; + +/*! +@brief a template for a bidirectional iterator for the @ref basic_json class +This class implements a both iterators (iterator and const_iterator) for the +@ref basic_json class. +@note An iterator is called *initialized* when a pointer to a JSON value has + been set (e.g., by a constructor or a copy assignment). If the iterator is + default-constructed, it is *uninitialized* and most methods are undefined. + **The library uses assertions to detect calls on uninitialized iterators.** +@requirement The class satisfies the following concept requirements: +- +[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator): + The iterator that can be moved can be moved in both directions (i.e. + incremented and decremented). +@since version 1.0.0, simplified in version 2.0.9, change to bidirectional + iterators in version 3.0.0 (see https://github.com/nlohmann/json/issues/593) +*/ +template +class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-special-member-functions) +{ + /// the iterator with BasicJsonType of different const-ness + using other_iter_impl = iter_impl::value, typename std::remove_const::type, const BasicJsonType>::type>; + /// allow basic_json to access private members + friend other_iter_impl; + friend BasicJsonType; + friend iteration_proxy; + friend iteration_proxy_value; + + using object_t = typename BasicJsonType::object_t; + using array_t = typename BasicJsonType::array_t; + // make sure BasicJsonType is basic_json or const basic_json + static_assert(is_basic_json::type>::value, + "iter_impl only accepts (const) basic_json"); + + public: + + /// The std::iterator class template (used as a base class to provide typedefs) is deprecated in C++17. + /// The C++ Standard has never required user-defined iterators to derive from std::iterator. + /// A user-defined iterator should provide publicly accessible typedefs named + /// iterator_category, value_type, difference_type, pointer, and reference. + /// Note that value_type is required to be non-const, even for constant iterators. + using iterator_category = std::bidirectional_iterator_tag; + + /// the type of the values when the iterator is dereferenced + using value_type = typename BasicJsonType::value_type; + /// a type to represent differences between iterators + using difference_type = typename BasicJsonType::difference_type; + /// defines a pointer to the type iterated over (value_type) + using pointer = typename std::conditional::value, + typename BasicJsonType::const_pointer, + typename BasicJsonType::pointer>::type; + /// defines a reference to the type iterated over (value_type) + using reference = + typename std::conditional::value, + typename BasicJsonType::const_reference, + typename BasicJsonType::reference>::type; + + iter_impl() = default; + ~iter_impl() = default; + iter_impl(iter_impl&&) noexcept = default; + iter_impl& operator=(iter_impl&&) noexcept = default; + + /*! + @brief constructor for a given JSON instance + @param[in] object pointer to a JSON object for this iterator + @pre object != nullptr + @post The iterator is initialized; i.e. `m_object != nullptr`. + */ + explicit iter_impl(pointer object) noexcept : m_object(object) + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + m_it.object_iterator = typename object_t::iterator(); + break; + } + + case value_t::array: + { + m_it.array_iterator = typename array_t::iterator(); + break; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + m_it.primitive_iterator = primitive_iterator_t(); + break; + } + } + } + + /*! + @note The conventional copy constructor and copy assignment are implicitly + defined. Combined with the following converting constructor and + assignment, they support: (1) copy from iterator to iterator, (2) + copy from const iterator to const iterator, and (3) conversion from + iterator to const iterator. However conversion from const iterator + to iterator is not defined. + */ + + /*! + @brief const copy constructor + @param[in] other const iterator to copy from + @note This copy constructor had to be defined explicitly to circumvent a bug + occurring on msvc v19.0 compiler (VS 2015) debug build. For more + information refer to: https://github.com/nlohmann/json/issues/1608 + */ + iter_impl(const iter_impl& other) noexcept + : m_object(other.m_object), m_it(other.m_it) + {} + + /*! + @brief converting assignment + @param[in] other const iterator to copy from + @return const/non-const iterator + @note It is not checked whether @a other is initialized. + */ + iter_impl& operator=(const iter_impl& other) noexcept + { + if (&other != this) + { + m_object = other.m_object; + m_it = other.m_it; + } + return *this; + } + + /*! + @brief converting constructor + @param[in] other non-const iterator to copy from + @note It is not checked whether @a other is initialized. + */ + iter_impl(const iter_impl::type>& other) noexcept + : m_object(other.m_object), m_it(other.m_it) + {} + + /*! + @brief converting assignment + @param[in] other non-const iterator to copy from + @return const/non-const iterator + @note It is not checked whether @a other is initialized. + */ + iter_impl& operator=(const iter_impl::type>& other) noexcept // NOLINT(cert-oop54-cpp) + { + m_object = other.m_object; + m_it = other.m_it; + return *this; + } + + JSON_PRIVATE_UNLESS_TESTED: + /*! + @brief set the iterator to the first value + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + void set_begin() noexcept + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + m_it.object_iterator = m_object->m_value.object->begin(); + break; + } + + case value_t::array: + { + m_it.array_iterator = m_object->m_value.array->begin(); + break; + } + + case value_t::null: + { + // set to end so begin()==end() is true: null is empty + m_it.primitive_iterator.set_end(); + break; + } + + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + m_it.primitive_iterator.set_begin(); + break; + } + } + } + + /*! + @brief set the iterator past the last value + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + void set_end() noexcept + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + m_it.object_iterator = m_object->m_value.object->end(); + break; + } + + case value_t::array: + { + m_it.array_iterator = m_object->m_value.array->end(); + break; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + m_it.primitive_iterator.set_end(); + break; + } + } + } + + public: + /*! + @brief return a reference to the value pointed to by the iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference operator*() const + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + JSON_ASSERT(m_it.object_iterator != m_object->m_value.object->end()); + return m_it.object_iterator->second; + } + + case value_t::array: + { + JSON_ASSERT(m_it.array_iterator != m_object->m_value.array->end()); + return *m_it.array_iterator; + } + + case value_t::null: + JSON_THROW(invalid_iterator::create(214, "cannot get value", *m_object)); + + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.is_begin())) + { + return *m_object; + } + + JSON_THROW(invalid_iterator::create(214, "cannot get value", *m_object)); + } + } + } + + /*! + @brief dereference the iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + pointer operator->() const + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + JSON_ASSERT(m_it.object_iterator != m_object->m_value.object->end()); + return &(m_it.object_iterator->second); + } + + case value_t::array: + { + JSON_ASSERT(m_it.array_iterator != m_object->m_value.array->end()); + return &*m_it.array_iterator; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.is_begin())) + { + return m_object; + } + + JSON_THROW(invalid_iterator::create(214, "cannot get value", *m_object)); + } + } + } + + /*! + @brief post-increment (it++) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl const operator++(int) // NOLINT(readability-const-return-type) + { + auto result = *this; + ++(*this); + return result; + } + + /*! + @brief pre-increment (++it) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl& operator++() + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + std::advance(m_it.object_iterator, 1); + break; + } + + case value_t::array: + { + std::advance(m_it.array_iterator, 1); + break; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + ++m_it.primitive_iterator; + break; + } + } + + return *this; + } + + /*! + @brief post-decrement (it--) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl const operator--(int) // NOLINT(readability-const-return-type) + { + auto result = *this; + --(*this); + return result; + } + + /*! + @brief pre-decrement (--it) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl& operator--() + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + std::advance(m_it.object_iterator, -1); + break; + } + + case value_t::array: + { + std::advance(m_it.array_iterator, -1); + break; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + --m_it.primitive_iterator; + break; + } + } + + return *this; + } + + /*! + @brief comparison: equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + template < typename IterImpl, detail::enable_if_t < (std::is_same::value || std::is_same::value), std::nullptr_t > = nullptr > + bool operator==(const IterImpl& other) const + { + // if objects are not the same, the comparison is undefined + if (JSON_HEDLEY_UNLIKELY(m_object != other.m_object)) + { + JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers", *m_object)); + } + + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + return (m_it.object_iterator == other.m_it.object_iterator); + + case value_t::array: + return (m_it.array_iterator == other.m_it.array_iterator); + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + return (m_it.primitive_iterator == other.m_it.primitive_iterator); + } + } + + /*! + @brief comparison: not equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + template < typename IterImpl, detail::enable_if_t < (std::is_same::value || std::is_same::value), std::nullptr_t > = nullptr > + bool operator!=(const IterImpl& other) const + { + return !operator==(other); + } + + /*! + @brief comparison: smaller + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator<(const iter_impl& other) const + { + // if objects are not the same, the comparison is undefined + if (JSON_HEDLEY_UNLIKELY(m_object != other.m_object)) + { + JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers", *m_object)); + } + + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + JSON_THROW(invalid_iterator::create(213, "cannot compare order of object iterators", *m_object)); + + case value_t::array: + return (m_it.array_iterator < other.m_it.array_iterator); + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + return (m_it.primitive_iterator < other.m_it.primitive_iterator); + } + } + + /*! + @brief comparison: less than or equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator<=(const iter_impl& other) const + { + return !other.operator < (*this); + } + + /*! + @brief comparison: greater than + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator>(const iter_impl& other) const + { + return !operator<=(other); + } + + /*! + @brief comparison: greater than or equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator>=(const iter_impl& other) const + { + return !operator<(other); + } + + /*! + @brief add to iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl& operator+=(difference_type i) + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators", *m_object)); + + case value_t::array: + { + std::advance(m_it.array_iterator, i); + break; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + m_it.primitive_iterator += i; + break; + } + } + + return *this; + } + + /*! + @brief subtract from iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl& operator-=(difference_type i) + { + return operator+=(-i); + } + + /*! + @brief add to iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl operator+(difference_type i) const + { + auto result = *this; + result += i; + return result; + } + + /*! + @brief addition of distance and iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + friend iter_impl operator+(difference_type i, const iter_impl& it) + { + auto result = it; + result += i; + return result; + } + + /*! + @brief subtract from iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl operator-(difference_type i) const + { + auto result = *this; + result -= i; + return result; + } + + /*! + @brief return difference + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + difference_type operator-(const iter_impl& other) const + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators", *m_object)); + + case value_t::array: + return m_it.array_iterator - other.m_it.array_iterator; + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + return m_it.primitive_iterator - other.m_it.primitive_iterator; + } + } + + /*! + @brief access to successor + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference operator[](difference_type n) const + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + JSON_THROW(invalid_iterator::create(208, "cannot use operator[] for object iterators", *m_object)); + + case value_t::array: + return *std::next(m_it.array_iterator, n); + + case value_t::null: + JSON_THROW(invalid_iterator::create(214, "cannot get value", *m_object)); + + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.get_value() == -n)) + { + return *m_object; + } + + JSON_THROW(invalid_iterator::create(214, "cannot get value", *m_object)); + } + } + } + + /*! + @brief return the key of an object iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const typename object_t::key_type& key() const + { + JSON_ASSERT(m_object != nullptr); + + if (JSON_HEDLEY_LIKELY(m_object->is_object())) + { + return m_it.object_iterator->first; + } + + JSON_THROW(invalid_iterator::create(207, "cannot use key() for non-object iterators", *m_object)); + } + + /*! + @brief return the value of an iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference value() const + { + return operator*(); + } + + JSON_PRIVATE_UNLESS_TESTED: + /// associated JSON instance + pointer m_object = nullptr; + /// the actual iterator of the associated instance + internal_iterator::type> m_it {}; +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + + +#include // ptrdiff_t +#include // reverse_iterator +#include // declval + +namespace nlohmann +{ +namespace detail +{ +////////////////////// +// reverse_iterator // +////////////////////// + +/*! +@brief a template for a reverse iterator class + +@tparam Base the base iterator type to reverse. Valid types are @ref +iterator (to create @ref reverse_iterator) and @ref const_iterator (to +create @ref const_reverse_iterator). + +@requirement The class satisfies the following concept requirements: +- +[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator): + The iterator that can be moved can be moved in both directions (i.e. + incremented and decremented). +- [OutputIterator](https://en.cppreference.com/w/cpp/named_req/OutputIterator): + It is possible to write to the pointed-to element (only if @a Base is + @ref iterator). + +@since version 1.0.0 +*/ +template +class json_reverse_iterator : public std::reverse_iterator +{ + public: + using difference_type = std::ptrdiff_t; + /// shortcut to the reverse iterator adapter + using base_iterator = std::reverse_iterator; + /// the reference type for the pointed-to element + using reference = typename Base::reference; + + /// create reverse iterator from iterator + explicit json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept + : base_iterator(it) {} + + /// create reverse iterator from base class + explicit json_reverse_iterator(const base_iterator& it) noexcept : base_iterator(it) {} + + /// post-increment (it++) + json_reverse_iterator const operator++(int) // NOLINT(readability-const-return-type) + { + return static_cast(base_iterator::operator++(1)); + } + + /// pre-increment (++it) + json_reverse_iterator& operator++() + { + return static_cast(base_iterator::operator++()); + } + + /// post-decrement (it--) + json_reverse_iterator const operator--(int) // NOLINT(readability-const-return-type) + { + return static_cast(base_iterator::operator--(1)); + } + + /// pre-decrement (--it) + json_reverse_iterator& operator--() + { + return static_cast(base_iterator::operator--()); + } + + /// add to iterator + json_reverse_iterator& operator+=(difference_type i) + { + return static_cast(base_iterator::operator+=(i)); + } + + /// add to iterator + json_reverse_iterator operator+(difference_type i) const + { + return static_cast(base_iterator::operator+(i)); + } + + /// subtract from iterator + json_reverse_iterator operator-(difference_type i) const + { + return static_cast(base_iterator::operator-(i)); + } + + /// return difference + difference_type operator-(const json_reverse_iterator& other) const + { + return base_iterator(*this) - base_iterator(other); + } + + /// access to successor + reference operator[](difference_type n) const + { + return *(this->operator+(n)); + } + + /// return the key of an object iterator + auto key() const -> decltype(std::declval().key()) + { + auto it = --this->base(); + return it.key(); + } + + /// return the value of an iterator + reference value() const + { + auto it = --this->base(); + return it.operator * (); + } +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + + +#include // all_of +#include // isdigit +#include // max +#include // accumulate +#include // string +#include // move +#include // vector + +// #include + +// #include + +// #include + +// #include + + +namespace nlohmann +{ + +/// @brief JSON Pointer defines a string syntax for identifying a specific value within a JSON document +/// @sa https://json.nlohmann.me/api/json_pointer/ +template +class json_pointer +{ + // allow basic_json to access private members + NLOHMANN_BASIC_JSON_TPL_DECLARATION + friend class basic_json; + + public: + /// @brief create JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/json_pointer/ + explicit json_pointer(const std::string& s = "") + : reference_tokens(split(s)) + {} + + /// @brief return a string representation of the JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/to_string/ + std::string to_string() const + { + return std::accumulate(reference_tokens.begin(), reference_tokens.end(), + std::string{}, + [](const std::string & a, const std::string & b) + { + return a + "/" + detail::escape(b); + }); + } + + /// @brief return a string representation of the JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/operator_string/ + operator std::string() const + { + return to_string(); + } + + /// @brief append another JSON pointer at the end of this JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/operator_slasheq/ + json_pointer& operator/=(const json_pointer& ptr) + { + reference_tokens.insert(reference_tokens.end(), + ptr.reference_tokens.begin(), + ptr.reference_tokens.end()); + return *this; + } + + /// @brief append an unescaped reference token at the end of this JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/operator_slasheq/ + json_pointer& operator/=(std::string token) + { + push_back(std::move(token)); + return *this; + } + + /// @brief append an array index at the end of this JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/operator_slasheq/ + json_pointer& operator/=(std::size_t array_idx) + { + return *this /= std::to_string(array_idx); + } + + /// @brief create a new JSON pointer by appending the right JSON pointer at the end of the left JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ + friend json_pointer operator/(const json_pointer& lhs, + const json_pointer& rhs) + { + return json_pointer(lhs) /= rhs; + } + + /// @brief create a new JSON pointer by appending the unescaped token at the end of the JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ + friend json_pointer operator/(const json_pointer& lhs, std::string token) // NOLINT(performance-unnecessary-value-param) + { + return json_pointer(lhs) /= std::move(token); + } + + /// @brief create a new JSON pointer by appending the array-index-token at the end of the JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ + friend json_pointer operator/(const json_pointer& lhs, std::size_t array_idx) + { + return json_pointer(lhs) /= array_idx; + } + + /// @brief returns the parent of this JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/parent_pointer/ + json_pointer parent_pointer() const + { + if (empty()) + { + return *this; + } + + json_pointer res = *this; + res.pop_back(); + return res; + } + + /// @brief remove last reference token + /// @sa https://json.nlohmann.me/api/json_pointer/pop_back/ + void pop_back() + { + if (JSON_HEDLEY_UNLIKELY(empty())) + { + JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent", BasicJsonType())); + } + + reference_tokens.pop_back(); + } + + /// @brief return last reference token + /// @sa https://json.nlohmann.me/api/json_pointer/back/ + const std::string& back() const + { + if (JSON_HEDLEY_UNLIKELY(empty())) + { + JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent", BasicJsonType())); + } + + return reference_tokens.back(); + } + + /// @brief append an unescaped token at the end of the reference pointer + /// @sa https://json.nlohmann.me/api/json_pointer/push_back/ + void push_back(const std::string& token) + { + reference_tokens.push_back(token); + } + + /// @brief append an unescaped token at the end of the reference pointer + /// @sa https://json.nlohmann.me/api/json_pointer/push_back/ + void push_back(std::string&& token) + { + reference_tokens.push_back(std::move(token)); + } + + /// @brief return whether pointer points to the root document + /// @sa https://json.nlohmann.me/api/json_pointer/empty/ + bool empty() const noexcept + { + return reference_tokens.empty(); + } + + private: + /*! + @param[in] s reference token to be converted into an array index + + @return integer representation of @a s + + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index begins not with a digit + @throw out_of_range.404 if string @a s could not be converted to an integer + @throw out_of_range.410 if an array index exceeds size_type + */ + static typename BasicJsonType::size_type array_index(const std::string& s) + { + using size_type = typename BasicJsonType::size_type; + + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && s[0] == '0')) + { + JSON_THROW(detail::parse_error::create(106, 0, "array index '" + s + "' must not begin with '0'", BasicJsonType())); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && !(s[0] >= '1' && s[0] <= '9'))) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + s + "' is not a number", BasicJsonType())); + } + + std::size_t processed_chars = 0; + unsigned long long res = 0; // NOLINT(runtime/int) + JSON_TRY + { + res = std::stoull(s, &processed_chars); + } + JSON_CATCH(std::out_of_range&) + { + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + s + "'", BasicJsonType())); + } + + // check if the string was completely read + if (JSON_HEDLEY_UNLIKELY(processed_chars != s.size())) + { + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + s + "'", BasicJsonType())); + } + + // only triggered on special platforms (like 32bit), see also + // https://github.com/nlohmann/json/pull/2203 + if (res >= static_cast((std::numeric_limits::max)())) // NOLINT(runtime/int) + { + JSON_THROW(detail::out_of_range::create(410, "array index " + s + " exceeds size_type", BasicJsonType())); // LCOV_EXCL_LINE + } + + return static_cast(res); + } + + JSON_PRIVATE_UNLESS_TESTED: + json_pointer top() const + { + if (JSON_HEDLEY_UNLIKELY(empty())) + { + JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent", BasicJsonType())); + } + + json_pointer result = *this; + result.reference_tokens = {reference_tokens[0]}; + return result; + } + + private: + /*! + @brief create and return a reference to the pointed to value + + @complexity Linear in the number of reference tokens. + + @throw parse_error.109 if array index is not a number + @throw type_error.313 if value cannot be unflattened + */ + BasicJsonType& get_and_create(BasicJsonType& j) const + { + auto* result = &j; + + // in case no reference tokens exist, return a reference to the JSON value + // j which will be overwritten by a primitive value + for (const auto& reference_token : reference_tokens) + { + switch (result->type()) + { + case detail::value_t::null: + { + if (reference_token == "0") + { + // start a new array if reference token is 0 + result = &result->operator[](0); + } + else + { + // start a new object otherwise + result = &result->operator[](reference_token); + } + break; + } + + case detail::value_t::object: + { + // create an entry in the object + result = &result->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + // create an entry in the array + result = &result->operator[](array_index(reference_token)); + break; + } + + /* + The following code is only reached if there exists a reference + token _and_ the current value is primitive. In this case, we have + an error situation, because primitive values may only occur as + single value; that is, with an empty list of reference tokens. + */ + case detail::value_t::string: + case detail::value_t::boolean: + case detail::value_t::number_integer: + case detail::value_t::number_unsigned: + case detail::value_t::number_float: + case detail::value_t::binary: + case detail::value_t::discarded: + default: + JSON_THROW(detail::type_error::create(313, "invalid value to unflatten", j)); + } + } + + return *result; + } + + /*! + @brief return a reference to the pointed to value + + @note This version does not throw if a value is not present, but tries to + create nested values instead. For instance, calling this function + with pointer `"/this/that"` on a null value is equivalent to calling + `operator[]("this").operator[]("that")` on that value, effectively + changing the null value to an object. + + @param[in] ptr a JSON value + + @return reference to the JSON value pointed to by the JSON pointer + + @complexity Linear in the length of the JSON pointer. + + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.404 if the JSON pointer can not be resolved + */ + BasicJsonType& get_unchecked(BasicJsonType* ptr) const + { + for (const auto& reference_token : reference_tokens) + { + // convert null values to arrays or objects before continuing + if (ptr->is_null()) + { + // check if reference token is a number + const bool nums = + std::all_of(reference_token.begin(), reference_token.end(), + [](const unsigned char x) + { + return std::isdigit(x); + }); + + // change value to array for numbers or "-" or to object otherwise + *ptr = (nums || reference_token == "-") + ? detail::value_t::array + : detail::value_t::object; + } + + switch (ptr->type()) + { + case detail::value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + if (reference_token == "-") + { + // explicitly treat "-" as index beyond the end + ptr = &ptr->operator[](ptr->m_value.array->size()); + } + else + { + // convert array index to number; unchecked access + ptr = &ptr->operator[](array_index(reference_token)); + } + break; + } + + case detail::value_t::null: + case detail::value_t::string: + case detail::value_t::boolean: + case detail::value_t::number_integer: + case detail::value_t::number_unsigned: + case detail::value_t::number_float: + case detail::value_t::binary: + case detail::value_t::discarded: + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'", *ptr)); + } + } + + return *ptr; + } + + /*! + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved + */ + BasicJsonType& get_checked(BasicJsonType* ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->type()) + { + case detail::value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_HEDLEY_UNLIKELY(reference_token == "-")) + { + // "-" always fails the range check + JSON_THROW(detail::out_of_range::create(402, + "array index '-' (" + std::to_string(ptr->m_value.array->size()) + + ") is out of range", *ptr)); + } + + // note: at performs range check + ptr = &ptr->at(array_index(reference_token)); + break; + } + + case detail::value_t::null: + case detail::value_t::string: + case detail::value_t::boolean: + case detail::value_t::number_integer: + case detail::value_t::number_unsigned: + case detail::value_t::number_float: + case detail::value_t::binary: + case detail::value_t::discarded: + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'", *ptr)); + } + } + + return *ptr; + } + + /*! + @brief return a const reference to the pointed to value + + @param[in] ptr a JSON value + + @return const reference to the JSON value pointed to by the JSON + pointer + + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved + */ + const BasicJsonType& get_unchecked(const BasicJsonType* ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->type()) + { + case detail::value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_HEDLEY_UNLIKELY(reference_token == "-")) + { + // "-" cannot be used for const access + JSON_THROW(detail::out_of_range::create(402, "array index '-' (" + std::to_string(ptr->m_value.array->size()) + ") is out of range", *ptr)); + } + + // use unchecked array access + ptr = &ptr->operator[](array_index(reference_token)); + break; + } + + case detail::value_t::null: + case detail::value_t::string: + case detail::value_t::boolean: + case detail::value_t::number_integer: + case detail::value_t::number_unsigned: + case detail::value_t::number_float: + case detail::value_t::binary: + case detail::value_t::discarded: + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'", *ptr)); + } + } + + return *ptr; + } + + /*! + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved + */ + const BasicJsonType& get_checked(const BasicJsonType* ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->type()) + { + case detail::value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_HEDLEY_UNLIKELY(reference_token == "-")) + { + // "-" always fails the range check + JSON_THROW(detail::out_of_range::create(402, + "array index '-' (" + std::to_string(ptr->m_value.array->size()) + + ") is out of range", *ptr)); + } + + // note: at performs range check + ptr = &ptr->at(array_index(reference_token)); + break; + } + + case detail::value_t::null: + case detail::value_t::string: + case detail::value_t::boolean: + case detail::value_t::number_integer: + case detail::value_t::number_unsigned: + case detail::value_t::number_float: + case detail::value_t::binary: + case detail::value_t::discarded: + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'", *ptr)); + } + } + + return *ptr; + } + + /*! + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + */ + bool contains(const BasicJsonType* ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->type()) + { + case detail::value_t::object: + { + if (!ptr->contains(reference_token)) + { + // we did not find the key in the object + return false; + } + + ptr = &ptr->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_HEDLEY_UNLIKELY(reference_token == "-")) + { + // "-" always fails the range check + return false; + } + if (JSON_HEDLEY_UNLIKELY(reference_token.size() == 1 && !("0" <= reference_token && reference_token <= "9"))) + { + // invalid char + return false; + } + if (JSON_HEDLEY_UNLIKELY(reference_token.size() > 1)) + { + if (JSON_HEDLEY_UNLIKELY(!('1' <= reference_token[0] && reference_token[0] <= '9'))) + { + // first char should be between '1' and '9' + return false; + } + for (std::size_t i = 1; i < reference_token.size(); i++) + { + if (JSON_HEDLEY_UNLIKELY(!('0' <= reference_token[i] && reference_token[i] <= '9'))) + { + // other char should be between '0' and '9' + return false; + } + } + } + + const auto idx = array_index(reference_token); + if (idx >= ptr->size()) + { + // index out of range + return false; + } + + ptr = &ptr->operator[](idx); + break; + } + + case detail::value_t::null: + case detail::value_t::string: + case detail::value_t::boolean: + case detail::value_t::number_integer: + case detail::value_t::number_unsigned: + case detail::value_t::number_float: + case detail::value_t::binary: + case detail::value_t::discarded: + default: + { + // we do not expect primitive values if there is still a + // reference token to process + return false; + } + } + } + + // no reference token left means we found a primitive value + return true; + } + + /*! + @brief split the string input to reference tokens + + @note This function is only called by the json_pointer constructor. + All exceptions below are documented there. + + @throw parse_error.107 if the pointer is not empty or begins with '/' + @throw parse_error.108 if character '~' is not followed by '0' or '1' + */ + static std::vector split(const std::string& reference_string) + { + std::vector result; + + // special case: empty reference string -> no reference tokens + if (reference_string.empty()) + { + return result; + } + + // check if nonempty reference string begins with slash + if (JSON_HEDLEY_UNLIKELY(reference_string[0] != '/')) + { + JSON_THROW(detail::parse_error::create(107, 1, "JSON pointer must be empty or begin with '/' - was: '" + reference_string + "'", BasicJsonType())); + } + + // extract the reference tokens: + // - slash: position of the last read slash (or end of string) + // - start: position after the previous slash + for ( + // search for the first slash after the first character + std::size_t slash = reference_string.find_first_of('/', 1), + // set the beginning of the first reference token + start = 1; + // we can stop if start == 0 (if slash == std::string::npos) + start != 0; + // set the beginning of the next reference token + // (will eventually be 0 if slash == std::string::npos) + start = (slash == std::string::npos) ? 0 : slash + 1, + // find next slash + slash = reference_string.find_first_of('/', start)) + { + // use the text between the beginning of the reference token + // (start) and the last slash (slash). + auto reference_token = reference_string.substr(start, slash - start); + + // check reference tokens are properly escaped + for (std::size_t pos = reference_token.find_first_of('~'); + pos != std::string::npos; + pos = reference_token.find_first_of('~', pos + 1)) + { + JSON_ASSERT(reference_token[pos] == '~'); + + // ~ must be followed by 0 or 1 + if (JSON_HEDLEY_UNLIKELY(pos == reference_token.size() - 1 || + (reference_token[pos + 1] != '0' && + reference_token[pos + 1] != '1'))) + { + JSON_THROW(detail::parse_error::create(108, 0, "escape character '~' must be followed with '0' or '1'", BasicJsonType())); + } + } + + // finally, store the reference token + detail::unescape(reference_token); + result.push_back(reference_token); + } + + return result; + } + + private: + /*! + @param[in] reference_string the reference string to the current value + @param[in] value the value to consider + @param[in,out] result the result object to insert values to + + @note Empty objects or arrays are flattened to `null`. + */ + static void flatten(const std::string& reference_string, + const BasicJsonType& value, + BasicJsonType& result) + { + switch (value.type()) + { + case detail::value_t::array: + { + if (value.m_value.array->empty()) + { + // flatten empty array as null + result[reference_string] = nullptr; + } + else + { + // iterate array and use index as reference string + for (std::size_t i = 0; i < value.m_value.array->size(); ++i) + { + flatten(reference_string + "/" + std::to_string(i), + value.m_value.array->operator[](i), result); + } + } + break; + } + + case detail::value_t::object: + { + if (value.m_value.object->empty()) + { + // flatten empty object as null + result[reference_string] = nullptr; + } + else + { + // iterate object and use keys as reference string + for (const auto& element : *value.m_value.object) + { + flatten(reference_string + "/" + detail::escape(element.first), element.second, result); + } + } + break; + } + + case detail::value_t::null: + case detail::value_t::string: + case detail::value_t::boolean: + case detail::value_t::number_integer: + case detail::value_t::number_unsigned: + case detail::value_t::number_float: + case detail::value_t::binary: + case detail::value_t::discarded: + default: + { + // add primitive value with its reference string + result[reference_string] = value; + break; + } + } + } + + /*! + @param[in] value flattened JSON + + @return unflattened JSON + + @throw parse_error.109 if array index is not a number + @throw type_error.314 if value is not an object + @throw type_error.315 if object values are not primitive + @throw type_error.313 if value cannot be unflattened + */ + static BasicJsonType + unflatten(const BasicJsonType& value) + { + if (JSON_HEDLEY_UNLIKELY(!value.is_object())) + { + JSON_THROW(detail::type_error::create(314, "only objects can be unflattened", value)); + } + + BasicJsonType result; + + // iterate the JSON object values + for (const auto& element : *value.m_value.object) + { + if (JSON_HEDLEY_UNLIKELY(!element.second.is_primitive())) + { + JSON_THROW(detail::type_error::create(315, "values in object must be primitive", element.second)); + } + + // assign value to reference pointed to by JSON pointer; Note that if + // the JSON pointer is "" (i.e., points to the whole value), function + // get_and_create returns a reference to result itself. An assignment + // will then create a primitive value. + json_pointer(element.first).get_and_create(result) = element.second; + } + + return result; + } + + /*! + @brief compares two JSON pointers for equality + + @param[in] lhs JSON pointer to compare + @param[in] rhs JSON pointer to compare + @return whether @a lhs is equal to @a rhs + + @complexity Linear in the length of the JSON pointer + + @exceptionsafety No-throw guarantee: this function never throws exceptions. + */ + friend bool operator==(json_pointer const& lhs, + json_pointer const& rhs) noexcept + { + return lhs.reference_tokens == rhs.reference_tokens; + } + + /*! + @brief compares two JSON pointers for inequality + + @param[in] lhs JSON pointer to compare + @param[in] rhs JSON pointer to compare + @return whether @a lhs is not equal @a rhs + + @complexity Linear in the length of the JSON pointer + + @exceptionsafety No-throw guarantee: this function never throws exceptions. + */ + friend bool operator!=(json_pointer const& lhs, + json_pointer const& rhs) noexcept + { + return !(lhs == rhs); + } + + /// the reference tokens + std::vector reference_tokens; +}; +} // namespace nlohmann + +// #include + + +#include +#include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template +class json_ref +{ + public: + using value_type = BasicJsonType; + + json_ref(value_type&& value) + : owned_value(std::move(value)) + {} + + json_ref(const value_type& value) + : value_ref(&value) + {} + + json_ref(std::initializer_list init) + : owned_value(init) + {} + + template < + class... Args, + enable_if_t::value, int> = 0 > + json_ref(Args && ... args) + : owned_value(std::forward(args)...) + {} + + // class should be movable only + json_ref(json_ref&&) noexcept = default; + json_ref(const json_ref&) = delete; + json_ref& operator=(const json_ref&) = delete; + json_ref& operator=(json_ref&&) = delete; + ~json_ref() = default; + + value_type moved_or_copied() const + { + if (value_ref == nullptr) + { + return std::move(owned_value); + } + return *value_ref; + } + + value_type const& operator*() const + { + return value_ref ? *value_ref : owned_value; + } + + value_type const* operator->() const + { + return &** this; + } + + private: + mutable value_type owned_value = nullptr; + value_type const* value_ref = nullptr; +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + +// #include + +// #include + +// #include + + +#include // reverse +#include // array +#include // isnan, isinf +#include // uint8_t, uint16_t, uint32_t, uint64_t +#include // memcpy +#include // numeric_limits +#include // string +#include // move + +// #include + +// #include + +// #include + + +#include // copy +#include // size_t +#include // back_inserter +#include // shared_ptr, make_shared +#include // basic_string +#include // vector + +#ifndef JSON_NO_IO + #include // streamsize + #include // basic_ostream +#endif // JSON_NO_IO + +// #include + + +namespace nlohmann +{ +namespace detail +{ +/// abstract output adapter interface +template struct output_adapter_protocol +{ + virtual void write_character(CharType c) = 0; + virtual void write_characters(const CharType* s, std::size_t length) = 0; + virtual ~output_adapter_protocol() = default; + + output_adapter_protocol() = default; + output_adapter_protocol(const output_adapter_protocol&) = default; + output_adapter_protocol(output_adapter_protocol&&) noexcept = default; + output_adapter_protocol& operator=(const output_adapter_protocol&) = default; + output_adapter_protocol& operator=(output_adapter_protocol&&) noexcept = default; +}; + +/// a type to simplify interfaces +template +using output_adapter_t = std::shared_ptr>; + +/// output adapter for byte vectors +template> +class output_vector_adapter : public output_adapter_protocol +{ + public: + explicit output_vector_adapter(std::vector& vec) noexcept + : v(vec) + {} + + void write_character(CharType c) override + { + v.push_back(c); + } + + JSON_HEDLEY_NON_NULL(2) + void write_characters(const CharType* s, std::size_t length) override + { + std::copy(s, s + length, std::back_inserter(v)); + } + + private: + std::vector& v; +}; + +#ifndef JSON_NO_IO +/// output adapter for output streams +template +class output_stream_adapter : public output_adapter_protocol +{ + public: + explicit output_stream_adapter(std::basic_ostream& s) noexcept + : stream(s) + {} + + void write_character(CharType c) override + { + stream.put(c); + } + + JSON_HEDLEY_NON_NULL(2) + void write_characters(const CharType* s, std::size_t length) override + { + stream.write(s, static_cast(length)); + } + + private: + std::basic_ostream& stream; +}; +#endif // JSON_NO_IO + +/// output adapter for basic_string +template> +class output_string_adapter : public output_adapter_protocol +{ + public: + explicit output_string_adapter(StringType& s) noexcept + : str(s) + {} + + void write_character(CharType c) override + { + str.push_back(c); + } + + JSON_HEDLEY_NON_NULL(2) + void write_characters(const CharType* s, std::size_t length) override + { + str.append(s, length); + } + + private: + StringType& str; +}; + +template> +class output_adapter +{ + public: + template> + output_adapter(std::vector& vec) + : oa(std::make_shared>(vec)) {} + +#ifndef JSON_NO_IO + output_adapter(std::basic_ostream& s) + : oa(std::make_shared>(s)) {} +#endif // JSON_NO_IO + + output_adapter(StringType& s) + : oa(std::make_shared>(s)) {} + + operator output_adapter_t() + { + return oa; + } + + private: + output_adapter_t oa = nullptr; +}; +} // namespace detail +} // namespace nlohmann + + +namespace nlohmann +{ +namespace detail +{ +/////////////////// +// binary writer // +/////////////////// + +/*! +@brief serialization to CBOR and MessagePack values +*/ +template +class binary_writer +{ + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + using number_float_t = typename BasicJsonType::number_float_t; + + public: + /*! + @brief create a binary writer + + @param[in] adapter output adapter to write to + */ + explicit binary_writer(output_adapter_t adapter) : oa(std::move(adapter)) + { + JSON_ASSERT(oa); + } + + /*! + @param[in] j JSON value to serialize + @pre j.type() == value_t::object + */ + void write_bson(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::object: + { + write_bson_object(*j.m_value.object); + break; + } + + case value_t::null: + case value_t::array: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + JSON_THROW(type_error::create(317, "to serialize to BSON, top-level type must be object, but is " + std::string(j.type_name()), j)); + } + } + } + + /*! + @param[in] j JSON value to serialize + */ + void write_cbor(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::null: + { + oa->write_character(to_char_type(0xF6)); + break; + } + + case value_t::boolean: + { + oa->write_character(j.m_value.boolean + ? to_char_type(0xF5) + : to_char_type(0xF4)); + break; + } + + case value_t::number_integer: + { + if (j.m_value.number_integer >= 0) + { + // CBOR does not differentiate between positive signed + // integers and unsigned integers. Therefore, we used the + // code from the value_t::number_unsigned case here. + if (j.m_value.number_integer <= 0x17) + { + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x18)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x19)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x1A)); + write_number(static_cast(j.m_value.number_integer)); + } + else + { + oa->write_character(to_char_type(0x1B)); + write_number(static_cast(j.m_value.number_integer)); + } + } + else + { + // The conversions below encode the sign in the first + // byte, and the value is converted to a positive number. + const auto positive_number = -1 - j.m_value.number_integer; + if (j.m_value.number_integer >= -24) + { + write_number(static_cast(0x20 + positive_number)); + } + else if (positive_number <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x38)); + write_number(static_cast(positive_number)); + } + else if (positive_number <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x39)); + write_number(static_cast(positive_number)); + } + else if (positive_number <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x3A)); + write_number(static_cast(positive_number)); + } + else + { + oa->write_character(to_char_type(0x3B)); + write_number(static_cast(positive_number)); + } + } + break; + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned <= 0x17) + { + write_number(static_cast(j.m_value.number_unsigned)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x18)); + write_number(static_cast(j.m_value.number_unsigned)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x19)); + write_number(static_cast(j.m_value.number_unsigned)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x1A)); + write_number(static_cast(j.m_value.number_unsigned)); + } + else + { + oa->write_character(to_char_type(0x1B)); + write_number(static_cast(j.m_value.number_unsigned)); + } + break; + } + + case value_t::number_float: + { + if (std::isnan(j.m_value.number_float)) + { + // NaN is 0xf97e00 in CBOR + oa->write_character(to_char_type(0xF9)); + oa->write_character(to_char_type(0x7E)); + oa->write_character(to_char_type(0x00)); + } + else if (std::isinf(j.m_value.number_float)) + { + // Infinity is 0xf97c00, -Infinity is 0xf9fc00 + oa->write_character(to_char_type(0xf9)); + oa->write_character(j.m_value.number_float > 0 ? to_char_type(0x7C) : to_char_type(0xFC)); + oa->write_character(to_char_type(0x00)); + } + else + { + write_compact_float(j.m_value.number_float, detail::input_format_t::cbor); + } + break; + } + + case value_t::string: + { + // step 1: write control byte and the string length + const auto N = j.m_value.string->size(); + if (N <= 0x17) + { + write_number(static_cast(0x60 + N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x78)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x79)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x7A)); + write_number(static_cast(N)); + } + // LCOV_EXCL_START + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x7B)); + write_number(static_cast(N)); + } + // LCOV_EXCL_STOP + + // step 2: write the string + oa->write_characters( + reinterpret_cast(j.m_value.string->c_str()), + j.m_value.string->size()); + break; + } + + case value_t::array: + { + // step 1: write control byte and the array size + const auto N = j.m_value.array->size(); + if (N <= 0x17) + { + write_number(static_cast(0x80 + N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x98)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x99)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x9A)); + write_number(static_cast(N)); + } + // LCOV_EXCL_START + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x9B)); + write_number(static_cast(N)); + } + // LCOV_EXCL_STOP + + // step 2: write each element + for (const auto& el : *j.m_value.array) + { + write_cbor(el); + } + break; + } + + case value_t::binary: + { + if (j.m_value.binary->has_subtype()) + { + if (j.m_value.binary->subtype() <= (std::numeric_limits::max)()) + { + write_number(static_cast(0xd8)); + write_number(static_cast(j.m_value.binary->subtype())); + } + else if (j.m_value.binary->subtype() <= (std::numeric_limits::max)()) + { + write_number(static_cast(0xd9)); + write_number(static_cast(j.m_value.binary->subtype())); + } + else if (j.m_value.binary->subtype() <= (std::numeric_limits::max)()) + { + write_number(static_cast(0xda)); + write_number(static_cast(j.m_value.binary->subtype())); + } + else if (j.m_value.binary->subtype() <= (std::numeric_limits::max)()) + { + write_number(static_cast(0xdb)); + write_number(static_cast(j.m_value.binary->subtype())); + } + } + + // step 1: write control byte and the binary array size + const auto N = j.m_value.binary->size(); + if (N <= 0x17) + { + write_number(static_cast(0x40 + N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x58)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x59)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x5A)); + write_number(static_cast(N)); + } + // LCOV_EXCL_START + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x5B)); + write_number(static_cast(N)); + } + // LCOV_EXCL_STOP + + // step 2: write each element + oa->write_characters( + reinterpret_cast(j.m_value.binary->data()), + N); + + break; + } + + case value_t::object: + { + // step 1: write control byte and the object size + const auto N = j.m_value.object->size(); + if (N <= 0x17) + { + write_number(static_cast(0xA0 + N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0xB8)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0xB9)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0xBA)); + write_number(static_cast(N)); + } + // LCOV_EXCL_START + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0xBB)); + write_number(static_cast(N)); + } + // LCOV_EXCL_STOP + + // step 2: write each element + for (const auto& el : *j.m_value.object) + { + write_cbor(el.first); + write_cbor(el.second); + } + break; + } + + case value_t::discarded: + default: + break; + } + } + + /*! + @param[in] j JSON value to serialize + */ + void write_msgpack(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::null: // nil + { + oa->write_character(to_char_type(0xC0)); + break; + } + + case value_t::boolean: // true and false + { + oa->write_character(j.m_value.boolean + ? to_char_type(0xC3) + : to_char_type(0xC2)); + break; + } + + case value_t::number_integer: + { + if (j.m_value.number_integer >= 0) + { + // MessagePack does not differentiate between positive + // signed integers and unsigned integers. Therefore, we used + // the code from the value_t::number_unsigned case here. + if (j.m_value.number_unsigned < 128) + { + // positive fixnum + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 8 + oa->write_character(to_char_type(0xCC)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 16 + oa->write_character(to_char_type(0xCD)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 32 + oa->write_character(to_char_type(0xCE)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 64 + oa->write_character(to_char_type(0xCF)); + write_number(static_cast(j.m_value.number_integer)); + } + } + else + { + if (j.m_value.number_integer >= -32) + { + // negative fixnum + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer >= (std::numeric_limits::min)() && + j.m_value.number_integer <= (std::numeric_limits::max)()) + { + // int 8 + oa->write_character(to_char_type(0xD0)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer >= (std::numeric_limits::min)() && + j.m_value.number_integer <= (std::numeric_limits::max)()) + { + // int 16 + oa->write_character(to_char_type(0xD1)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer >= (std::numeric_limits::min)() && + j.m_value.number_integer <= (std::numeric_limits::max)()) + { + // int 32 + oa->write_character(to_char_type(0xD2)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer >= (std::numeric_limits::min)() && + j.m_value.number_integer <= (std::numeric_limits::max)()) + { + // int 64 + oa->write_character(to_char_type(0xD3)); + write_number(static_cast(j.m_value.number_integer)); + } + } + break; + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned < 128) + { + // positive fixnum + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 8 + oa->write_character(to_char_type(0xCC)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 16 + oa->write_character(to_char_type(0xCD)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 32 + oa->write_character(to_char_type(0xCE)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 64 + oa->write_character(to_char_type(0xCF)); + write_number(static_cast(j.m_value.number_integer)); + } + break; + } + + case value_t::number_float: + { + write_compact_float(j.m_value.number_float, detail::input_format_t::msgpack); + break; + } + + case value_t::string: + { + // step 1: write control byte and the string length + const auto N = j.m_value.string->size(); + if (N <= 31) + { + // fixstr + write_number(static_cast(0xA0 | N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // str 8 + oa->write_character(to_char_type(0xD9)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // str 16 + oa->write_character(to_char_type(0xDA)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // str 32 + oa->write_character(to_char_type(0xDB)); + write_number(static_cast(N)); + } + + // step 2: write the string + oa->write_characters( + reinterpret_cast(j.m_value.string->c_str()), + j.m_value.string->size()); + break; + } + + case value_t::array: + { + // step 1: write control byte and the array size + const auto N = j.m_value.array->size(); + if (N <= 15) + { + // fixarray + write_number(static_cast(0x90 | N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // array 16 + oa->write_character(to_char_type(0xDC)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // array 32 + oa->write_character(to_char_type(0xDD)); + write_number(static_cast(N)); + } + + // step 2: write each element + for (const auto& el : *j.m_value.array) + { + write_msgpack(el); + } + break; + } + + case value_t::binary: + { + // step 0: determine if the binary type has a set subtype to + // determine whether or not to use the ext or fixext types + const bool use_ext = j.m_value.binary->has_subtype(); + + // step 1: write control byte and the byte string length + const auto N = j.m_value.binary->size(); + if (N <= (std::numeric_limits::max)()) + { + std::uint8_t output_type{}; + bool fixed = true; + if (use_ext) + { + switch (N) + { + case 1: + output_type = 0xD4; // fixext 1 + break; + case 2: + output_type = 0xD5; // fixext 2 + break; + case 4: + output_type = 0xD6; // fixext 4 + break; + case 8: + output_type = 0xD7; // fixext 8 + break; + case 16: + output_type = 0xD8; // fixext 16 + break; + default: + output_type = 0xC7; // ext 8 + fixed = false; + break; + } + + } + else + { + output_type = 0xC4; // bin 8 + fixed = false; + } + + oa->write_character(to_char_type(output_type)); + if (!fixed) + { + write_number(static_cast(N)); + } + } + else if (N <= (std::numeric_limits::max)()) + { + std::uint8_t output_type = use_ext + ? 0xC8 // ext 16 + : 0xC5; // bin 16 + + oa->write_character(to_char_type(output_type)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + std::uint8_t output_type = use_ext + ? 0xC9 // ext 32 + : 0xC6; // bin 32 + + oa->write_character(to_char_type(output_type)); + write_number(static_cast(N)); + } + + // step 1.5: if this is an ext type, write the subtype + if (use_ext) + { + write_number(static_cast(j.m_value.binary->subtype())); + } + + // step 2: write the byte string + oa->write_characters( + reinterpret_cast(j.m_value.binary->data()), + N); + + break; + } + + case value_t::object: + { + // step 1: write control byte and the object size + const auto N = j.m_value.object->size(); + if (N <= 15) + { + // fixmap + write_number(static_cast(0x80 | (N & 0xF))); + } + else if (N <= (std::numeric_limits::max)()) + { + // map 16 + oa->write_character(to_char_type(0xDE)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // map 32 + oa->write_character(to_char_type(0xDF)); + write_number(static_cast(N)); + } + + // step 2: write each element + for (const auto& el : *j.m_value.object) + { + write_msgpack(el.first); + write_msgpack(el.second); + } + break; + } + + case value_t::discarded: + default: + break; + } + } + + /*! + @param[in] j JSON value to serialize + @param[in] use_count whether to use '#' prefixes (optimized format) + @param[in] use_type whether to use '$' prefixes (optimized format) + @param[in] add_prefix whether prefixes need to be used for this value + */ + void write_ubjson(const BasicJsonType& j, const bool use_count, + const bool use_type, const bool add_prefix = true) + { + switch (j.type()) + { + case value_t::null: + { + if (add_prefix) + { + oa->write_character(to_char_type('Z')); + } + break; + } + + case value_t::boolean: + { + if (add_prefix) + { + oa->write_character(j.m_value.boolean + ? to_char_type('T') + : to_char_type('F')); + } + break; + } + + case value_t::number_integer: + { + write_number_with_ubjson_prefix(j.m_value.number_integer, add_prefix); + break; + } + + case value_t::number_unsigned: + { + write_number_with_ubjson_prefix(j.m_value.number_unsigned, add_prefix); + break; + } + + case value_t::number_float: + { + write_number_with_ubjson_prefix(j.m_value.number_float, add_prefix); + break; + } + + case value_t::string: + { + if (add_prefix) + { + oa->write_character(to_char_type('S')); + } + write_number_with_ubjson_prefix(j.m_value.string->size(), true); + oa->write_characters( + reinterpret_cast(j.m_value.string->c_str()), + j.m_value.string->size()); + break; + } + + case value_t::array: + { + if (add_prefix) + { + oa->write_character(to_char_type('[')); + } + + bool prefix_required = true; + if (use_type && !j.m_value.array->empty()) + { + JSON_ASSERT(use_count); + const CharType first_prefix = ubjson_prefix(j.front()); + const bool same_prefix = std::all_of(j.begin() + 1, j.end(), + [this, first_prefix](const BasicJsonType & v) + { + return ubjson_prefix(v) == first_prefix; + }); + + if (same_prefix) + { + prefix_required = false; + oa->write_character(to_char_type('$')); + oa->write_character(first_prefix); + } + } + + if (use_count) + { + oa->write_character(to_char_type('#')); + write_number_with_ubjson_prefix(j.m_value.array->size(), true); + } + + for (const auto& el : *j.m_value.array) + { + write_ubjson(el, use_count, use_type, prefix_required); + } + + if (!use_count) + { + oa->write_character(to_char_type(']')); + } + + break; + } + + case value_t::binary: + { + if (add_prefix) + { + oa->write_character(to_char_type('[')); + } + + if (use_type && !j.m_value.binary->empty()) + { + JSON_ASSERT(use_count); + oa->write_character(to_char_type('$')); + oa->write_character('U'); + } + + if (use_count) + { + oa->write_character(to_char_type('#')); + write_number_with_ubjson_prefix(j.m_value.binary->size(), true); + } + + if (use_type) + { + oa->write_characters( + reinterpret_cast(j.m_value.binary->data()), + j.m_value.binary->size()); + } + else + { + for (size_t i = 0; i < j.m_value.binary->size(); ++i) + { + oa->write_character(to_char_type('U')); + oa->write_character(j.m_value.binary->data()[i]); + } + } + + if (!use_count) + { + oa->write_character(to_char_type(']')); + } + + break; + } + + case value_t::object: + { + if (add_prefix) + { + oa->write_character(to_char_type('{')); + } + + bool prefix_required = true; + if (use_type && !j.m_value.object->empty()) + { + JSON_ASSERT(use_count); + const CharType first_prefix = ubjson_prefix(j.front()); + const bool same_prefix = std::all_of(j.begin(), j.end(), + [this, first_prefix](const BasicJsonType & v) + { + return ubjson_prefix(v) == first_prefix; + }); + + if (same_prefix) + { + prefix_required = false; + oa->write_character(to_char_type('$')); + oa->write_character(first_prefix); + } + } + + if (use_count) + { + oa->write_character(to_char_type('#')); + write_number_with_ubjson_prefix(j.m_value.object->size(), true); + } + + for (const auto& el : *j.m_value.object) + { + write_number_with_ubjson_prefix(el.first.size(), true); + oa->write_characters( + reinterpret_cast(el.first.c_str()), + el.first.size()); + write_ubjson(el.second, use_count, use_type, prefix_required); + } + + if (!use_count) + { + oa->write_character(to_char_type('}')); + } + + break; + } + + case value_t::discarded: + default: + break; + } + } + + private: + ////////// + // BSON // + ////////// + + /*! + @return The size of a BSON document entry header, including the id marker + and the entry name size (and its null-terminator). + */ + static std::size_t calc_bson_entry_header_size(const string_t& name, const BasicJsonType& j) + { + const auto it = name.find(static_cast(0)); + if (JSON_HEDLEY_UNLIKELY(it != BasicJsonType::string_t::npos)) + { + JSON_THROW(out_of_range::create(409, "BSON key cannot contain code point U+0000 (at byte " + std::to_string(it) + ")", j)); + static_cast(j); + } + + return /*id*/ 1ul + name.size() + /*zero-terminator*/1u; + } + + /*! + @brief Writes the given @a element_type and @a name to the output adapter + */ + void write_bson_entry_header(const string_t& name, + const std::uint8_t element_type) + { + oa->write_character(to_char_type(element_type)); // boolean + oa->write_characters( + reinterpret_cast(name.c_str()), + name.size() + 1u); + } + + /*! + @brief Writes a BSON element with key @a name and boolean value @a value + */ + void write_bson_boolean(const string_t& name, + const bool value) + { + write_bson_entry_header(name, 0x08); + oa->write_character(value ? to_char_type(0x01) : to_char_type(0x00)); + } + + /*! + @brief Writes a BSON element with key @a name and double value @a value + */ + void write_bson_double(const string_t& name, + const double value) + { + write_bson_entry_header(name, 0x01); + write_number(value); + } + + /*! + @return The size of the BSON-encoded string in @a value + */ + static std::size_t calc_bson_string_size(const string_t& value) + { + return sizeof(std::int32_t) + value.size() + 1ul; + } + + /*! + @brief Writes a BSON element with key @a name and string value @a value + */ + void write_bson_string(const string_t& name, + const string_t& value) + { + write_bson_entry_header(name, 0x02); + + write_number(static_cast(value.size() + 1ul)); + oa->write_characters( + reinterpret_cast(value.c_str()), + value.size() + 1); + } + + /*! + @brief Writes a BSON element with key @a name and null value + */ + void write_bson_null(const string_t& name) + { + write_bson_entry_header(name, 0x0A); + } + + /*! + @return The size of the BSON-encoded integer @a value + */ + static std::size_t calc_bson_integer_size(const std::int64_t value) + { + return (std::numeric_limits::min)() <= value && value <= (std::numeric_limits::max)() + ? sizeof(std::int32_t) + : sizeof(std::int64_t); + } + + /*! + @brief Writes a BSON element with key @a name and integer @a value + */ + void write_bson_integer(const string_t& name, + const std::int64_t value) + { + if ((std::numeric_limits::min)() <= value && value <= (std::numeric_limits::max)()) + { + write_bson_entry_header(name, 0x10); // int32 + write_number(static_cast(value)); + } + else + { + write_bson_entry_header(name, 0x12); // int64 + write_number(static_cast(value)); + } + } + + /*! + @return The size of the BSON-encoded unsigned integer in @a j + */ + static constexpr std::size_t calc_bson_unsigned_size(const std::uint64_t value) noexcept + { + return (value <= static_cast((std::numeric_limits::max)())) + ? sizeof(std::int32_t) + : sizeof(std::int64_t); + } + + /*! + @brief Writes a BSON element with key @a name and unsigned @a value + */ + void write_bson_unsigned(const string_t& name, + const BasicJsonType& j) + { + if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + write_bson_entry_header(name, 0x10 /* int32 */); + write_number(static_cast(j.m_value.number_unsigned)); + } + else if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + write_bson_entry_header(name, 0x12 /* int64 */); + write_number(static_cast(j.m_value.number_unsigned)); + } + else + { + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(j.m_value.number_unsigned) + " cannot be represented by BSON as it does not fit int64", j)); + } + } + + /*! + @brief Writes a BSON element with key @a name and object @a value + */ + void write_bson_object_entry(const string_t& name, + const typename BasicJsonType::object_t& value) + { + write_bson_entry_header(name, 0x03); // object + write_bson_object(value); + } + + /*! + @return The size of the BSON-encoded array @a value + */ + static std::size_t calc_bson_array_size(const typename BasicJsonType::array_t& value) + { + std::size_t array_index = 0ul; + + const std::size_t embedded_document_size = std::accumulate(std::begin(value), std::end(value), static_cast(0), [&array_index](std::size_t result, const typename BasicJsonType::array_t::value_type & el) + { + return result + calc_bson_element_size(std::to_string(array_index++), el); + }); + + return sizeof(std::int32_t) + embedded_document_size + 1ul; + } + + /*! + @return The size of the BSON-encoded binary array @a value + */ + static std::size_t calc_bson_binary_size(const typename BasicJsonType::binary_t& value) + { + return sizeof(std::int32_t) + value.size() + 1ul; + } + + /*! + @brief Writes a BSON element with key @a name and array @a value + */ + void write_bson_array(const string_t& name, + const typename BasicJsonType::array_t& value) + { + write_bson_entry_header(name, 0x04); // array + write_number(static_cast(calc_bson_array_size(value))); + + std::size_t array_index = 0ul; + + for (const auto& el : value) + { + write_bson_element(std::to_string(array_index++), el); + } + + oa->write_character(to_char_type(0x00)); + } + + /*! + @brief Writes a BSON element with key @a name and binary value @a value + */ + void write_bson_binary(const string_t& name, + const binary_t& value) + { + write_bson_entry_header(name, 0x05); + + write_number(static_cast(value.size())); + write_number(value.has_subtype() ? static_cast(value.subtype()) : static_cast(0x00)); + + oa->write_characters(reinterpret_cast(value.data()), value.size()); + } + + /*! + @brief Calculates the size necessary to serialize the JSON value @a j with its @a name + @return The calculated size for the BSON document entry for @a j with the given @a name. + */ + static std::size_t calc_bson_element_size(const string_t& name, + const BasicJsonType& j) + { + const auto header_size = calc_bson_entry_header_size(name, j); + switch (j.type()) + { + case value_t::object: + return header_size + calc_bson_object_size(*j.m_value.object); + + case value_t::array: + return header_size + calc_bson_array_size(*j.m_value.array); + + case value_t::binary: + return header_size + calc_bson_binary_size(*j.m_value.binary); + + case value_t::boolean: + return header_size + 1ul; + + case value_t::number_float: + return header_size + 8ul; + + case value_t::number_integer: + return header_size + calc_bson_integer_size(j.m_value.number_integer); + + case value_t::number_unsigned: + return header_size + calc_bson_unsigned_size(j.m_value.number_unsigned); + + case value_t::string: + return header_size + calc_bson_string_size(*j.m_value.string); + + case value_t::null: + return header_size + 0ul; + + // LCOV_EXCL_START + case value_t::discarded: + default: + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) + return 0ul; + // LCOV_EXCL_STOP + } + } + + /*! + @brief Serializes the JSON value @a j to BSON and associates it with the + key @a name. + @param name The name to associate with the JSON entity @a j within the + current BSON document + */ + void write_bson_element(const string_t& name, + const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::object: + return write_bson_object_entry(name, *j.m_value.object); + + case value_t::array: + return write_bson_array(name, *j.m_value.array); + + case value_t::binary: + return write_bson_binary(name, *j.m_value.binary); + + case value_t::boolean: + return write_bson_boolean(name, j.m_value.boolean); + + case value_t::number_float: + return write_bson_double(name, j.m_value.number_float); + + case value_t::number_integer: + return write_bson_integer(name, j.m_value.number_integer); + + case value_t::number_unsigned: + return write_bson_unsigned(name, j); + + case value_t::string: + return write_bson_string(name, *j.m_value.string); + + case value_t::null: + return write_bson_null(name); + + // LCOV_EXCL_START + case value_t::discarded: + default: + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) + return; + // LCOV_EXCL_STOP + } + } + + /*! + @brief Calculates the size of the BSON serialization of the given + JSON-object @a j. + @param[in] value JSON value to serialize + @pre value.type() == value_t::object + */ + static std::size_t calc_bson_object_size(const typename BasicJsonType::object_t& value) + { + std::size_t document_size = std::accumulate(value.begin(), value.end(), static_cast(0), + [](size_t result, const typename BasicJsonType::object_t::value_type & el) + { + return result += calc_bson_element_size(el.first, el.second); + }); + + return sizeof(std::int32_t) + document_size + 1ul; + } + + /*! + @param[in] value JSON value to serialize + @pre value.type() == value_t::object + */ + void write_bson_object(const typename BasicJsonType::object_t& value) + { + write_number(static_cast(calc_bson_object_size(value))); + + for (const auto& el : value) + { + write_bson_element(el.first, el.second); + } + + oa->write_character(to_char_type(0x00)); + } + + ////////// + // CBOR // + ////////// + + static constexpr CharType get_cbor_float_prefix(float /*unused*/) + { + return to_char_type(0xFA); // Single-Precision Float + } + + static constexpr CharType get_cbor_float_prefix(double /*unused*/) + { + return to_char_type(0xFB); // Double-Precision Float + } + + ///////////// + // MsgPack // + ///////////// + + static constexpr CharType get_msgpack_float_prefix(float /*unused*/) + { + return to_char_type(0xCA); // float 32 + } + + static constexpr CharType get_msgpack_float_prefix(double /*unused*/) + { + return to_char_type(0xCB); // float 64 + } + + //////////// + // UBJSON // + //////////// + + // UBJSON: write number (floating point) + template::value, int>::type = 0> + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if (add_prefix) + { + oa->write_character(get_ubjson_float_prefix(n)); + } + write_number(n); + } + + // UBJSON: write number (unsigned integer) + template::value, int>::type = 0> + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if (n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('i')); // int8 + } + write_number(static_cast(n)); + } + else if (n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('U')); // uint8 + } + write_number(static_cast(n)); + } + else if (n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('I')); // int16 + } + write_number(static_cast(n)); + } + else if (n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('l')); // int32 + } + write_number(static_cast(n)); + } + else if (n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('L')); // int64 + } + write_number(static_cast(n)); + } + else + { + if (add_prefix) + { + oa->write_character(to_char_type('H')); // high-precision number + } + + const auto number = BasicJsonType(n).dump(); + write_number_with_ubjson_prefix(number.size(), true); + for (std::size_t i = 0; i < number.size(); ++i) + { + oa->write_character(to_char_type(static_cast(number[i]))); + } + } + } + + // UBJSON: write number (signed integer) + template < typename NumberType, typename std::enable_if < + std::is_signed::value&& + !std::is_floating_point::value, int >::type = 0 > + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if ((std::numeric_limits::min)() <= n && n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('i')); // int8 + } + write_number(static_cast(n)); + } + else if (static_cast((std::numeric_limits::min)()) <= n && n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('U')); // uint8 + } + write_number(static_cast(n)); + } + else if ((std::numeric_limits::min)() <= n && n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('I')); // int16 + } + write_number(static_cast(n)); + } + else if ((std::numeric_limits::min)() <= n && n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('l')); // int32 + } + write_number(static_cast(n)); + } + else if ((std::numeric_limits::min)() <= n && n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('L')); // int64 + } + write_number(static_cast(n)); + } + // LCOV_EXCL_START + else + { + if (add_prefix) + { + oa->write_character(to_char_type('H')); // high-precision number + } + + const auto number = BasicJsonType(n).dump(); + write_number_with_ubjson_prefix(number.size(), true); + for (std::size_t i = 0; i < number.size(); ++i) + { + oa->write_character(to_char_type(static_cast(number[i]))); + } + } + // LCOV_EXCL_STOP + } + + /*! + @brief determine the type prefix of container values + */ + CharType ubjson_prefix(const BasicJsonType& j) const noexcept + { + switch (j.type()) + { + case value_t::null: + return 'Z'; + + case value_t::boolean: + return j.m_value.boolean ? 'T' : 'F'; + + case value_t::number_integer: + { + if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'i'; + } + if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'U'; + } + if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'I'; + } + if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'l'; + } + if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'L'; + } + // anything else is treated as high-precision number + return 'H'; // LCOV_EXCL_LINE + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + return 'i'; + } + if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + return 'U'; + } + if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + return 'I'; + } + if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + return 'l'; + } + if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + return 'L'; + } + // anything else is treated as high-precision number + return 'H'; // LCOV_EXCL_LINE + } + + case value_t::number_float: + return get_ubjson_float_prefix(j.m_value.number_float); + + case value_t::string: + return 'S'; + + case value_t::array: // fallthrough + case value_t::binary: + return '['; + + case value_t::object: + return '{'; + + case value_t::discarded: + default: // discarded values + return 'N'; + } + } + + static constexpr CharType get_ubjson_float_prefix(float /*unused*/) + { + return 'd'; // float 32 + } + + static constexpr CharType get_ubjson_float_prefix(double /*unused*/) + { + return 'D'; // float 64 + } + + /////////////////////// + // Utility functions // + /////////////////////// + + /* + @brief write a number to output input + @param[in] n number of type @a NumberType + @tparam NumberType the type of the number + @tparam OutputIsLittleEndian Set to true if output data is + required to be little endian + + @note This function needs to respect the system's endianness, because bytes + in CBOR, MessagePack, and UBJSON are stored in network order (big + endian) and therefore need reordering on little endian systems. + */ + template + void write_number(const NumberType n) + { + // step 1: write number to array of length NumberType + std::array vec{}; + std::memcpy(vec.data(), &n, sizeof(NumberType)); + + // step 2: write array to output (with possible reordering) + if (is_little_endian != OutputIsLittleEndian) + { + // reverse byte order prior to conversion if necessary + std::reverse(vec.begin(), vec.end()); + } + + oa->write_characters(vec.data(), sizeof(NumberType)); + } + + void write_compact_float(const number_float_t n, detail::input_format_t format) + { +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + if (static_cast(n) >= static_cast(std::numeric_limits::lowest()) && + static_cast(n) <= static_cast((std::numeric_limits::max)()) && + static_cast(static_cast(n)) == static_cast(n)) + { + oa->write_character(format == detail::input_format_t::cbor + ? get_cbor_float_prefix(static_cast(n)) + : get_msgpack_float_prefix(static_cast(n))); + write_number(static_cast(n)); + } + else + { + oa->write_character(format == detail::input_format_t::cbor + ? get_cbor_float_prefix(n) + : get_msgpack_float_prefix(n)); + write_number(n); + } +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + } + + public: + // The following to_char_type functions are implement the conversion + // between uint8_t and CharType. In case CharType is not unsigned, + // such a conversion is required to allow values greater than 128. + // See for a discussion. + template < typename C = CharType, + enable_if_t < std::is_signed::value && std::is_signed::value > * = nullptr > + static constexpr CharType to_char_type(std::uint8_t x) noexcept + { + return *reinterpret_cast(&x); + } + + template < typename C = CharType, + enable_if_t < std::is_signed::value && std::is_unsigned::value > * = nullptr > + static CharType to_char_type(std::uint8_t x) noexcept + { + static_assert(sizeof(std::uint8_t) == sizeof(CharType), "size of CharType must be equal to std::uint8_t"); + static_assert(std::is_trivial::value, "CharType must be trivial"); + CharType result; + std::memcpy(&result, &x, sizeof(x)); + return result; + } + + template::value>* = nullptr> + static constexpr CharType to_char_type(std::uint8_t x) noexcept + { + return x; + } + + template < typename InputCharType, typename C = CharType, + enable_if_t < + std::is_signed::value && + std::is_signed::value && + std::is_same::type>::value + > * = nullptr > + static constexpr CharType to_char_type(InputCharType x) noexcept + { + return x; + } + + private: + /// whether we can assume little endianness + const bool is_little_endian = little_endianness(); + + /// the output + output_adapter_t oa = nullptr; +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + + +#include // reverse, remove, fill, find, none_of +#include // array +#include // localeconv, lconv +#include // labs, isfinite, isnan, signbit +#include // size_t, ptrdiff_t +#include // uint8_t +#include // snprintf +#include // numeric_limits +#include // string, char_traits +#include // setfill, setw +#include // is_same +#include // move + +// #include + + +#include // array +#include // signbit, isfinite +#include // intN_t, uintN_t +#include // memcpy, memmove +#include // numeric_limits +#include // conditional + +// #include + + +namespace nlohmann +{ +namespace detail +{ + +/*! +@brief implements the Grisu2 algorithm for binary to decimal floating-point +conversion. + +This implementation is a slightly modified version of the reference +implementation which may be obtained from +http://florian.loitsch.com/publications (bench.tar.gz). + +The code is distributed under the MIT license, Copyright (c) 2009 Florian Loitsch. + +For a detailed description of the algorithm see: + +[1] Loitsch, "Printing Floating-Point Numbers Quickly and Accurately with + Integers", Proceedings of the ACM SIGPLAN 2010 Conference on Programming + Language Design and Implementation, PLDI 2010 +[2] Burger, Dybvig, "Printing Floating-Point Numbers Quickly and Accurately", + Proceedings of the ACM SIGPLAN 1996 Conference on Programming Language + Design and Implementation, PLDI 1996 +*/ +namespace dtoa_impl +{ + +template +Target reinterpret_bits(const Source source) +{ + static_assert(sizeof(Target) == sizeof(Source), "size mismatch"); + + Target target; + std::memcpy(&target, &source, sizeof(Source)); + return target; +} + +struct diyfp // f * 2^e +{ + static constexpr int kPrecision = 64; // = q + + std::uint64_t f = 0; + int e = 0; + + constexpr diyfp(std::uint64_t f_, int e_) noexcept : f(f_), e(e_) {} + + /*! + @brief returns x - y + @pre x.e == y.e and x.f >= y.f + */ + static diyfp sub(const diyfp& x, const diyfp& y) noexcept + { + JSON_ASSERT(x.e == y.e); + JSON_ASSERT(x.f >= y.f); + + return {x.f - y.f, x.e}; + } + + /*! + @brief returns x * y + @note The result is rounded. (Only the upper q bits are returned.) + */ + static diyfp mul(const diyfp& x, const diyfp& y) noexcept + { + static_assert(kPrecision == 64, "internal error"); + + // Computes: + // f = round((x.f * y.f) / 2^q) + // e = x.e + y.e + q + + // Emulate the 64-bit * 64-bit multiplication: + // + // p = u * v + // = (u_lo + 2^32 u_hi) (v_lo + 2^32 v_hi) + // = (u_lo v_lo ) + 2^32 ((u_lo v_hi ) + (u_hi v_lo )) + 2^64 (u_hi v_hi ) + // = (p0 ) + 2^32 ((p1 ) + (p2 )) + 2^64 (p3 ) + // = (p0_lo + 2^32 p0_hi) + 2^32 ((p1_lo + 2^32 p1_hi) + (p2_lo + 2^32 p2_hi)) + 2^64 (p3 ) + // = (p0_lo ) + 2^32 (p0_hi + p1_lo + p2_lo ) + 2^64 (p1_hi + p2_hi + p3) + // = (p0_lo ) + 2^32 (Q ) + 2^64 (H ) + // = (p0_lo ) + 2^32 (Q_lo + 2^32 Q_hi ) + 2^64 (H ) + // + // (Since Q might be larger than 2^32 - 1) + // + // = (p0_lo + 2^32 Q_lo) + 2^64 (Q_hi + H) + // + // (Q_hi + H does not overflow a 64-bit int) + // + // = p_lo + 2^64 p_hi + + const std::uint64_t u_lo = x.f & 0xFFFFFFFFu; + const std::uint64_t u_hi = x.f >> 32u; + const std::uint64_t v_lo = y.f & 0xFFFFFFFFu; + const std::uint64_t v_hi = y.f >> 32u; + + const std::uint64_t p0 = u_lo * v_lo; + const std::uint64_t p1 = u_lo * v_hi; + const std::uint64_t p2 = u_hi * v_lo; + const std::uint64_t p3 = u_hi * v_hi; + + const std::uint64_t p0_hi = p0 >> 32u; + const std::uint64_t p1_lo = p1 & 0xFFFFFFFFu; + const std::uint64_t p1_hi = p1 >> 32u; + const std::uint64_t p2_lo = p2 & 0xFFFFFFFFu; + const std::uint64_t p2_hi = p2 >> 32u; + + std::uint64_t Q = p0_hi + p1_lo + p2_lo; + + // The full product might now be computed as + // + // p_hi = p3 + p2_hi + p1_hi + (Q >> 32) + // p_lo = p0_lo + (Q << 32) + // + // But in this particular case here, the full p_lo is not required. + // Effectively we only need to add the highest bit in p_lo to p_hi (and + // Q_hi + 1 does not overflow). + + Q += std::uint64_t{1} << (64u - 32u - 1u); // round, ties up + + const std::uint64_t h = p3 + p2_hi + p1_hi + (Q >> 32u); + + return {h, x.e + y.e + 64}; + } + + /*! + @brief normalize x such that the significand is >= 2^(q-1) + @pre x.f != 0 + */ + static diyfp normalize(diyfp x) noexcept + { + JSON_ASSERT(x.f != 0); + + while ((x.f >> 63u) == 0) + { + x.f <<= 1u; + x.e--; + } + + return x; + } + + /*! + @brief normalize x such that the result has the exponent E + @pre e >= x.e and the upper e - x.e bits of x.f must be zero. + */ + static diyfp normalize_to(const diyfp& x, const int target_exponent) noexcept + { + const int delta = x.e - target_exponent; + + JSON_ASSERT(delta >= 0); + JSON_ASSERT(((x.f << delta) >> delta) == x.f); + + return {x.f << delta, target_exponent}; + } +}; + +struct boundaries +{ + diyfp w; + diyfp minus; + diyfp plus; +}; + +/*! +Compute the (normalized) diyfp representing the input number 'value' and its +boundaries. + +@pre value must be finite and positive +*/ +template +boundaries compute_boundaries(FloatType value) +{ + JSON_ASSERT(std::isfinite(value)); + JSON_ASSERT(value > 0); + + // Convert the IEEE representation into a diyfp. + // + // If v is denormal: + // value = 0.F * 2^(1 - bias) = ( F) * 2^(1 - bias - (p-1)) + // If v is normalized: + // value = 1.F * 2^(E - bias) = (2^(p-1) + F) * 2^(E - bias - (p-1)) + + static_assert(std::numeric_limits::is_iec559, + "internal error: dtoa_short requires an IEEE-754 floating-point implementation"); + + constexpr int kPrecision = std::numeric_limits::digits; // = p (includes the hidden bit) + constexpr int kBias = std::numeric_limits::max_exponent - 1 + (kPrecision - 1); + constexpr int kMinExp = 1 - kBias; + constexpr std::uint64_t kHiddenBit = std::uint64_t{1} << (kPrecision - 1); // = 2^(p-1) + + using bits_type = typename std::conditional::type; + + const auto bits = static_cast(reinterpret_bits(value)); + const std::uint64_t E = bits >> (kPrecision - 1); + const std::uint64_t F = bits & (kHiddenBit - 1); + + const bool is_denormal = E == 0; + const diyfp v = is_denormal + ? diyfp(F, kMinExp) + : diyfp(F + kHiddenBit, static_cast(E) - kBias); + + // Compute the boundaries m- and m+ of the floating-point value + // v = f * 2^e. + // + // Determine v- and v+, the floating-point predecessor and successor if v, + // respectively. + // + // v- = v - 2^e if f != 2^(p-1) or e == e_min (A) + // = v - 2^(e-1) if f == 2^(p-1) and e > e_min (B) + // + // v+ = v + 2^e + // + // Let m- = (v- + v) / 2 and m+ = (v + v+) / 2. All real numbers _strictly_ + // between m- and m+ round to v, regardless of how the input rounding + // algorithm breaks ties. + // + // ---+-------------+-------------+-------------+-------------+--- (A) + // v- m- v m+ v+ + // + // -----------------+------+------+-------------+-------------+--- (B) + // v- m- v m+ v+ + + const bool lower_boundary_is_closer = F == 0 && E > 1; + const diyfp m_plus = diyfp(2 * v.f + 1, v.e - 1); + const diyfp m_minus = lower_boundary_is_closer + ? diyfp(4 * v.f - 1, v.e - 2) // (B) + : diyfp(2 * v.f - 1, v.e - 1); // (A) + + // Determine the normalized w+ = m+. + const diyfp w_plus = diyfp::normalize(m_plus); + + // Determine w- = m- such that e_(w-) = e_(w+). + const diyfp w_minus = diyfp::normalize_to(m_minus, w_plus.e); + + return {diyfp::normalize(v), w_minus, w_plus}; +} + +// Given normalized diyfp w, Grisu needs to find a (normalized) cached +// power-of-ten c, such that the exponent of the product c * w = f * 2^e lies +// within a certain range [alpha, gamma] (Definition 3.2 from [1]) +// +// alpha <= e = e_c + e_w + q <= gamma +// +// or +// +// f_c * f_w * 2^alpha <= f_c 2^(e_c) * f_w 2^(e_w) * 2^q +// <= f_c * f_w * 2^gamma +// +// Since c and w are normalized, i.e. 2^(q-1) <= f < 2^q, this implies +// +// 2^(q-1) * 2^(q-1) * 2^alpha <= c * w * 2^q < 2^q * 2^q * 2^gamma +// +// or +// +// 2^(q - 2 + alpha) <= c * w < 2^(q + gamma) +// +// The choice of (alpha,gamma) determines the size of the table and the form of +// the digit generation procedure. Using (alpha,gamma)=(-60,-32) works out well +// in practice: +// +// The idea is to cut the number c * w = f * 2^e into two parts, which can be +// processed independently: An integral part p1, and a fractional part p2: +// +// f * 2^e = ( (f div 2^-e) * 2^-e + (f mod 2^-e) ) * 2^e +// = (f div 2^-e) + (f mod 2^-e) * 2^e +// = p1 + p2 * 2^e +// +// The conversion of p1 into decimal form requires a series of divisions and +// modulos by (a power of) 10. These operations are faster for 32-bit than for +// 64-bit integers, so p1 should ideally fit into a 32-bit integer. This can be +// achieved by choosing +// +// -e >= 32 or e <= -32 := gamma +// +// In order to convert the fractional part +// +// p2 * 2^e = p2 / 2^-e = d[-1] / 10^1 + d[-2] / 10^2 + ... +// +// into decimal form, the fraction is repeatedly multiplied by 10 and the digits +// d[-i] are extracted in order: +// +// (10 * p2) div 2^-e = d[-1] +// (10 * p2) mod 2^-e = d[-2] / 10^1 + ... +// +// The multiplication by 10 must not overflow. It is sufficient to choose +// +// 10 * p2 < 16 * p2 = 2^4 * p2 <= 2^64. +// +// Since p2 = f mod 2^-e < 2^-e, +// +// -e <= 60 or e >= -60 := alpha + +constexpr int kAlpha = -60; +constexpr int kGamma = -32; + +struct cached_power // c = f * 2^e ~= 10^k +{ + std::uint64_t f; + int e; + int k; +}; + +/*! +For a normalized diyfp w = f * 2^e, this function returns a (normalized) cached +power-of-ten c = f_c * 2^e_c, such that the exponent of the product w * c +satisfies (Definition 3.2 from [1]) + + alpha <= e_c + e + q <= gamma. +*/ +inline cached_power get_cached_power_for_binary_exponent(int e) +{ + // Now + // + // alpha <= e_c + e + q <= gamma (1) + // ==> f_c * 2^alpha <= c * 2^e * 2^q + // + // and since the c's are normalized, 2^(q-1) <= f_c, + // + // ==> 2^(q - 1 + alpha) <= c * 2^(e + q) + // ==> 2^(alpha - e - 1) <= c + // + // If c were an exact power of ten, i.e. c = 10^k, one may determine k as + // + // k = ceil( log_10( 2^(alpha - e - 1) ) ) + // = ceil( (alpha - e - 1) * log_10(2) ) + // + // From the paper: + // "In theory the result of the procedure could be wrong since c is rounded, + // and the computation itself is approximated [...]. In practice, however, + // this simple function is sufficient." + // + // For IEEE double precision floating-point numbers converted into + // normalized diyfp's w = f * 2^e, with q = 64, + // + // e >= -1022 (min IEEE exponent) + // -52 (p - 1) + // -52 (p - 1, possibly normalize denormal IEEE numbers) + // -11 (normalize the diyfp) + // = -1137 + // + // and + // + // e <= +1023 (max IEEE exponent) + // -52 (p - 1) + // -11 (normalize the diyfp) + // = 960 + // + // This binary exponent range [-1137,960] results in a decimal exponent + // range [-307,324]. One does not need to store a cached power for each + // k in this range. For each such k it suffices to find a cached power + // such that the exponent of the product lies in [alpha,gamma]. + // This implies that the difference of the decimal exponents of adjacent + // table entries must be less than or equal to + // + // floor( (gamma - alpha) * log_10(2) ) = 8. + // + // (A smaller distance gamma-alpha would require a larger table.) + + // NB: + // Actually this function returns c, such that -60 <= e_c + e + 64 <= -34. + + constexpr int kCachedPowersMinDecExp = -300; + constexpr int kCachedPowersDecStep = 8; + + static constexpr std::array kCachedPowers = + { + { + { 0xAB70FE17C79AC6CA, -1060, -300 }, + { 0xFF77B1FCBEBCDC4F, -1034, -292 }, + { 0xBE5691EF416BD60C, -1007, -284 }, + { 0x8DD01FAD907FFC3C, -980, -276 }, + { 0xD3515C2831559A83, -954, -268 }, + { 0x9D71AC8FADA6C9B5, -927, -260 }, + { 0xEA9C227723EE8BCB, -901, -252 }, + { 0xAECC49914078536D, -874, -244 }, + { 0x823C12795DB6CE57, -847, -236 }, + { 0xC21094364DFB5637, -821, -228 }, + { 0x9096EA6F3848984F, -794, -220 }, + { 0xD77485CB25823AC7, -768, -212 }, + { 0xA086CFCD97BF97F4, -741, -204 }, + { 0xEF340A98172AACE5, -715, -196 }, + { 0xB23867FB2A35B28E, -688, -188 }, + { 0x84C8D4DFD2C63F3B, -661, -180 }, + { 0xC5DD44271AD3CDBA, -635, -172 }, + { 0x936B9FCEBB25C996, -608, -164 }, + { 0xDBAC6C247D62A584, -582, -156 }, + { 0xA3AB66580D5FDAF6, -555, -148 }, + { 0xF3E2F893DEC3F126, -529, -140 }, + { 0xB5B5ADA8AAFF80B8, -502, -132 }, + { 0x87625F056C7C4A8B, -475, -124 }, + { 0xC9BCFF6034C13053, -449, -116 }, + { 0x964E858C91BA2655, -422, -108 }, + { 0xDFF9772470297EBD, -396, -100 }, + { 0xA6DFBD9FB8E5B88F, -369, -92 }, + { 0xF8A95FCF88747D94, -343, -84 }, + { 0xB94470938FA89BCF, -316, -76 }, + { 0x8A08F0F8BF0F156B, -289, -68 }, + { 0xCDB02555653131B6, -263, -60 }, + { 0x993FE2C6D07B7FAC, -236, -52 }, + { 0xE45C10C42A2B3B06, -210, -44 }, + { 0xAA242499697392D3, -183, -36 }, + { 0xFD87B5F28300CA0E, -157, -28 }, + { 0xBCE5086492111AEB, -130, -20 }, + { 0x8CBCCC096F5088CC, -103, -12 }, + { 0xD1B71758E219652C, -77, -4 }, + { 0x9C40000000000000, -50, 4 }, + { 0xE8D4A51000000000, -24, 12 }, + { 0xAD78EBC5AC620000, 3, 20 }, + { 0x813F3978F8940984, 30, 28 }, + { 0xC097CE7BC90715B3, 56, 36 }, + { 0x8F7E32CE7BEA5C70, 83, 44 }, + { 0xD5D238A4ABE98068, 109, 52 }, + { 0x9F4F2726179A2245, 136, 60 }, + { 0xED63A231D4C4FB27, 162, 68 }, + { 0xB0DE65388CC8ADA8, 189, 76 }, + { 0x83C7088E1AAB65DB, 216, 84 }, + { 0xC45D1DF942711D9A, 242, 92 }, + { 0x924D692CA61BE758, 269, 100 }, + { 0xDA01EE641A708DEA, 295, 108 }, + { 0xA26DA3999AEF774A, 322, 116 }, + { 0xF209787BB47D6B85, 348, 124 }, + { 0xB454E4A179DD1877, 375, 132 }, + { 0x865B86925B9BC5C2, 402, 140 }, + { 0xC83553C5C8965D3D, 428, 148 }, + { 0x952AB45CFA97A0B3, 455, 156 }, + { 0xDE469FBD99A05FE3, 481, 164 }, + { 0xA59BC234DB398C25, 508, 172 }, + { 0xF6C69A72A3989F5C, 534, 180 }, + { 0xB7DCBF5354E9BECE, 561, 188 }, + { 0x88FCF317F22241E2, 588, 196 }, + { 0xCC20CE9BD35C78A5, 614, 204 }, + { 0x98165AF37B2153DF, 641, 212 }, + { 0xE2A0B5DC971F303A, 667, 220 }, + { 0xA8D9D1535CE3B396, 694, 228 }, + { 0xFB9B7CD9A4A7443C, 720, 236 }, + { 0xBB764C4CA7A44410, 747, 244 }, + { 0x8BAB8EEFB6409C1A, 774, 252 }, + { 0xD01FEF10A657842C, 800, 260 }, + { 0x9B10A4E5E9913129, 827, 268 }, + { 0xE7109BFBA19C0C9D, 853, 276 }, + { 0xAC2820D9623BF429, 880, 284 }, + { 0x80444B5E7AA7CF85, 907, 292 }, + { 0xBF21E44003ACDD2D, 933, 300 }, + { 0x8E679C2F5E44FF8F, 960, 308 }, + { 0xD433179D9C8CB841, 986, 316 }, + { 0x9E19DB92B4E31BA9, 1013, 324 }, + } + }; + + // This computation gives exactly the same results for k as + // k = ceil((kAlpha - e - 1) * 0.30102999566398114) + // for |e| <= 1500, but doesn't require floating-point operations. + // NB: log_10(2) ~= 78913 / 2^18 + JSON_ASSERT(e >= -1500); + JSON_ASSERT(e <= 1500); + const int f = kAlpha - e - 1; + const int k = (f * 78913) / (1 << 18) + static_cast(f > 0); + + const int index = (-kCachedPowersMinDecExp + k + (kCachedPowersDecStep - 1)) / kCachedPowersDecStep; + JSON_ASSERT(index >= 0); + JSON_ASSERT(static_cast(index) < kCachedPowers.size()); + + const cached_power cached = kCachedPowers[static_cast(index)]; + JSON_ASSERT(kAlpha <= cached.e + e + 64); + JSON_ASSERT(kGamma >= cached.e + e + 64); + + return cached; +} + +/*! +For n != 0, returns k, such that pow10 := 10^(k-1) <= n < 10^k. +For n == 0, returns 1 and sets pow10 := 1. +*/ +inline int find_largest_pow10(const std::uint32_t n, std::uint32_t& pow10) +{ + // LCOV_EXCL_START + if (n >= 1000000000) + { + pow10 = 1000000000; + return 10; + } + // LCOV_EXCL_STOP + if (n >= 100000000) + { + pow10 = 100000000; + return 9; + } + if (n >= 10000000) + { + pow10 = 10000000; + return 8; + } + if (n >= 1000000) + { + pow10 = 1000000; + return 7; + } + if (n >= 100000) + { + pow10 = 100000; + return 6; + } + if (n >= 10000) + { + pow10 = 10000; + return 5; + } + if (n >= 1000) + { + pow10 = 1000; + return 4; + } + if (n >= 100) + { + pow10 = 100; + return 3; + } + if (n >= 10) + { + pow10 = 10; + return 2; + } + + pow10 = 1; + return 1; +} + +inline void grisu2_round(char* buf, int len, std::uint64_t dist, std::uint64_t delta, + std::uint64_t rest, std::uint64_t ten_k) +{ + JSON_ASSERT(len >= 1); + JSON_ASSERT(dist <= delta); + JSON_ASSERT(rest <= delta); + JSON_ASSERT(ten_k > 0); + + // <--------------------------- delta ----> + // <---- dist ---------> + // --------------[------------------+-------------------]-------------- + // M- w M+ + // + // ten_k + // <------> + // <---- rest ----> + // --------------[------------------+----+--------------]-------------- + // w V + // = buf * 10^k + // + // ten_k represents a unit-in-the-last-place in the decimal representation + // stored in buf. + // Decrement buf by ten_k while this takes buf closer to w. + + // The tests are written in this order to avoid overflow in unsigned + // integer arithmetic. + + while (rest < dist + && delta - rest >= ten_k + && (rest + ten_k < dist || dist - rest > rest + ten_k - dist)) + { + JSON_ASSERT(buf[len - 1] != '0'); + buf[len - 1]--; + rest += ten_k; + } +} + +/*! +Generates V = buffer * 10^decimal_exponent, such that M- <= V <= M+. +M- and M+ must be normalized and share the same exponent -60 <= e <= -32. +*/ +inline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent, + diyfp M_minus, diyfp w, diyfp M_plus) +{ + static_assert(kAlpha >= -60, "internal error"); + static_assert(kGamma <= -32, "internal error"); + + // Generates the digits (and the exponent) of a decimal floating-point + // number V = buffer * 10^decimal_exponent in the range [M-, M+]. The diyfp's + // w, M- and M+ share the same exponent e, which satisfies alpha <= e <= gamma. + // + // <--------------------------- delta ----> + // <---- dist ---------> + // --------------[------------------+-------------------]-------------- + // M- w M+ + // + // Grisu2 generates the digits of M+ from left to right and stops as soon as + // V is in [M-,M+]. + + JSON_ASSERT(M_plus.e >= kAlpha); + JSON_ASSERT(M_plus.e <= kGamma); + + std::uint64_t delta = diyfp::sub(M_plus, M_minus).f; // (significand of (M+ - M-), implicit exponent is e) + std::uint64_t dist = diyfp::sub(M_plus, w ).f; // (significand of (M+ - w ), implicit exponent is e) + + // Split M+ = f * 2^e into two parts p1 and p2 (note: e < 0): + // + // M+ = f * 2^e + // = ((f div 2^-e) * 2^-e + (f mod 2^-e)) * 2^e + // = ((p1 ) * 2^-e + (p2 )) * 2^e + // = p1 + p2 * 2^e + + const diyfp one(std::uint64_t{1} << -M_plus.e, M_plus.e); + + auto p1 = static_cast(M_plus.f >> -one.e); // p1 = f div 2^-e (Since -e >= 32, p1 fits into a 32-bit int.) + std::uint64_t p2 = M_plus.f & (one.f - 1); // p2 = f mod 2^-e + + // 1) + // + // Generate the digits of the integral part p1 = d[n-1]...d[1]d[0] + + JSON_ASSERT(p1 > 0); + + std::uint32_t pow10{}; + const int k = find_largest_pow10(p1, pow10); + + // 10^(k-1) <= p1 < 10^k, pow10 = 10^(k-1) + // + // p1 = (p1 div 10^(k-1)) * 10^(k-1) + (p1 mod 10^(k-1)) + // = (d[k-1] ) * 10^(k-1) + (p1 mod 10^(k-1)) + // + // M+ = p1 + p2 * 2^e + // = d[k-1] * 10^(k-1) + (p1 mod 10^(k-1)) + p2 * 2^e + // = d[k-1] * 10^(k-1) + ((p1 mod 10^(k-1)) * 2^-e + p2) * 2^e + // = d[k-1] * 10^(k-1) + ( rest) * 2^e + // + // Now generate the digits d[n] of p1 from left to right (n = k-1,...,0) + // + // p1 = d[k-1]...d[n] * 10^n + d[n-1]...d[0] + // + // but stop as soon as + // + // rest * 2^e = (d[n-1]...d[0] * 2^-e + p2) * 2^e <= delta * 2^e + + int n = k; + while (n > 0) + { + // Invariants: + // M+ = buffer * 10^n + (p1 + p2 * 2^e) (buffer = 0 for n = k) + // pow10 = 10^(n-1) <= p1 < 10^n + // + const std::uint32_t d = p1 / pow10; // d = p1 div 10^(n-1) + const std::uint32_t r = p1 % pow10; // r = p1 mod 10^(n-1) + // + // M+ = buffer * 10^n + (d * 10^(n-1) + r) + p2 * 2^e + // = (buffer * 10 + d) * 10^(n-1) + (r + p2 * 2^e) + // + JSON_ASSERT(d <= 9); + buffer[length++] = static_cast('0' + d); // buffer := buffer * 10 + d + // + // M+ = buffer * 10^(n-1) + (r + p2 * 2^e) + // + p1 = r; + n--; + // + // M+ = buffer * 10^n + (p1 + p2 * 2^e) + // pow10 = 10^n + // + + // Now check if enough digits have been generated. + // Compute + // + // p1 + p2 * 2^e = (p1 * 2^-e + p2) * 2^e = rest * 2^e + // + // Note: + // Since rest and delta share the same exponent e, it suffices to + // compare the significands. + const std::uint64_t rest = (std::uint64_t{p1} << -one.e) + p2; + if (rest <= delta) + { + // V = buffer * 10^n, with M- <= V <= M+. + + decimal_exponent += n; + + // We may now just stop. But instead look if the buffer could be + // decremented to bring V closer to w. + // + // pow10 = 10^n is now 1 ulp in the decimal representation V. + // The rounding procedure works with diyfp's with an implicit + // exponent of e. + // + // 10^n = (10^n * 2^-e) * 2^e = ulp * 2^e + // + const std::uint64_t ten_n = std::uint64_t{pow10} << -one.e; + grisu2_round(buffer, length, dist, delta, rest, ten_n); + + return; + } + + pow10 /= 10; + // + // pow10 = 10^(n-1) <= p1 < 10^n + // Invariants restored. + } + + // 2) + // + // The digits of the integral part have been generated: + // + // M+ = d[k-1]...d[1]d[0] + p2 * 2^e + // = buffer + p2 * 2^e + // + // Now generate the digits of the fractional part p2 * 2^e. + // + // Note: + // No decimal point is generated: the exponent is adjusted instead. + // + // p2 actually represents the fraction + // + // p2 * 2^e + // = p2 / 2^-e + // = d[-1] / 10^1 + d[-2] / 10^2 + ... + // + // Now generate the digits d[-m] of p1 from left to right (m = 1,2,...) + // + // p2 * 2^e = d[-1]d[-2]...d[-m] * 10^-m + // + 10^-m * (d[-m-1] / 10^1 + d[-m-2] / 10^2 + ...) + // + // using + // + // 10^m * p2 = ((10^m * p2) div 2^-e) * 2^-e + ((10^m * p2) mod 2^-e) + // = ( d) * 2^-e + ( r) + // + // or + // 10^m * p2 * 2^e = d + r * 2^e + // + // i.e. + // + // M+ = buffer + p2 * 2^e + // = buffer + 10^-m * (d + r * 2^e) + // = (buffer * 10^m + d) * 10^-m + 10^-m * r * 2^e + // + // and stop as soon as 10^-m * r * 2^e <= delta * 2^e + + JSON_ASSERT(p2 > delta); + + int m = 0; + for (;;) + { + // Invariant: + // M+ = buffer * 10^-m + 10^-m * (d[-m-1] / 10 + d[-m-2] / 10^2 + ...) * 2^e + // = buffer * 10^-m + 10^-m * (p2 ) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * (10 * p2) ) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * ((10*p2 div 2^-e) * 2^-e + (10*p2 mod 2^-e)) * 2^e + // + JSON_ASSERT(p2 <= (std::numeric_limits::max)() / 10); + p2 *= 10; + const std::uint64_t d = p2 >> -one.e; // d = (10 * p2) div 2^-e + const std::uint64_t r = p2 & (one.f - 1); // r = (10 * p2) mod 2^-e + // + // M+ = buffer * 10^-m + 10^-m * (1/10 * (d * 2^-e + r) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * (d + r * 2^e)) + // = (buffer * 10 + d) * 10^(-m-1) + 10^(-m-1) * r * 2^e + // + JSON_ASSERT(d <= 9); + buffer[length++] = static_cast('0' + d); // buffer := buffer * 10 + d + // + // M+ = buffer * 10^(-m-1) + 10^(-m-1) * r * 2^e + // + p2 = r; + m++; + // + // M+ = buffer * 10^-m + 10^-m * p2 * 2^e + // Invariant restored. + + // Check if enough digits have been generated. + // + // 10^-m * p2 * 2^e <= delta * 2^e + // p2 * 2^e <= 10^m * delta * 2^e + // p2 <= 10^m * delta + delta *= 10; + dist *= 10; + if (p2 <= delta) + { + break; + } + } + + // V = buffer * 10^-m, with M- <= V <= M+. + + decimal_exponent -= m; + + // 1 ulp in the decimal representation is now 10^-m. + // Since delta and dist are now scaled by 10^m, we need to do the + // same with ulp in order to keep the units in sync. + // + // 10^m * 10^-m = 1 = 2^-e * 2^e = ten_m * 2^e + // + const std::uint64_t ten_m = one.f; + grisu2_round(buffer, length, dist, delta, p2, ten_m); + + // By construction this algorithm generates the shortest possible decimal + // number (Loitsch, Theorem 6.2) which rounds back to w. + // For an input number of precision p, at least + // + // N = 1 + ceil(p * log_10(2)) + // + // decimal digits are sufficient to identify all binary floating-point + // numbers (Matula, "In-and-Out conversions"). + // This implies that the algorithm does not produce more than N decimal + // digits. + // + // N = 17 for p = 53 (IEEE double precision) + // N = 9 for p = 24 (IEEE single precision) +} + +/*! +v = buf * 10^decimal_exponent +len is the length of the buffer (number of decimal digits) +The buffer must be large enough, i.e. >= max_digits10. +*/ +JSON_HEDLEY_NON_NULL(1) +inline void grisu2(char* buf, int& len, int& decimal_exponent, + diyfp m_minus, diyfp v, diyfp m_plus) +{ + JSON_ASSERT(m_plus.e == m_minus.e); + JSON_ASSERT(m_plus.e == v.e); + + // --------(-----------------------+-----------------------)-------- (A) + // m- v m+ + // + // --------------------(-----------+-----------------------)-------- (B) + // m- v m+ + // + // First scale v (and m- and m+) such that the exponent is in the range + // [alpha, gamma]. + + const cached_power cached = get_cached_power_for_binary_exponent(m_plus.e); + + const diyfp c_minus_k(cached.f, cached.e); // = c ~= 10^-k + + // The exponent of the products is = v.e + c_minus_k.e + q and is in the range [alpha,gamma] + const diyfp w = diyfp::mul(v, c_minus_k); + const diyfp w_minus = diyfp::mul(m_minus, c_minus_k); + const diyfp w_plus = diyfp::mul(m_plus, c_minus_k); + + // ----(---+---)---------------(---+---)---------------(---+---)---- + // w- w w+ + // = c*m- = c*v = c*m+ + // + // diyfp::mul rounds its result and c_minus_k is approximated too. w, w- and + // w+ are now off by a small amount. + // In fact: + // + // w - v * 10^k < 1 ulp + // + // To account for this inaccuracy, add resp. subtract 1 ulp. + // + // --------+---[---------------(---+---)---------------]---+-------- + // w- M- w M+ w+ + // + // Now any number in [M-, M+] (bounds included) will round to w when input, + // regardless of how the input rounding algorithm breaks ties. + // + // And digit_gen generates the shortest possible such number in [M-, M+]. + // Note that this does not mean that Grisu2 always generates the shortest + // possible number in the interval (m-, m+). + const diyfp M_minus(w_minus.f + 1, w_minus.e); + const diyfp M_plus (w_plus.f - 1, w_plus.e ); + + decimal_exponent = -cached.k; // = -(-k) = k + + grisu2_digit_gen(buf, len, decimal_exponent, M_minus, w, M_plus); +} + +/*! +v = buf * 10^decimal_exponent +len is the length of the buffer (number of decimal digits) +The buffer must be large enough, i.e. >= max_digits10. +*/ +template +JSON_HEDLEY_NON_NULL(1) +void grisu2(char* buf, int& len, int& decimal_exponent, FloatType value) +{ + static_assert(diyfp::kPrecision >= std::numeric_limits::digits + 3, + "internal error: not enough precision"); + + JSON_ASSERT(std::isfinite(value)); + JSON_ASSERT(value > 0); + + // If the neighbors (and boundaries) of 'value' are always computed for double-precision + // numbers, all float's can be recovered using strtod (and strtof). However, the resulting + // decimal representations are not exactly "short". + // + // The documentation for 'std::to_chars' (https://en.cppreference.com/w/cpp/utility/to_chars) + // says "value is converted to a string as if by std::sprintf in the default ("C") locale" + // and since sprintf promotes floats to doubles, I think this is exactly what 'std::to_chars' + // does. + // On the other hand, the documentation for 'std::to_chars' requires that "parsing the + // representation using the corresponding std::from_chars function recovers value exactly". That + // indicates that single precision floating-point numbers should be recovered using + // 'std::strtof'. + // + // NB: If the neighbors are computed for single-precision numbers, there is a single float + // (7.0385307e-26f) which can't be recovered using strtod. The resulting double precision + // value is off by 1 ulp. +#if 0 + const boundaries w = compute_boundaries(static_cast(value)); +#else + const boundaries w = compute_boundaries(value); +#endif + + grisu2(buf, len, decimal_exponent, w.minus, w.w, w.plus); +} + +/*! +@brief appends a decimal representation of e to buf +@return a pointer to the element following the exponent. +@pre -1000 < e < 1000 +*/ +JSON_HEDLEY_NON_NULL(1) +JSON_HEDLEY_RETURNS_NON_NULL +inline char* append_exponent(char* buf, int e) +{ + JSON_ASSERT(e > -1000); + JSON_ASSERT(e < 1000); + + if (e < 0) + { + e = -e; + *buf++ = '-'; + } + else + { + *buf++ = '+'; + } + + auto k = static_cast(e); + if (k < 10) + { + // Always print at least two digits in the exponent. + // This is for compatibility with printf("%g"). + *buf++ = '0'; + *buf++ = static_cast('0' + k); + } + else if (k < 100) + { + *buf++ = static_cast('0' + k / 10); + k %= 10; + *buf++ = static_cast('0' + k); + } + else + { + *buf++ = static_cast('0' + k / 100); + k %= 100; + *buf++ = static_cast('0' + k / 10); + k %= 10; + *buf++ = static_cast('0' + k); + } + + return buf; +} + +/*! +@brief prettify v = buf * 10^decimal_exponent + +If v is in the range [10^min_exp, 10^max_exp) it will be printed in fixed-point +notation. Otherwise it will be printed in exponential notation. + +@pre min_exp < 0 +@pre max_exp > 0 +*/ +JSON_HEDLEY_NON_NULL(1) +JSON_HEDLEY_RETURNS_NON_NULL +inline char* format_buffer(char* buf, int len, int decimal_exponent, + int min_exp, int max_exp) +{ + JSON_ASSERT(min_exp < 0); + JSON_ASSERT(max_exp > 0); + + const int k = len; + const int n = len + decimal_exponent; + + // v = buf * 10^(n-k) + // k is the length of the buffer (number of decimal digits) + // n is the position of the decimal point relative to the start of the buffer. + + if (k <= n && n <= max_exp) + { + // digits[000] + // len <= max_exp + 2 + + std::memset(buf + k, '0', static_cast(n) - static_cast(k)); + // Make it look like a floating-point number (#362, #378) + buf[n + 0] = '.'; + buf[n + 1] = '0'; + return buf + (static_cast(n) + 2); + } + + if (0 < n && n <= max_exp) + { + // dig.its + // len <= max_digits10 + 1 + + JSON_ASSERT(k > n); + + std::memmove(buf + (static_cast(n) + 1), buf + n, static_cast(k) - static_cast(n)); + buf[n] = '.'; + return buf + (static_cast(k) + 1U); + } + + if (min_exp < n && n <= 0) + { + // 0.[000]digits + // len <= 2 + (-min_exp - 1) + max_digits10 + + std::memmove(buf + (2 + static_cast(-n)), buf, static_cast(k)); + buf[0] = '0'; + buf[1] = '.'; + std::memset(buf + 2, '0', static_cast(-n)); + return buf + (2U + static_cast(-n) + static_cast(k)); + } + + if (k == 1) + { + // dE+123 + // len <= 1 + 5 + + buf += 1; + } + else + { + // d.igitsE+123 + // len <= max_digits10 + 1 + 5 + + std::memmove(buf + 2, buf + 1, static_cast(k) - 1); + buf[1] = '.'; + buf += 1 + static_cast(k); + } + + *buf++ = 'e'; + return append_exponent(buf, n - 1); +} + +} // namespace dtoa_impl + +/*! +@brief generates a decimal representation of the floating-point number value in [first, last). + +The format of the resulting decimal representation is similar to printf's %g +format. Returns an iterator pointing past-the-end of the decimal representation. + +@note The input number must be finite, i.e. NaN's and Inf's are not supported. +@note The buffer must be large enough. +@note The result is NOT null-terminated. +*/ +template +JSON_HEDLEY_NON_NULL(1, 2) +JSON_HEDLEY_RETURNS_NON_NULL +char* to_chars(char* first, const char* last, FloatType value) +{ + static_cast(last); // maybe unused - fix warning + JSON_ASSERT(std::isfinite(value)); + + // Use signbit(value) instead of (value < 0) since signbit works for -0. + if (std::signbit(value)) + { + value = -value; + *first++ = '-'; + } + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + if (value == 0) // +-0 + { + *first++ = '0'; + // Make it look like a floating-point number (#362, #378) + *first++ = '.'; + *first++ = '0'; + return first; + } +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + + JSON_ASSERT(last - first >= std::numeric_limits::max_digits10); + + // Compute v = buffer * 10^decimal_exponent. + // The decimal digits are stored in the buffer, which needs to be interpreted + // as an unsigned decimal integer. + // len is the length of the buffer, i.e. the number of decimal digits. + int len = 0; + int decimal_exponent = 0; + dtoa_impl::grisu2(first, len, decimal_exponent, value); + + JSON_ASSERT(len <= std::numeric_limits::max_digits10); + + // Format the buffer like printf("%.*g", prec, value) + constexpr int kMinExp = -4; + // Use digits10 here to increase compatibility with version 2. + constexpr int kMaxExp = std::numeric_limits::digits10; + + JSON_ASSERT(last - first >= kMaxExp + 2); + JSON_ASSERT(last - first >= 2 + (-kMinExp - 1) + std::numeric_limits::max_digits10); + JSON_ASSERT(last - first >= std::numeric_limits::max_digits10 + 6); + + return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp); +} + +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + +// #include + +// #include + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +/////////////////// +// serialization // +/////////////////// + +/// how to treat decoding errors +enum class error_handler_t +{ + strict, ///< throw a type_error exception in case of invalid UTF-8 + replace, ///< replace invalid UTF-8 sequences with U+FFFD + ignore ///< ignore invalid UTF-8 sequences +}; + +template +class serializer +{ + using string_t = typename BasicJsonType::string_t; + using number_float_t = typename BasicJsonType::number_float_t; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using binary_char_t = typename BasicJsonType::binary_t::value_type; + static constexpr std::uint8_t UTF8_ACCEPT = 0; + static constexpr std::uint8_t UTF8_REJECT = 1; + + public: + /*! + @param[in] s output stream to serialize to + @param[in] ichar indentation character to use + @param[in] error_handler_ how to react on decoding errors + */ + serializer(output_adapter_t s, const char ichar, + error_handler_t error_handler_ = error_handler_t::strict) + : o(std::move(s)) + , loc(std::localeconv()) + , thousands_sep(loc->thousands_sep == nullptr ? '\0' : std::char_traits::to_char_type(* (loc->thousands_sep))) + , decimal_point(loc->decimal_point == nullptr ? '\0' : std::char_traits::to_char_type(* (loc->decimal_point))) + , indent_char(ichar) + , indent_string(512, indent_char) + , error_handler(error_handler_) + {} + + // delete because of pointer members + serializer(const serializer&) = delete; + serializer& operator=(const serializer&) = delete; + serializer(serializer&&) = delete; + serializer& operator=(serializer&&) = delete; + ~serializer() = default; + + /*! + @brief internal implementation of the serialization function + + This function is called by the public member function dump and organizes + the serialization internally. The indentation level is propagated as + additional parameter. In case of arrays and objects, the function is + called recursively. + + - strings and object keys are escaped using `escape_string()` + - integer numbers are converted implicitly via `operator<<` + - floating-point numbers are converted to a string using `"%g"` format + - binary values are serialized as objects containing the subtype and the + byte array + + @param[in] val value to serialize + @param[in] pretty_print whether the output shall be pretty-printed + @param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters + in the output are escaped with `\uXXXX` sequences, and the result consists + of ASCII characters only. + @param[in] indent_step the indent level + @param[in] current_indent the current indent level (only used internally) + */ + void dump(const BasicJsonType& val, + const bool pretty_print, + const bool ensure_ascii, + const unsigned int indent_step, + const unsigned int current_indent = 0) + { + switch (val.m_type) + { + case value_t::object: + { + if (val.m_value.object->empty()) + { + o->write_characters("{}", 2); + return; + } + + if (pretty_print) + { + o->write_characters("{\n", 2); + + // variable to hold indentation for recursive calls + const auto new_indent = current_indent + indent_step; + if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent)) + { + indent_string.resize(indent_string.size() * 2, ' '); + } + + // first n-1 elements + auto i = val.m_value.object->cbegin(); + for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i) + { + o->write_characters(indent_string.c_str(), new_indent); + o->write_character('\"'); + dump_escaped(i->first, ensure_ascii); + o->write_characters("\": ", 3); + dump(i->second, true, ensure_ascii, indent_step, new_indent); + o->write_characters(",\n", 2); + } + + // last element + JSON_ASSERT(i != val.m_value.object->cend()); + JSON_ASSERT(std::next(i) == val.m_value.object->cend()); + o->write_characters(indent_string.c_str(), new_indent); + o->write_character('\"'); + dump_escaped(i->first, ensure_ascii); + o->write_characters("\": ", 3); + dump(i->second, true, ensure_ascii, indent_step, new_indent); + + o->write_character('\n'); + o->write_characters(indent_string.c_str(), current_indent); + o->write_character('}'); + } + else + { + o->write_character('{'); + + // first n-1 elements + auto i = val.m_value.object->cbegin(); + for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i) + { + o->write_character('\"'); + dump_escaped(i->first, ensure_ascii); + o->write_characters("\":", 2); + dump(i->second, false, ensure_ascii, indent_step, current_indent); + o->write_character(','); + } + + // last element + JSON_ASSERT(i != val.m_value.object->cend()); + JSON_ASSERT(std::next(i) == val.m_value.object->cend()); + o->write_character('\"'); + dump_escaped(i->first, ensure_ascii); + o->write_characters("\":", 2); + dump(i->second, false, ensure_ascii, indent_step, current_indent); + + o->write_character('}'); + } + + return; + } + + case value_t::array: + { + if (val.m_value.array->empty()) + { + o->write_characters("[]", 2); + return; + } + + if (pretty_print) + { + o->write_characters("[\n", 2); + + // variable to hold indentation for recursive calls + const auto new_indent = current_indent + indent_step; + if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent)) + { + indent_string.resize(indent_string.size() * 2, ' '); + } + + // first n-1 elements + for (auto i = val.m_value.array->cbegin(); + i != val.m_value.array->cend() - 1; ++i) + { + o->write_characters(indent_string.c_str(), new_indent); + dump(*i, true, ensure_ascii, indent_step, new_indent); + o->write_characters(",\n", 2); + } + + // last element + JSON_ASSERT(!val.m_value.array->empty()); + o->write_characters(indent_string.c_str(), new_indent); + dump(val.m_value.array->back(), true, ensure_ascii, indent_step, new_indent); + + o->write_character('\n'); + o->write_characters(indent_string.c_str(), current_indent); + o->write_character(']'); + } + else + { + o->write_character('['); + + // first n-1 elements + for (auto i = val.m_value.array->cbegin(); + i != val.m_value.array->cend() - 1; ++i) + { + dump(*i, false, ensure_ascii, indent_step, current_indent); + o->write_character(','); + } + + // last element + JSON_ASSERT(!val.m_value.array->empty()); + dump(val.m_value.array->back(), false, ensure_ascii, indent_step, current_indent); + + o->write_character(']'); + } + + return; + } + + case value_t::string: + { + o->write_character('\"'); + dump_escaped(*val.m_value.string, ensure_ascii); + o->write_character('\"'); + return; + } + + case value_t::binary: + { + if (pretty_print) + { + o->write_characters("{\n", 2); + + // variable to hold indentation for recursive calls + const auto new_indent = current_indent + indent_step; + if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent)) + { + indent_string.resize(indent_string.size() * 2, ' '); + } + + o->write_characters(indent_string.c_str(), new_indent); + + o->write_characters("\"bytes\": [", 10); + + if (!val.m_value.binary->empty()) + { + for (auto i = val.m_value.binary->cbegin(); + i != val.m_value.binary->cend() - 1; ++i) + { + dump_integer(*i); + o->write_characters(", ", 2); + } + dump_integer(val.m_value.binary->back()); + } + + o->write_characters("],\n", 3); + o->write_characters(indent_string.c_str(), new_indent); + + o->write_characters("\"subtype\": ", 11); + if (val.m_value.binary->has_subtype()) + { + dump_integer(val.m_value.binary->subtype()); + } + else + { + o->write_characters("null", 4); + } + o->write_character('\n'); + o->write_characters(indent_string.c_str(), current_indent); + o->write_character('}'); + } + else + { + o->write_characters("{\"bytes\":[", 10); + + if (!val.m_value.binary->empty()) + { + for (auto i = val.m_value.binary->cbegin(); + i != val.m_value.binary->cend() - 1; ++i) + { + dump_integer(*i); + o->write_character(','); + } + dump_integer(val.m_value.binary->back()); + } + + o->write_characters("],\"subtype\":", 12); + if (val.m_value.binary->has_subtype()) + { + dump_integer(val.m_value.binary->subtype()); + o->write_character('}'); + } + else + { + o->write_characters("null}", 5); + } + } + return; + } + + case value_t::boolean: + { + if (val.m_value.boolean) + { + o->write_characters("true", 4); + } + else + { + o->write_characters("false", 5); + } + return; + } + + case value_t::number_integer: + { + dump_integer(val.m_value.number_integer); + return; + } + + case value_t::number_unsigned: + { + dump_integer(val.m_value.number_unsigned); + return; + } + + case value_t::number_float: + { + dump_float(val.m_value.number_float); + return; + } + + case value_t::discarded: + { + o->write_characters("", 11); + return; + } + + case value_t::null: + { + o->write_characters("null", 4); + return; + } + + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + } + + JSON_PRIVATE_UNLESS_TESTED: + /*! + @brief dump escaped string + + Escape a string by replacing certain special characters by a sequence of an + escape character (backslash) and another character and other control + characters by a sequence of "\u" followed by a four-digit hex + representation. The escaped string is written to output stream @a o. + + @param[in] s the string to escape + @param[in] ensure_ascii whether to escape non-ASCII characters with + \uXXXX sequences + + @complexity Linear in the length of string @a s. + */ + void dump_escaped(const string_t& s, const bool ensure_ascii) + { + std::uint32_t codepoint{}; + std::uint8_t state = UTF8_ACCEPT; + std::size_t bytes = 0; // number of bytes written to string_buffer + + // number of bytes written at the point of the last valid byte + std::size_t bytes_after_last_accept = 0; + std::size_t undumped_chars = 0; + + for (std::size_t i = 0; i < s.size(); ++i) + { + const auto byte = static_cast(s[i]); + + switch (decode(state, codepoint, byte)) + { + case UTF8_ACCEPT: // decode found a new code point + { + switch (codepoint) + { + case 0x08: // backspace + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'b'; + break; + } + + case 0x09: // horizontal tab + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 't'; + break; + } + + case 0x0A: // newline + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'n'; + break; + } + + case 0x0C: // formfeed + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'f'; + break; + } + + case 0x0D: // carriage return + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'r'; + break; + } + + case 0x22: // quotation mark + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = '\"'; + break; + } + + case 0x5C: // reverse solidus + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = '\\'; + break; + } + + default: + { + // escape control characters (0x00..0x1F) or, if + // ensure_ascii parameter is used, non-ASCII characters + if ((codepoint <= 0x1F) || (ensure_ascii && (codepoint >= 0x7F))) + { + if (codepoint <= 0xFFFF) + { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + static_cast((std::snprintf)(string_buffer.data() + bytes, 7, "\\u%04x", + static_cast(codepoint))); + bytes += 6; + } + else + { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + static_cast((std::snprintf)(string_buffer.data() + bytes, 13, "\\u%04x\\u%04x", + static_cast(0xD7C0u + (codepoint >> 10u)), + static_cast(0xDC00u + (codepoint & 0x3FFu)))); + bytes += 12; + } + } + else + { + // copy byte to buffer (all previous bytes + // been copied have in default case above) + string_buffer[bytes++] = s[i]; + } + break; + } + } + + // write buffer and reset index; there must be 13 bytes + // left, as this is the maximal number of bytes to be + // written ("\uxxxx\uxxxx\0") for one code point + if (string_buffer.size() - bytes < 13) + { + o->write_characters(string_buffer.data(), bytes); + bytes = 0; + } + + // remember the byte position of this accept + bytes_after_last_accept = bytes; + undumped_chars = 0; + break; + } + + case UTF8_REJECT: // decode found invalid UTF-8 byte + { + switch (error_handler) + { + case error_handler_t::strict: + { + JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + hex_bytes(byte | 0), BasicJsonType())); + } + + case error_handler_t::ignore: + case error_handler_t::replace: + { + // in case we saw this character the first time, we + // would like to read it again, because the byte + // may be OK for itself, but just not OK for the + // previous sequence + if (undumped_chars > 0) + { + --i; + } + + // reset length buffer to the last accepted index; + // thus removing/ignoring the invalid characters + bytes = bytes_after_last_accept; + + if (error_handler == error_handler_t::replace) + { + // add a replacement character + if (ensure_ascii) + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'u'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'd'; + } + else + { + string_buffer[bytes++] = detail::binary_writer::to_char_type('\xEF'); + string_buffer[bytes++] = detail::binary_writer::to_char_type('\xBF'); + string_buffer[bytes++] = detail::binary_writer::to_char_type('\xBD'); + } + + // write buffer and reset index; there must be 13 bytes + // left, as this is the maximal number of bytes to be + // written ("\uxxxx\uxxxx\0") for one code point + if (string_buffer.size() - bytes < 13) + { + o->write_characters(string_buffer.data(), bytes); + bytes = 0; + } + + bytes_after_last_accept = bytes; + } + + undumped_chars = 0; + + // continue processing the string + state = UTF8_ACCEPT; + break; + } + + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + break; + } + + default: // decode found yet incomplete multi-byte code point + { + if (!ensure_ascii) + { + // code point will not be escaped - copy byte to buffer + string_buffer[bytes++] = s[i]; + } + ++undumped_chars; + break; + } + } + } + + // we finished processing the string + if (JSON_HEDLEY_LIKELY(state == UTF8_ACCEPT)) + { + // write buffer + if (bytes > 0) + { + o->write_characters(string_buffer.data(), bytes); + } + } + else + { + // we finish reading, but do not accept: string was incomplete + switch (error_handler) + { + case error_handler_t::strict: + { + JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + hex_bytes(static_cast(s.back() | 0)), BasicJsonType())); + } + + case error_handler_t::ignore: + { + // write all accepted bytes + o->write_characters(string_buffer.data(), bytes_after_last_accept); + break; + } + + case error_handler_t::replace: + { + // write all accepted bytes + o->write_characters(string_buffer.data(), bytes_after_last_accept); + // add a replacement character + if (ensure_ascii) + { + o->write_characters("\\ufffd", 6); + } + else + { + o->write_characters("\xEF\xBF\xBD", 3); + } + break; + } + + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + } + } + + private: + /*! + @brief count digits + + Count the number of decimal (base 10) digits for an input unsigned integer. + + @param[in] x unsigned integer number to count its digits + @return number of decimal digits + */ + inline unsigned int count_digits(number_unsigned_t x) noexcept + { + unsigned int n_digits = 1; + for (;;) + { + if (x < 10) + { + return n_digits; + } + if (x < 100) + { + return n_digits + 1; + } + if (x < 1000) + { + return n_digits + 2; + } + if (x < 10000) + { + return n_digits + 3; + } + x = x / 10000u; + n_digits += 4; + } + } + + /*! + * @brief convert a byte to a uppercase hex representation + * @param[in] byte byte to represent + * @return representation ("00".."FF") + */ + static std::string hex_bytes(std::uint8_t byte) + { + std::string result = "FF"; + constexpr const char* nibble_to_hex = "0123456789ABCDEF"; + result[0] = nibble_to_hex[byte / 16]; + result[1] = nibble_to_hex[byte % 16]; + return result; + } + + // templates to avoid warnings about useless casts + template ::value, int> = 0> + bool is_negative_number(NumberType x) + { + return x < 0; + } + + template < typename NumberType, enable_if_t ::value, int > = 0 > + bool is_negative_number(NumberType /*unused*/) + { + return false; + } + + /*! + @brief dump an integer + + Dump a given integer to output stream @a o. Works internally with + @a number_buffer. + + @param[in] x integer number (signed or unsigned) to dump + @tparam NumberType either @a number_integer_t or @a number_unsigned_t + */ + template < typename NumberType, detail::enable_if_t < + std::is_integral::value || + std::is_same::value || + std::is_same::value || + std::is_same::value, + int > = 0 > + void dump_integer(NumberType x) + { + static constexpr std::array, 100> digits_to_99 + { + { + {{'0', '0'}}, {{'0', '1'}}, {{'0', '2'}}, {{'0', '3'}}, {{'0', '4'}}, {{'0', '5'}}, {{'0', '6'}}, {{'0', '7'}}, {{'0', '8'}}, {{'0', '9'}}, + {{'1', '0'}}, {{'1', '1'}}, {{'1', '2'}}, {{'1', '3'}}, {{'1', '4'}}, {{'1', '5'}}, {{'1', '6'}}, {{'1', '7'}}, {{'1', '8'}}, {{'1', '9'}}, + {{'2', '0'}}, {{'2', '1'}}, {{'2', '2'}}, {{'2', '3'}}, {{'2', '4'}}, {{'2', '5'}}, {{'2', '6'}}, {{'2', '7'}}, {{'2', '8'}}, {{'2', '9'}}, + {{'3', '0'}}, {{'3', '1'}}, {{'3', '2'}}, {{'3', '3'}}, {{'3', '4'}}, {{'3', '5'}}, {{'3', '6'}}, {{'3', '7'}}, {{'3', '8'}}, {{'3', '9'}}, + {{'4', '0'}}, {{'4', '1'}}, {{'4', '2'}}, {{'4', '3'}}, {{'4', '4'}}, {{'4', '5'}}, {{'4', '6'}}, {{'4', '7'}}, {{'4', '8'}}, {{'4', '9'}}, + {{'5', '0'}}, {{'5', '1'}}, {{'5', '2'}}, {{'5', '3'}}, {{'5', '4'}}, {{'5', '5'}}, {{'5', '6'}}, {{'5', '7'}}, {{'5', '8'}}, {{'5', '9'}}, + {{'6', '0'}}, {{'6', '1'}}, {{'6', '2'}}, {{'6', '3'}}, {{'6', '4'}}, {{'6', '5'}}, {{'6', '6'}}, {{'6', '7'}}, {{'6', '8'}}, {{'6', '9'}}, + {{'7', '0'}}, {{'7', '1'}}, {{'7', '2'}}, {{'7', '3'}}, {{'7', '4'}}, {{'7', '5'}}, {{'7', '6'}}, {{'7', '7'}}, {{'7', '8'}}, {{'7', '9'}}, + {{'8', '0'}}, {{'8', '1'}}, {{'8', '2'}}, {{'8', '3'}}, {{'8', '4'}}, {{'8', '5'}}, {{'8', '6'}}, {{'8', '7'}}, {{'8', '8'}}, {{'8', '9'}}, + {{'9', '0'}}, {{'9', '1'}}, {{'9', '2'}}, {{'9', '3'}}, {{'9', '4'}}, {{'9', '5'}}, {{'9', '6'}}, {{'9', '7'}}, {{'9', '8'}}, {{'9', '9'}}, + } + }; + + // special case for "0" + if (x == 0) + { + o->write_character('0'); + return; + } + + // use a pointer to fill the buffer + auto buffer_ptr = number_buffer.begin(); // NOLINT(llvm-qualified-auto,readability-qualified-auto,cppcoreguidelines-pro-type-vararg,hicpp-vararg) + + number_unsigned_t abs_value; + + unsigned int n_chars{}; + + if (is_negative_number(x)) + { + *buffer_ptr = '-'; + abs_value = remove_sign(static_cast(x)); + + // account one more byte for the minus sign + n_chars = 1 + count_digits(abs_value); + } + else + { + abs_value = static_cast(x); + n_chars = count_digits(abs_value); + } + + // spare 1 byte for '\0' + JSON_ASSERT(n_chars < number_buffer.size() - 1); + + // jump to the end to generate the string from backward, + // so we later avoid reversing the result + buffer_ptr += n_chars; + + // Fast int2ascii implementation inspired by "Fastware" talk by Andrei Alexandrescu + // See: https://www.youtube.com/watch?v=o4-CwDo2zpg + while (abs_value >= 100) + { + const auto digits_index = static_cast((abs_value % 100)); + abs_value /= 100; + *(--buffer_ptr) = digits_to_99[digits_index][1]; + *(--buffer_ptr) = digits_to_99[digits_index][0]; + } + + if (abs_value >= 10) + { + const auto digits_index = static_cast(abs_value); + *(--buffer_ptr) = digits_to_99[digits_index][1]; + *(--buffer_ptr) = digits_to_99[digits_index][0]; + } + else + { + *(--buffer_ptr) = static_cast('0' + abs_value); + } + + o->write_characters(number_buffer.data(), n_chars); + } + + /*! + @brief dump a floating-point number + + Dump a given floating-point number to output stream @a o. Works internally + with @a number_buffer. + + @param[in] x floating-point number to dump + */ + void dump_float(number_float_t x) + { + // NaN / inf + if (!std::isfinite(x)) + { + o->write_characters("null", 4); + return; + } + + // If number_float_t is an IEEE-754 single or double precision number, + // use the Grisu2 algorithm to produce short numbers which are + // guaranteed to round-trip, using strtof and strtod, resp. + // + // NB: The test below works if == . + static constexpr bool is_ieee_single_or_double + = (std::numeric_limits::is_iec559 && std::numeric_limits::digits == 24 && std::numeric_limits::max_exponent == 128) || + (std::numeric_limits::is_iec559 && std::numeric_limits::digits == 53 && std::numeric_limits::max_exponent == 1024); + + dump_float(x, std::integral_constant()); + } + + void dump_float(number_float_t x, std::true_type /*is_ieee_single_or_double*/) + { + auto* begin = number_buffer.data(); + auto* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x); + + o->write_characters(begin, static_cast(end - begin)); + } + + void dump_float(number_float_t x, std::false_type /*is_ieee_single_or_double*/) + { + // get number of digits for a float -> text -> float round-trip + static constexpr auto d = std::numeric_limits::max_digits10; + + // the actual conversion + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + std::ptrdiff_t len = (std::snprintf)(number_buffer.data(), number_buffer.size(), "%.*g", d, x); + + // negative value indicates an error + JSON_ASSERT(len > 0); + // check if buffer was large enough + JSON_ASSERT(static_cast(len) < number_buffer.size()); + + // erase thousands separator + if (thousands_sep != '\0') + { + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::remove returns an iterator, see https://github.com/nlohmann/json/issues/3081 + const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, thousands_sep); + std::fill(end, number_buffer.end(), '\0'); + JSON_ASSERT((end - number_buffer.begin()) <= len); + len = (end - number_buffer.begin()); + } + + // convert decimal point to '.' + if (decimal_point != '\0' && decimal_point != '.') + { + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::find returns an iterator, see https://github.com/nlohmann/json/issues/3081 + const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point); + if (dec_pos != number_buffer.end()) + { + *dec_pos = '.'; + } + } + + o->write_characters(number_buffer.data(), static_cast(len)); + + // determine if we need to append ".0" + const bool value_is_int_like = + std::none_of(number_buffer.begin(), number_buffer.begin() + len + 1, + [](char c) + { + return c == '.' || c == 'e'; + }); + + if (value_is_int_like) + { + o->write_characters(".0", 2); + } + } + + /*! + @brief check whether a string is UTF-8 encoded + + The function checks each byte of a string whether it is UTF-8 encoded. The + result of the check is stored in the @a state parameter. The function must + be called initially with state 0 (accept). State 1 means the string must + be rejected, because the current byte is not allowed. If the string is + completely processed, but the state is non-zero, the string ended + prematurely; that is, the last byte indicated more bytes should have + followed. + + @param[in,out] state the state of the decoding + @param[in,out] codep codepoint (valid only if resulting state is UTF8_ACCEPT) + @param[in] byte next byte to decode + @return new state + + @note The function has been edited: a std::array is used. + + @copyright Copyright (c) 2008-2009 Bjoern Hoehrmann + @sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ + */ + static std::uint8_t decode(std::uint8_t& state, std::uint32_t& codep, const std::uint8_t byte) noexcept + { + static const std::array utf8d = + { + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7F + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9F + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // A0..BF + 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0..DF + 0xA, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // E0..EF + 0xB, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // F0..FF + 0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2 + 1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4 + 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6 + 1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // s7..s8 + } + }; + + JSON_ASSERT(byte < utf8d.size()); + const std::uint8_t type = utf8d[byte]; + + codep = (state != UTF8_ACCEPT) + ? (byte & 0x3fu) | (codep << 6u) + : (0xFFu >> type) & (byte); + + std::size_t index = 256u + static_cast(state) * 16u + static_cast(type); + JSON_ASSERT(index < 400); + state = utf8d[index]; + return state; + } + + /* + * Overload to make the compiler happy while it is instantiating + * dump_integer for number_unsigned_t. + * Must never be called. + */ + number_unsigned_t remove_sign(number_unsigned_t x) + { + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + return x; // LCOV_EXCL_LINE + } + + /* + * Helper function for dump_integer + * + * This function takes a negative signed integer and returns its absolute + * value as unsigned integer. The plus/minus shuffling is necessary as we can + * not directly remove the sign of an arbitrary signed integer as the + * absolute values of INT_MIN and INT_MAX are usually not the same. See + * #1708 for details. + */ + inline number_unsigned_t remove_sign(number_integer_t x) noexcept + { + JSON_ASSERT(x < 0 && x < (std::numeric_limits::max)()); // NOLINT(misc-redundant-expression) + return static_cast(-(x + 1)) + 1; + } + + private: + /// the output of the serializer + output_adapter_t o = nullptr; + + /// a (hopefully) large enough character buffer + std::array number_buffer{{}}; + + /// the locale + const std::lconv* loc = nullptr; + /// the locale's thousand separator character + const char thousands_sep = '\0'; + /// the locale's decimal point character + const char decimal_point = '\0'; + + /// string buffer + std::array string_buffer{{}}; + + /// the indentation character + const char indent_char; + /// the indentation string + string_t indent_string; + + /// error_handler how to react on decoding errors + const error_handler_t error_handler; +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + +// #include + + +#include // less +#include // initializer_list +#include // input_iterator_tag, iterator_traits +#include // allocator +#include // for out_of_range +#include // enable_if, is_convertible +#include // pair +#include // vector + +// #include + + +namespace nlohmann +{ + +/// ordered_map: a minimal map-like container that preserves insertion order +/// for use within nlohmann::basic_json +template , + class Allocator = std::allocator>> + struct ordered_map : std::vector, Allocator> +{ + using key_type = Key; + using mapped_type = T; + using Container = std::vector, Allocator>; + using iterator = typename Container::iterator; + using const_iterator = typename Container::const_iterator; + using size_type = typename Container::size_type; + using value_type = typename Container::value_type; + + // Explicit constructors instead of `using Container::Container` + // otherwise older compilers choke on it (GCC <= 5.5, xcode <= 9.4) + ordered_map(const Allocator& alloc = Allocator()) : Container{alloc} {} + template + ordered_map(It first, It last, const Allocator& alloc = Allocator()) + : Container{first, last, alloc} {} + ordered_map(std::initializer_list init, const Allocator& alloc = Allocator() ) + : Container{init, alloc} {} + + std::pair emplace(const key_type& key, T&& t) + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == key) + { + return {it, false}; + } + } + Container::emplace_back(key, t); + return {--this->end(), true}; + } + + T& operator[](const Key& key) + { + return emplace(key, T{}).first->second; + } + + const T& operator[](const Key& key) const + { + return at(key); + } + + T& at(const Key& key) + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == key) + { + return it->second; + } + } + + JSON_THROW(std::out_of_range("key not found")); + } + + const T& at(const Key& key) const + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == key) + { + return it->second; + } + } + + JSON_THROW(std::out_of_range("key not found")); + } + + size_type erase(const Key& key) + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == key) + { + // Since we cannot move const Keys, re-construct them in place + for (auto next = it; ++next != this->end(); ++it) + { + it->~value_type(); // Destroy but keep allocation + new (&*it) value_type{std::move(*next)}; + } + Container::pop_back(); + return 1; + } + } + return 0; + } + + iterator erase(iterator pos) + { + return erase(pos, std::next(pos)); + } + + iterator erase(iterator first, iterator last) + { + const auto elements_affected = std::distance(first, last); + const auto offset = std::distance(Container::begin(), first); + + // This is the start situation. We need to delete elements_affected + // elements (3 in this example: e, f, g), and need to return an + // iterator past the last deleted element (h in this example). + // Note that offset is the distance from the start of the vector + // to first. We will need this later. + + // [ a, b, c, d, e, f, g, h, i, j ] + // ^ ^ + // first last + + // Since we cannot move const Keys, we re-construct them in place. + // We start at first and re-construct (viz. copy) the elements from + // the back of the vector. Example for first iteration: + + // ,--------. + // v | destroy e and re-construct with h + // [ a, b, c, d, e, f, g, h, i, j ] + // ^ ^ + // it it + elements_affected + + for (auto it = first; std::next(it, elements_affected) != Container::end(); ++it) + { + it->~value_type(); // destroy but keep allocation + new (&*it) value_type{std::move(*std::next(it, elements_affected))}; // "move" next element to it + } + + // [ a, b, c, d, h, i, j, h, i, j ] + // ^ ^ + // first last + + // remove the unneeded elements at the end of the vector + Container::resize(this->size() - static_cast(elements_affected)); + + // [ a, b, c, d, h, i, j ] + // ^ ^ + // first last + + // first is now pointing past the last deleted element, but we cannot + // use this iterator, because it may have been invalidated by the + // resize call. Instead, we can return begin() + offset. + return Container::begin() + offset; + } + + size_type count(const Key& key) const + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == key) + { + return 1; + } + } + return 0; + } + + iterator find(const Key& key) + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == key) + { + return it; + } + } + return Container::end(); + } + + const_iterator find(const Key& key) const + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == key) + { + return it; + } + } + return Container::end(); + } + + std::pair insert( value_type&& value ) + { + return emplace(value.first, std::move(value.second)); + } + + std::pair insert( const value_type& value ) + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == value.first) + { + return {it, false}; + } + } + Container::push_back(value); + return {--this->end(), true}; + } + + template + using require_input_iter = typename std::enable_if::iterator_category, + std::input_iterator_tag>::value>::type; + + template> + void insert(InputIt first, InputIt last) + { + for (auto it = first; it != last; ++it) + { + insert(*it); + } + } +}; + +} // namespace nlohmann + + +#if defined(JSON_HAS_CPP_17) + #include +#endif + +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ + +/*! +@brief a class to store JSON values + +@internal +@invariant The member variables @a m_value and @a m_type have the following +relationship: +- If `m_type == value_t::object`, then `m_value.object != nullptr`. +- If `m_type == value_t::array`, then `m_value.array != nullptr`. +- If `m_type == value_t::string`, then `m_value.string != nullptr`. +The invariants are checked by member function assert_invariant(). + +@note ObjectType trick from https://stackoverflow.com/a/9860911 +@endinternal + +@since version 1.0.0 + +@nosubgrouping +*/ +NLOHMANN_BASIC_JSON_TPL_DECLARATION +class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-special-member-functions) +{ + private: + template friend struct detail::external_constructor; + friend ::nlohmann::json_pointer; + + template + friend class ::nlohmann::detail::parser; + friend ::nlohmann::detail::serializer; + template + friend class ::nlohmann::detail::iter_impl; + template + friend class ::nlohmann::detail::binary_writer; + template + friend class ::nlohmann::detail::binary_reader; + template + friend class ::nlohmann::detail::json_sax_dom_parser; + template + friend class ::nlohmann::detail::json_sax_dom_callback_parser; + friend class ::nlohmann::detail::exception; + + /// workaround type for MSVC + using basic_json_t = NLOHMANN_BASIC_JSON_TPL; + + JSON_PRIVATE_UNLESS_TESTED: + // convenience aliases for types residing in namespace detail; + using lexer = ::nlohmann::detail::lexer_base; + + template + static ::nlohmann::detail::parser parser( + InputAdapterType adapter, + detail::parser_callback_tcb = nullptr, + const bool allow_exceptions = true, + const bool ignore_comments = false + ) + { + return ::nlohmann::detail::parser(std::move(adapter), + std::move(cb), allow_exceptions, ignore_comments); + } + + private: + using primitive_iterator_t = ::nlohmann::detail::primitive_iterator_t; + template + using internal_iterator = ::nlohmann::detail::internal_iterator; + template + using iter_impl = ::nlohmann::detail::iter_impl; + template + using iteration_proxy = ::nlohmann::detail::iteration_proxy; + template using json_reverse_iterator = ::nlohmann::detail::json_reverse_iterator; + + template + using output_adapter_t = ::nlohmann::detail::output_adapter_t; + + template + using binary_reader = ::nlohmann::detail::binary_reader; + template using binary_writer = ::nlohmann::detail::binary_writer; + + JSON_PRIVATE_UNLESS_TESTED: + using serializer = ::nlohmann::detail::serializer; + + public: + using value_t = detail::value_t; + /// JSON Pointer, see @ref nlohmann::json_pointer + using json_pointer = ::nlohmann::json_pointer; + template + using json_serializer = JSONSerializer; + /// how to treat decoding errors + using error_handler_t = detail::error_handler_t; + /// how to treat CBOR tags + using cbor_tag_handler_t = detail::cbor_tag_handler_t; + /// helper type for initializer lists of basic_json values + using initializer_list_t = std::initializer_list>; + + using input_format_t = detail::input_format_t; + /// SAX interface type, see @ref nlohmann::json_sax + using json_sax_t = json_sax; + + //////////////// + // exceptions // + //////////////// + + /// @name exceptions + /// Classes to implement user-defined exceptions. + /// @{ + + using exception = detail::exception; + using parse_error = detail::parse_error; + using invalid_iterator = detail::invalid_iterator; + using type_error = detail::type_error; + using out_of_range = detail::out_of_range; + using other_error = detail::other_error; + + /// @} + + + ///////////////////// + // container types // + ///////////////////// + + /// @name container types + /// The canonic container types to use @ref basic_json like any other STL + /// container. + /// @{ + + /// the type of elements in a basic_json container + using value_type = basic_json; + + /// the type of an element reference + using reference = value_type&; + /// the type of an element const reference + using const_reference = const value_type&; + + /// a type to represent differences between iterators + using difference_type = std::ptrdiff_t; + /// a type to represent container sizes + using size_type = std::size_t; + + /// the allocator type + using allocator_type = AllocatorType; + + /// the type of an element pointer + using pointer = typename std::allocator_traits::pointer; + /// the type of an element const pointer + using const_pointer = typename std::allocator_traits::const_pointer; + + /// an iterator for a basic_json container + using iterator = iter_impl; + /// a const iterator for a basic_json container + using const_iterator = iter_impl; + /// a reverse iterator for a basic_json container + using reverse_iterator = json_reverse_iterator; + /// a const reverse iterator for a basic_json container + using const_reverse_iterator = json_reverse_iterator; + + /// @} + + + /// @brief returns the allocator associated with the container + /// @sa https://json.nlohmann.me/api/basic_json/get_allocator/ + static allocator_type get_allocator() + { + return allocator_type(); + } + + /// @brief returns version information on the library + /// @sa https://json.nlohmann.me/api/basic_json/meta/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json meta() + { + basic_json result; + + result["copyright"] = "(C) 2013-2022 Niels Lohmann"; + result["name"] = "JSON for Modern C++"; + result["url"] = "https://github.com/nlohmann/json"; + result["version"]["string"] = + std::to_string(NLOHMANN_JSON_VERSION_MAJOR) + "." + + std::to_string(NLOHMANN_JSON_VERSION_MINOR) + "." + + std::to_string(NLOHMANN_JSON_VERSION_PATCH); + result["version"]["major"] = NLOHMANN_JSON_VERSION_MAJOR; + result["version"]["minor"] = NLOHMANN_JSON_VERSION_MINOR; + result["version"]["patch"] = NLOHMANN_JSON_VERSION_PATCH; + +#ifdef _WIN32 + result["platform"] = "win32"; +#elif defined __linux__ + result["platform"] = "linux"; +#elif defined __APPLE__ + result["platform"] = "apple"; +#elif defined __unix__ + result["platform"] = "unix"; +#else + result["platform"] = "unknown"; +#endif + +#if defined(__ICC) || defined(__INTEL_COMPILER) + result["compiler"] = {{"family", "icc"}, {"version", __INTEL_COMPILER}}; +#elif defined(__clang__) + result["compiler"] = {{"family", "clang"}, {"version", __clang_version__}}; +#elif defined(__GNUC__) || defined(__GNUG__) + result["compiler"] = {{"family", "gcc"}, {"version", std::to_string(__GNUC__) + "." + std::to_string(__GNUC_MINOR__) + "." + std::to_string(__GNUC_PATCHLEVEL__)}}; +#elif defined(__HP_cc) || defined(__HP_aCC) + result["compiler"] = "hp" +#elif defined(__IBMCPP__) + result["compiler"] = {{"family", "ilecpp"}, {"version", __IBMCPP__}}; +#elif defined(_MSC_VER) + result["compiler"] = {{"family", "msvc"}, {"version", _MSC_VER}}; +#elif defined(__PGI) + result["compiler"] = {{"family", "pgcpp"}, {"version", __PGI}}; +#elif defined(__SUNPRO_CC) + result["compiler"] = {{"family", "sunpro"}, {"version", __SUNPRO_CC}}; +#else + result["compiler"] = {{"family", "unknown"}, {"version", "unknown"}}; +#endif + +#ifdef __cplusplus + result["compiler"]["c++"] = std::to_string(__cplusplus); +#else + result["compiler"]["c++"] = "unknown"; +#endif + return result; + } + + + /////////////////////////// + // JSON value data types // + /////////////////////////// + + /// @name JSON value data types + /// The data types to store a JSON value. These types are derived from + /// the template arguments passed to class @ref basic_json. + /// @{ + + /// @brief object key comparator type + /// @sa https://json.nlohmann.me/api/basic_json/object_comparator_t/ +#if defined(JSON_HAS_CPP_14) + // Use transparent comparator if possible, combined with perfect forwarding + // on find() and count() calls prevents unnecessary string construction. + using object_comparator_t = std::less<>; +#else + using object_comparator_t = std::less; +#endif + + /// @brief a type for an object + /// @sa https://json.nlohmann.me/api/basic_json/object_t/ + using object_t = ObjectType>>; + + /// @brief a type for an array + /// @sa https://json.nlohmann.me/api/basic_json/array_t/ + using array_t = ArrayType>; + + /// @brief a type for a string + /// @sa https://json.nlohmann.me/api/basic_json/string_t/ + using string_t = StringType; + + /// @brief a type for a boolean + /// @sa https://json.nlohmann.me/api/basic_json/boolean_t/ + using boolean_t = BooleanType; + + /// @brief a type for a number (integer) + /// @sa https://json.nlohmann.me/api/basic_json/number_integer_t/ + using number_integer_t = NumberIntegerType; + + /// @brief a type for a number (unsigned) + /// @sa https://json.nlohmann.me/api/basic_json/number_unsigned_t/ + using number_unsigned_t = NumberUnsignedType; + + /// @brief a type for a number (floating-point) + /// @sa https://json.nlohmann.me/api/basic_json/number_float_t/ + using number_float_t = NumberFloatType; + + /// @brief a type for a packed binary type + /// @sa https://json.nlohmann.me/api/basic_json/binary_t/ + using binary_t = nlohmann::byte_container_with_subtype; + + /// @} + + private: + + /// helper for exception-safe object creation + template + JSON_HEDLEY_RETURNS_NON_NULL + static T* create(Args&& ... args) + { + AllocatorType alloc; + using AllocatorTraits = std::allocator_traits>; + + auto deleter = [&](T * obj) + { + AllocatorTraits::deallocate(alloc, obj, 1); + }; + std::unique_ptr obj(AllocatorTraits::allocate(alloc, 1), deleter); + AllocatorTraits::construct(alloc, obj.get(), std::forward(args)...); + JSON_ASSERT(obj != nullptr); + return obj.release(); + } + + //////////////////////// + // JSON value storage // + //////////////////////// + + JSON_PRIVATE_UNLESS_TESTED: + /*! + @brief a JSON value + + The actual storage for a JSON value of the @ref basic_json class. This + union combines the different storage types for the JSON value types + defined in @ref value_t. + + JSON type | value_t type | used type + --------- | --------------- | ------------------------ + object | object | pointer to @ref object_t + array | array | pointer to @ref array_t + string | string | pointer to @ref string_t + boolean | boolean | @ref boolean_t + number | number_integer | @ref number_integer_t + number | number_unsigned | @ref number_unsigned_t + number | number_float | @ref number_float_t + binary | binary | pointer to @ref binary_t + null | null | *no value is stored* + + @note Variable-length types (objects, arrays, and strings) are stored as + pointers. The size of the union should not exceed 64 bits if the default + value types are used. + + @since version 1.0.0 + */ + union json_value + { + /// object (stored with pointer to save storage) + object_t* object; + /// array (stored with pointer to save storage) + array_t* array; + /// string (stored with pointer to save storage) + string_t* string; + /// binary (stored with pointer to save storage) + binary_t* binary; + /// boolean + boolean_t boolean; + /// number (integer) + number_integer_t number_integer; + /// number (unsigned integer) + number_unsigned_t number_unsigned; + /// number (floating-point) + number_float_t number_float; + + /// default constructor (for null values) + json_value() = default; + /// constructor for booleans + json_value(boolean_t v) noexcept : boolean(v) {} + /// constructor for numbers (integer) + json_value(number_integer_t v) noexcept : number_integer(v) {} + /// constructor for numbers (unsigned) + json_value(number_unsigned_t v) noexcept : number_unsigned(v) {} + /// constructor for numbers (floating-point) + json_value(number_float_t v) noexcept : number_float(v) {} + /// constructor for empty values of a given type + json_value(value_t t) + { + switch (t) + { + case value_t::object: + { + object = create(); + break; + } + + case value_t::array: + { + array = create(); + break; + } + + case value_t::string: + { + string = create(""); + break; + } + + case value_t::binary: + { + binary = create(); + break; + } + + case value_t::boolean: + { + boolean = static_cast(false); + break; + } + + case value_t::number_integer: + { + number_integer = static_cast(0); + break; + } + + case value_t::number_unsigned: + { + number_unsigned = static_cast(0); + break; + } + + case value_t::number_float: + { + number_float = static_cast(0.0); + break; + } + + case value_t::null: + { + object = nullptr; // silence warning, see #821 + break; + } + + case value_t::discarded: + default: + { + object = nullptr; // silence warning, see #821 + if (JSON_HEDLEY_UNLIKELY(t == value_t::null)) + { + JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.10.5", basic_json())); // LCOV_EXCL_LINE + } + break; + } + } + } + + /// constructor for strings + json_value(const string_t& value) : string(create(value)) {} + + /// constructor for rvalue strings + json_value(string_t&& value) : string(create(std::move(value))) {} + + /// constructor for objects + json_value(const object_t& value) : object(create(value)) {} + + /// constructor for rvalue objects + json_value(object_t&& value) : object(create(std::move(value))) {} + + /// constructor for arrays + json_value(const array_t& value) : array(create(value)) {} + + /// constructor for rvalue arrays + json_value(array_t&& value) : array(create(std::move(value))) {} + + /// constructor for binary arrays + json_value(const typename binary_t::container_type& value) : binary(create(value)) {} + + /// constructor for rvalue binary arrays + json_value(typename binary_t::container_type&& value) : binary(create(std::move(value))) {} + + /// constructor for binary arrays (internal type) + json_value(const binary_t& value) : binary(create(value)) {} + + /// constructor for rvalue binary arrays (internal type) + json_value(binary_t&& value) : binary(create(std::move(value))) {} + + void destroy(value_t t) + { + if (t == value_t::array || t == value_t::object) + { + // flatten the current json_value to a heap-allocated stack + std::vector stack; + + // move the top-level items to stack + if (t == value_t::array) + { + stack.reserve(array->size()); + std::move(array->begin(), array->end(), std::back_inserter(stack)); + } + else + { + stack.reserve(object->size()); + for (auto&& it : *object) + { + stack.push_back(std::move(it.second)); + } + } + + while (!stack.empty()) + { + // move the last item to local variable to be processed + basic_json current_item(std::move(stack.back())); + stack.pop_back(); + + // if current_item is array/object, move + // its children to the stack to be processed later + if (current_item.is_array()) + { + std::move(current_item.m_value.array->begin(), current_item.m_value.array->end(), std::back_inserter(stack)); + + current_item.m_value.array->clear(); + } + else if (current_item.is_object()) + { + for (auto&& it : *current_item.m_value.object) + { + stack.push_back(std::move(it.second)); + } + + current_item.m_value.object->clear(); + } + + // it's now safe that current_item get destructed + // since it doesn't have any children + } + } + + switch (t) + { + case value_t::object: + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, object); + std::allocator_traits::deallocate(alloc, object, 1); + break; + } + + case value_t::array: + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, array); + std::allocator_traits::deallocate(alloc, array, 1); + break; + } + + case value_t::string: + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, string); + std::allocator_traits::deallocate(alloc, string, 1); + break; + } + + case value_t::binary: + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, binary); + std::allocator_traits::deallocate(alloc, binary, 1); + break; + } + + case value_t::null: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::discarded: + default: + { + break; + } + } + } + }; + + private: + /*! + @brief checks the class invariants + + This function asserts the class invariants. It needs to be called at the + end of every constructor to make sure that created objects respect the + invariant. Furthermore, it has to be called each time the type of a JSON + value is changed, because the invariant expresses a relationship between + @a m_type and @a m_value. + + Furthermore, the parent relation is checked for arrays and objects: If + @a check_parents true and the value is an array or object, then the + container's elements must have the current value as parent. + + @param[in] check_parents whether the parent relation should be checked. + The value is true by default and should only be set to false + during destruction of objects when the invariant does not + need to hold. + */ + void assert_invariant(bool check_parents = true) const noexcept + { + JSON_ASSERT(m_type != value_t::object || m_value.object != nullptr); + JSON_ASSERT(m_type != value_t::array || m_value.array != nullptr); + JSON_ASSERT(m_type != value_t::string || m_value.string != nullptr); + JSON_ASSERT(m_type != value_t::binary || m_value.binary != nullptr); + +#if JSON_DIAGNOSTICS + JSON_TRY + { + // cppcheck-suppress assertWithSideEffect + JSON_ASSERT(!check_parents || !is_structured() || std::all_of(begin(), end(), [this](const basic_json & j) + { + return j.m_parent == this; + })); + } + JSON_CATCH(...) {} // LCOV_EXCL_LINE +#endif + static_cast(check_parents); + } + + void set_parents() + { +#if JSON_DIAGNOSTICS + switch (m_type) + { + case value_t::array: + { + for (auto& element : *m_value.array) + { + element.m_parent = this; + } + break; + } + + case value_t::object: + { + for (auto& element : *m_value.object) + { + element.second.m_parent = this; + } + break; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + break; + } +#endif + } + + iterator set_parents(iterator it, typename iterator::difference_type count_set_parents) + { +#if JSON_DIAGNOSTICS + for (typename iterator::difference_type i = 0; i < count_set_parents; ++i) + { + (it + i)->m_parent = this; + } +#else + static_cast(count_set_parents); +#endif + return it; + } + + reference set_parent(reference j, std::size_t old_capacity = static_cast(-1)) + { +#if JSON_DIAGNOSTICS + if (old_capacity != static_cast(-1)) + { + // see https://github.com/nlohmann/json/issues/2838 + JSON_ASSERT(type() == value_t::array); + if (JSON_HEDLEY_UNLIKELY(m_value.array->capacity() != old_capacity)) + { + // capacity has changed: update all parents + set_parents(); + return j; + } + } + + // ordered_json uses a vector internally, so pointers could have + // been invalidated; see https://github.com/nlohmann/json/issues/2962 +#ifdef JSON_HEDLEY_MSVC_VERSION +#pragma warning(push ) +#pragma warning(disable : 4127) // ignore warning to replace if with if constexpr +#endif + if (detail::is_ordered_map::value) + { + set_parents(); + return j; + } +#ifdef JSON_HEDLEY_MSVC_VERSION +#pragma warning( pop ) +#endif + + j.m_parent = this; +#else + static_cast(j); + static_cast(old_capacity); +#endif + return j; + } + + public: + ////////////////////////// + // JSON parser callback // + ////////////////////////// + + /// @brief parser event types + /// @sa https://json.nlohmann.me/api/basic_json/parse_event_t/ + using parse_event_t = detail::parse_event_t; + + /// @brief per-element parser callback type + /// @sa https://json.nlohmann.me/api/basic_json/parser_callback_t/ + using parser_callback_t = detail::parser_callback_t; + + ////////////////// + // constructors // + ////////////////// + + /// @name constructors and destructors + /// Constructors of class @ref basic_json, copy/move constructor, copy + /// assignment, static functions creating objects, and the destructor. + /// @{ + + /// @brief create an empty value with a given type + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + basic_json(const value_t v) + : m_type(v), m_value(v) + { + assert_invariant(); + } + + /// @brief create a null object + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + basic_json(std::nullptr_t = nullptr) noexcept + : basic_json(value_t::null) + { + assert_invariant(); + } + + /// @brief create a JSON value from compatible types + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + template < typename CompatibleType, + typename U = detail::uncvref_t, + detail::enable_if_t < + !detail::is_basic_json::value && detail::is_compatible_type::value, int > = 0 > + basic_json(CompatibleType && val) noexcept(noexcept( // NOLINT(bugprone-forwarding-reference-overload,bugprone-exception-escape) + JSONSerializer::to_json(std::declval(), + std::forward(val)))) + { + JSONSerializer::to_json(*this, std::forward(val)); + set_parents(); + assert_invariant(); + } + + /// @brief create a JSON value from an existing one + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + template < typename BasicJsonType, + detail::enable_if_t < + detail::is_basic_json::value&& !std::is_same::value, int > = 0 > + basic_json(const BasicJsonType& val) + { + using other_boolean_t = typename BasicJsonType::boolean_t; + using other_number_float_t = typename BasicJsonType::number_float_t; + using other_number_integer_t = typename BasicJsonType::number_integer_t; + using other_number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using other_string_t = typename BasicJsonType::string_t; + using other_object_t = typename BasicJsonType::object_t; + using other_array_t = typename BasicJsonType::array_t; + using other_binary_t = typename BasicJsonType::binary_t; + + switch (val.type()) + { + case value_t::boolean: + JSONSerializer::to_json(*this, val.template get()); + break; + case value_t::number_float: + JSONSerializer::to_json(*this, val.template get()); + break; + case value_t::number_integer: + JSONSerializer::to_json(*this, val.template get()); + break; + case value_t::number_unsigned: + JSONSerializer::to_json(*this, val.template get()); + break; + case value_t::string: + JSONSerializer::to_json(*this, val.template get_ref()); + break; + case value_t::object: + JSONSerializer::to_json(*this, val.template get_ref()); + break; + case value_t::array: + JSONSerializer::to_json(*this, val.template get_ref()); + break; + case value_t::binary: + JSONSerializer::to_json(*this, val.template get_ref()); + break; + case value_t::null: + *this = nullptr; + break; + case value_t::discarded: + m_type = value_t::discarded; + break; + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + set_parents(); + assert_invariant(); + } + + /// @brief create a container (array or object) from an initializer list + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + basic_json(initializer_list_t init, + bool type_deduction = true, + value_t manual_type = value_t::array) + { + // check if each element is an array with two elements whose first + // element is a string + bool is_an_object = std::all_of(init.begin(), init.end(), + [](const detail::json_ref& element_ref) + { + return element_ref->is_array() && element_ref->size() == 2 && (*element_ref)[0].is_string(); + }); + + // adjust type if type deduction is not wanted + if (!type_deduction) + { + // if array is wanted, do not create an object though possible + if (manual_type == value_t::array) + { + is_an_object = false; + } + + // if object is wanted but impossible, throw an exception + if (JSON_HEDLEY_UNLIKELY(manual_type == value_t::object && !is_an_object)) + { + JSON_THROW(type_error::create(301, "cannot create object from initializer list", basic_json())); + } + } + + if (is_an_object) + { + // the initializer list is a list of pairs -> create object + m_type = value_t::object; + m_value = value_t::object; + + for (auto& element_ref : init) + { + auto element = element_ref.moved_or_copied(); + m_value.object->emplace( + std::move(*((*element.m_value.array)[0].m_value.string)), + std::move((*element.m_value.array)[1])); + } + } + else + { + // the initializer list describes an array -> create array + m_type = value_t::array; + m_value.array = create(init.begin(), init.end()); + } + + set_parents(); + assert_invariant(); + } + + /// @brief explicitly create a binary array (without subtype) + /// @sa https://json.nlohmann.me/api/basic_json/binary/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json binary(const typename binary_t::container_type& init) + { + auto res = basic_json(); + res.m_type = value_t::binary; + res.m_value = init; + return res; + } + + /// @brief explicitly create a binary array (with subtype) + /// @sa https://json.nlohmann.me/api/basic_json/binary/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json binary(const typename binary_t::container_type& init, typename binary_t::subtype_type subtype) + { + auto res = basic_json(); + res.m_type = value_t::binary; + res.m_value = binary_t(init, subtype); + return res; + } + + /// @brief explicitly create a binary array + /// @sa https://json.nlohmann.me/api/basic_json/binary/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json binary(typename binary_t::container_type&& init) + { + auto res = basic_json(); + res.m_type = value_t::binary; + res.m_value = std::move(init); + return res; + } + + /// @brief explicitly create a binary array (with subtype) + /// @sa https://json.nlohmann.me/api/basic_json/binary/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json binary(typename binary_t::container_type&& init, typename binary_t::subtype_type subtype) + { + auto res = basic_json(); + res.m_type = value_t::binary; + res.m_value = binary_t(std::move(init), subtype); + return res; + } + + /// @brief explicitly create an array from an initializer list + /// @sa https://json.nlohmann.me/api/basic_json/array/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json array(initializer_list_t init = {}) + { + return basic_json(init, false, value_t::array); + } + + /// @brief explicitly create an object from an initializer list + /// @sa https://json.nlohmann.me/api/basic_json/object/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json object(initializer_list_t init = {}) + { + return basic_json(init, false, value_t::object); + } + + /// @brief construct an array with count copies of given value + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + basic_json(size_type cnt, const basic_json& val) + : m_type(value_t::array) + { + m_value.array = create(cnt, val); + set_parents(); + assert_invariant(); + } + + /// @brief construct a JSON container given an iterator range + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + template < class InputIT, typename std::enable_if < + std::is_same::value || + std::is_same::value, int >::type = 0 > + basic_json(InputIT first, InputIT last) + { + JSON_ASSERT(first.m_object != nullptr); + JSON_ASSERT(last.m_object != nullptr); + + // make sure iterator fits the current value + if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) + { + JSON_THROW(invalid_iterator::create(201, "iterators are not compatible", basic_json())); + } + + // copy type from first iterator + m_type = first.m_object->m_type; + + // check if iterator range is complete for primitive values + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + { + if (JSON_HEDLEY_UNLIKELY(!first.m_it.primitive_iterator.is_begin() + || !last.m_it.primitive_iterator.is_end())) + { + JSON_THROW(invalid_iterator::create(204, "iterators out of range", *first.m_object)); + } + break; + } + + case value_t::null: + case value_t::object: + case value_t::array: + case value_t::binary: + case value_t::discarded: + default: + break; + } + + switch (m_type) + { + case value_t::number_integer: + { + m_value.number_integer = first.m_object->m_value.number_integer; + break; + } + + case value_t::number_unsigned: + { + m_value.number_unsigned = first.m_object->m_value.number_unsigned; + break; + } + + case value_t::number_float: + { + m_value.number_float = first.m_object->m_value.number_float; + break; + } + + case value_t::boolean: + { + m_value.boolean = first.m_object->m_value.boolean; + break; + } + + case value_t::string: + { + m_value = *first.m_object->m_value.string; + break; + } + + case value_t::object: + { + m_value.object = create(first.m_it.object_iterator, + last.m_it.object_iterator); + break; + } + + case value_t::array: + { + m_value.array = create(first.m_it.array_iterator, + last.m_it.array_iterator); + break; + } + + case value_t::binary: + { + m_value = *first.m_object->m_value.binary; + break; + } + + case value_t::null: + case value_t::discarded: + default: + JSON_THROW(invalid_iterator::create(206, "cannot construct with iterators from " + std::string(first.m_object->type_name()), *first.m_object)); + } + + set_parents(); + assert_invariant(); + } + + + /////////////////////////////////////// + // other constructors and destructor // + /////////////////////////////////////// + + template, + std::is_same>::value, int> = 0 > + basic_json(const JsonRef& ref) : basic_json(ref.moved_or_copied()) {} + + /// @brief copy constructor + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + basic_json(const basic_json& other) + : m_type(other.m_type) + { + // check of passed value is valid + other.assert_invariant(); + + switch (m_type) + { + case value_t::object: + { + m_value = *other.m_value.object; + break; + } + + case value_t::array: + { + m_value = *other.m_value.array; + break; + } + + case value_t::string: + { + m_value = *other.m_value.string; + break; + } + + case value_t::boolean: + { + m_value = other.m_value.boolean; + break; + } + + case value_t::number_integer: + { + m_value = other.m_value.number_integer; + break; + } + + case value_t::number_unsigned: + { + m_value = other.m_value.number_unsigned; + break; + } + + case value_t::number_float: + { + m_value = other.m_value.number_float; + break; + } + + case value_t::binary: + { + m_value = *other.m_value.binary; + break; + } + + case value_t::null: + case value_t::discarded: + default: + break; + } + + set_parents(); + assert_invariant(); + } + + /// @brief move constructor + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + basic_json(basic_json&& other) noexcept + : m_type(std::move(other.m_type)), + m_value(std::move(other.m_value)) + { + // check that passed value is valid + other.assert_invariant(false); + + // invalidate payload + other.m_type = value_t::null; + other.m_value = {}; + + set_parents(); + assert_invariant(); + } + + /// @brief copy assignment + /// @sa https://json.nlohmann.me/api/basic_json/operator=/ + basic_json& operator=(basic_json other) noexcept ( + std::is_nothrow_move_constructible::value&& + std::is_nothrow_move_assignable::value&& + std::is_nothrow_move_constructible::value&& + std::is_nothrow_move_assignable::value + ) + { + // check that passed value is valid + other.assert_invariant(); + + using std::swap; + swap(m_type, other.m_type); + swap(m_value, other.m_value); + + set_parents(); + assert_invariant(); + return *this; + } + + /// @brief destructor + /// @sa https://json.nlohmann.me/api/basic_json/~basic_json/ + ~basic_json() noexcept + { + assert_invariant(false); + m_value.destroy(m_type); + } + + /// @} + + public: + /////////////////////// + // object inspection // + /////////////////////// + + /// @name object inspection + /// Functions to inspect the type of a JSON value. + /// @{ + + /// @brief serialization + /// @sa https://json.nlohmann.me/api/basic_json/dump/ + string_t dump(const int indent = -1, + const char indent_char = ' ', + const bool ensure_ascii = false, + const error_handler_t error_handler = error_handler_t::strict) const + { + string_t result; + serializer s(detail::output_adapter(result), indent_char, error_handler); + + if (indent >= 0) + { + s.dump(*this, true, ensure_ascii, static_cast(indent)); + } + else + { + s.dump(*this, false, ensure_ascii, 0); + } + + return result; + } + + /// @brief return the type of the JSON value (explicit) + /// @sa https://json.nlohmann.me/api/basic_json/type/ + constexpr value_t type() const noexcept + { + return m_type; + } + + /// @brief return whether type is primitive + /// @sa https://json.nlohmann.me/api/basic_json/is_primitive/ + constexpr bool is_primitive() const noexcept + { + return is_null() || is_string() || is_boolean() || is_number() || is_binary(); + } + + /// @brief return whether type is structured + /// @sa https://json.nlohmann.me/api/basic_json/is_structured/ + constexpr bool is_structured() const noexcept + { + return is_array() || is_object(); + } + + /// @brief return whether value is null + /// @sa https://json.nlohmann.me/api/basic_json/is_null/ + constexpr bool is_null() const noexcept + { + return m_type == value_t::null; + } + + /// @brief return whether value is a boolean + /// @sa https://json.nlohmann.me/api/basic_json/is_boolean/ + constexpr bool is_boolean() const noexcept + { + return m_type == value_t::boolean; + } + + /// @brief return whether value is a number + /// @sa https://json.nlohmann.me/api/basic_json/is_number/ + constexpr bool is_number() const noexcept + { + return is_number_integer() || is_number_float(); + } + + /// @brief return whether value is an integer number + /// @sa https://json.nlohmann.me/api/basic_json/is_number_integer/ + constexpr bool is_number_integer() const noexcept + { + return m_type == value_t::number_integer || m_type == value_t::number_unsigned; + } + + /// @brief return whether value is an unsigned integer number + /// @sa https://json.nlohmann.me/api/basic_json/is_number_unsigned/ + constexpr bool is_number_unsigned() const noexcept + { + return m_type == value_t::number_unsigned; + } + + /// @brief return whether value is a floating-point number + /// @sa https://json.nlohmann.me/api/basic_json/is_number_float/ + constexpr bool is_number_float() const noexcept + { + return m_type == value_t::number_float; + } + + /// @brief return whether value is an object + /// @sa https://json.nlohmann.me/api/basic_json/is_object/ + constexpr bool is_object() const noexcept + { + return m_type == value_t::object; + } + + /// @brief return whether value is an array + /// @sa https://json.nlohmann.me/api/basic_json/is_array/ + constexpr bool is_array() const noexcept + { + return m_type == value_t::array; + } + + /// @brief return whether value is a string + /// @sa https://json.nlohmann.me/api/basic_json/is_string/ + constexpr bool is_string() const noexcept + { + return m_type == value_t::string; + } + + /// @brief return whether value is a binary array + /// @sa https://json.nlohmann.me/api/basic_json/is_binary/ + constexpr bool is_binary() const noexcept + { + return m_type == value_t::binary; + } + + /// @brief return whether value is discarded + /// @sa https://json.nlohmann.me/api/basic_json/is_discarded/ + constexpr bool is_discarded() const noexcept + { + return m_type == value_t::discarded; + } + + /// @brief return the type of the JSON value (implicit) + /// @sa https://json.nlohmann.me/api/basic_json/operator_value_t/ + constexpr operator value_t() const noexcept + { + return m_type; + } + + /// @} + + private: + ////////////////// + // value access // + ////////////////// + + /// get a boolean (explicit) + boolean_t get_impl(boolean_t* /*unused*/) const + { + if (JSON_HEDLEY_LIKELY(is_boolean())) + { + return m_value.boolean; + } + + JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(type_name()), *this)); + } + + /// get a pointer to the value (object) + object_t* get_impl_ptr(object_t* /*unused*/) noexcept + { + return is_object() ? m_value.object : nullptr; + } + + /// get a pointer to the value (object) + constexpr const object_t* get_impl_ptr(const object_t* /*unused*/) const noexcept + { + return is_object() ? m_value.object : nullptr; + } + + /// get a pointer to the value (array) + array_t* get_impl_ptr(array_t* /*unused*/) noexcept + { + return is_array() ? m_value.array : nullptr; + } + + /// get a pointer to the value (array) + constexpr const array_t* get_impl_ptr(const array_t* /*unused*/) const noexcept + { + return is_array() ? m_value.array : nullptr; + } + + /// get a pointer to the value (string) + string_t* get_impl_ptr(string_t* /*unused*/) noexcept + { + return is_string() ? m_value.string : nullptr; + } + + /// get a pointer to the value (string) + constexpr const string_t* get_impl_ptr(const string_t* /*unused*/) const noexcept + { + return is_string() ? m_value.string : nullptr; + } + + /// get a pointer to the value (boolean) + boolean_t* get_impl_ptr(boolean_t* /*unused*/) noexcept + { + return is_boolean() ? &m_value.boolean : nullptr; + } + + /// get a pointer to the value (boolean) + constexpr const boolean_t* get_impl_ptr(const boolean_t* /*unused*/) const noexcept + { + return is_boolean() ? &m_value.boolean : nullptr; + } + + /// get a pointer to the value (integer number) + number_integer_t* get_impl_ptr(number_integer_t* /*unused*/) noexcept + { + return is_number_integer() ? &m_value.number_integer : nullptr; + } + + /// get a pointer to the value (integer number) + constexpr const number_integer_t* get_impl_ptr(const number_integer_t* /*unused*/) const noexcept + { + return is_number_integer() ? &m_value.number_integer : nullptr; + } + + /// get a pointer to the value (unsigned number) + number_unsigned_t* get_impl_ptr(number_unsigned_t* /*unused*/) noexcept + { + return is_number_unsigned() ? &m_value.number_unsigned : nullptr; + } + + /// get a pointer to the value (unsigned number) + constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t* /*unused*/) const noexcept + { + return is_number_unsigned() ? &m_value.number_unsigned : nullptr; + } + + /// get a pointer to the value (floating-point number) + number_float_t* get_impl_ptr(number_float_t* /*unused*/) noexcept + { + return is_number_float() ? &m_value.number_float : nullptr; + } + + /// get a pointer to the value (floating-point number) + constexpr const number_float_t* get_impl_ptr(const number_float_t* /*unused*/) const noexcept + { + return is_number_float() ? &m_value.number_float : nullptr; + } + + /// get a pointer to the value (binary) + binary_t* get_impl_ptr(binary_t* /*unused*/) noexcept + { + return is_binary() ? m_value.binary : nullptr; + } + + /// get a pointer to the value (binary) + constexpr const binary_t* get_impl_ptr(const binary_t* /*unused*/) const noexcept + { + return is_binary() ? m_value.binary : nullptr; + } + + /*! + @brief helper function to implement get_ref() + + This function helps to implement get_ref() without code duplication for + const and non-const overloads + + @tparam ThisType will be deduced as `basic_json` or `const basic_json` + + @throw type_error.303 if ReferenceType does not match underlying value + type of the current JSON + */ + template + static ReferenceType get_ref_impl(ThisType& obj) + { + // delegate the call to get_ptr<>() + auto* ptr = obj.template get_ptr::type>(); + + if (JSON_HEDLEY_LIKELY(ptr != nullptr)) + { + return *ptr; + } + + JSON_THROW(type_error::create(303, "incompatible ReferenceType for get_ref, actual type is " + std::string(obj.type_name()), obj)); + } + + public: + /// @name value access + /// Direct access to the stored value of a JSON value. + /// @{ + + /// @brief get a pointer value (implicit) + /// @sa https://json.nlohmann.me/api/basic_json/get_ptr/ + template::value, int>::type = 0> + auto get_ptr() noexcept -> decltype(std::declval().get_impl_ptr(std::declval())) + { + // delegate the call to get_impl_ptr<>() + return get_impl_ptr(static_cast(nullptr)); + } + + /// @brief get a pointer value (implicit) + /// @sa https://json.nlohmann.me/api/basic_json/get_ptr/ + template < typename PointerType, typename std::enable_if < + std::is_pointer::value&& + std::is_const::type>::value, int >::type = 0 > + constexpr auto get_ptr() const noexcept -> decltype(std::declval().get_impl_ptr(std::declval())) + { + // delegate the call to get_impl_ptr<>() const + return get_impl_ptr(static_cast(nullptr)); + } + + private: + /*! + @brief get a value (explicit) + + Explicit type conversion between the JSON value and a compatible value + which is [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible) + and [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible). + The value is converted by calling the @ref json_serializer + `from_json()` method. + + The function is equivalent to executing + @code {.cpp} + ValueType ret; + JSONSerializer::from_json(*this, ret); + return ret; + @endcode + + This overloads is chosen if: + - @a ValueType is not @ref basic_json, + - @ref json_serializer has a `from_json()` method of the form + `void from_json(const basic_json&, ValueType&)`, and + - @ref json_serializer does not have a `from_json()` method of + the form `ValueType from_json(const basic_json&)` + + @tparam ValueType the returned value type + + @return copy of the JSON value, converted to @a ValueType + + @throw what @ref json_serializer `from_json()` method throws + + @liveexample{The example below shows several conversions from JSON values + to other types. There a few things to note: (1) Floating-point numbers can + be converted to integers\, (2) A JSON array can be converted to a standard + `std::vector`\, (3) A JSON object can be converted to C++ + associative containers such as `std::unordered_map`.,get__ValueType_const} + + @since version 2.1.0 + */ + template < typename ValueType, + detail::enable_if_t < + detail::is_default_constructible::value&& + detail::has_from_json::value, + int > = 0 > + ValueType get_impl(detail::priority_tag<0> /*unused*/) const noexcept(noexcept( + JSONSerializer::from_json(std::declval(), std::declval()))) + { + auto ret = ValueType(); + JSONSerializer::from_json(*this, ret); + return ret; + } + + /*! + @brief get a value (explicit); special case + + Explicit type conversion between the JSON value and a compatible value + which is **not** [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible) + and **not** [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible). + The value is converted by calling the @ref json_serializer + `from_json()` method. + + The function is equivalent to executing + @code {.cpp} + return JSONSerializer::from_json(*this); + @endcode + + This overloads is chosen if: + - @a ValueType is not @ref basic_json and + - @ref json_serializer has a `from_json()` method of the form + `ValueType from_json(const basic_json&)` + + @note If @ref json_serializer has both overloads of + `from_json()`, this one is chosen. + + @tparam ValueType the returned value type + + @return copy of the JSON value, converted to @a ValueType + + @throw what @ref json_serializer `from_json()` method throws + + @since version 2.1.0 + */ + template < typename ValueType, + detail::enable_if_t < + detail::has_non_default_from_json::value, + int > = 0 > + ValueType get_impl(detail::priority_tag<1> /*unused*/) const noexcept(noexcept( + JSONSerializer::from_json(std::declval()))) + { + return JSONSerializer::from_json(*this); + } + + /*! + @brief get special-case overload + + This overloads converts the current @ref basic_json in a different + @ref basic_json type + + @tparam BasicJsonType == @ref basic_json + + @return a copy of *this, converted into @a BasicJsonType + + @complexity Depending on the implementation of the called `from_json()` + method. + + @since version 3.2.0 + */ + template < typename BasicJsonType, + detail::enable_if_t < + detail::is_basic_json::value, + int > = 0 > + BasicJsonType get_impl(detail::priority_tag<2> /*unused*/) const + { + return *this; + } + + /*! + @brief get special-case overload + + This overloads avoids a lot of template boilerplate, it can be seen as the + identity method + + @tparam BasicJsonType == @ref basic_json + + @return a copy of *this + + @complexity Constant. + + @since version 2.1.0 + */ + template::value, + int> = 0> + basic_json get_impl(detail::priority_tag<3> /*unused*/) const + { + return *this; + } + + /*! + @brief get a pointer value (explicit) + @copydoc get() + */ + template::value, + int> = 0> + constexpr auto get_impl(detail::priority_tag<4> /*unused*/) const noexcept + -> decltype(std::declval().template get_ptr()) + { + // delegate the call to get_ptr + return get_ptr(); + } + + public: + /*! + @brief get a (pointer) value (explicit) + + Performs explicit type conversion between the JSON value and a compatible value if required. + + - If the requested type is a pointer to the internally stored JSON value that pointer is returned. + No copies are made. + + - If the requested type is the current @ref basic_json, or a different @ref basic_json convertible + from the current @ref basic_json. + + - Otherwise the value is converted by calling the @ref json_serializer `from_json()` + method. + + @tparam ValueTypeCV the provided value type + @tparam ValueType the returned value type + + @return copy of the JSON value, converted to @tparam ValueType if necessary + + @throw what @ref json_serializer `from_json()` method throws if conversion is required + + @since version 2.1.0 + */ + template < typename ValueTypeCV, typename ValueType = detail::uncvref_t> +#if defined(JSON_HAS_CPP_14) + constexpr +#endif + auto get() const noexcept( + noexcept(std::declval().template get_impl(detail::priority_tag<4> {}))) + -> decltype(std::declval().template get_impl(detail::priority_tag<4> {})) + { + // we cannot static_assert on ValueTypeCV being non-const, because + // there is support for get(), which is why we + // still need the uncvref + static_assert(!std::is_reference::value, + "get() cannot be used with reference types, you might want to use get_ref()"); + return get_impl(detail::priority_tag<4> {}); + } + + /*! + @brief get a pointer value (explicit) + + Explicit pointer access to the internally stored JSON value. No copies are + made. + + @warning The pointer becomes invalid if the underlying JSON object + changes. + + @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref + object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, + @ref number_unsigned_t, or @ref number_float_t. + + @return pointer to the internally stored JSON value if the requested + pointer type @a PointerType fits to the JSON value; `nullptr` otherwise + + @complexity Constant. + + @liveexample{The example below shows how pointers to internal values of a + JSON value can be requested. Note that no type conversions are made and a + `nullptr` is returned if the value and the requested pointer type does not + match.,get__PointerType} + + @sa see @ref get_ptr() for explicit pointer-member access + + @since version 1.0.0 + */ + template::value, int>::type = 0> + auto get() noexcept -> decltype(std::declval().template get_ptr()) + { + // delegate the call to get_ptr + return get_ptr(); + } + + /// @brief get a value (explicit) + /// @sa https://json.nlohmann.me/api/basic_json/get_to/ + template < typename ValueType, + detail::enable_if_t < + !detail::is_basic_json::value&& + detail::has_from_json::value, + int > = 0 > + ValueType & get_to(ValueType& v) const noexcept(noexcept( + JSONSerializer::from_json(std::declval(), v))) + { + JSONSerializer::from_json(*this, v); + return v; + } + + // specialization to allow calling get_to with a basic_json value + // see https://github.com/nlohmann/json/issues/2175 + template::value, + int> = 0> + ValueType & get_to(ValueType& v) const + { + v = *this; + return v; + } + + template < + typename T, std::size_t N, + typename Array = T (&)[N], // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + detail::enable_if_t < + detail::has_from_json::value, int > = 0 > + Array get_to(T (&v)[N]) const // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + noexcept(noexcept(JSONSerializer::from_json( + std::declval(), v))) + { + JSONSerializer::from_json(*this, v); + return v; + } + + /// @brief get a reference value (implicit) + /// @sa https://json.nlohmann.me/api/basic_json/get_ref/ + template::value, int>::type = 0> + ReferenceType get_ref() + { + // delegate call to get_ref_impl + return get_ref_impl(*this); + } + + /// @brief get a reference value (implicit) + /// @sa https://json.nlohmann.me/api/basic_json/get_ref/ + template < typename ReferenceType, typename std::enable_if < + std::is_reference::value&& + std::is_const::type>::value, int >::type = 0 > + ReferenceType get_ref() const + { + // delegate call to get_ref_impl + return get_ref_impl(*this); + } + + /*! + @brief get a value (implicit) + + Implicit type conversion between the JSON value and a compatible value. + The call is realized by calling @ref get() const. + + @tparam ValueType non-pointer type compatible to the JSON value, for + instance `int` for JSON integer numbers, `bool` for JSON booleans, or + `std::vector` types for JSON arrays. The character type of @ref string_t + as well as an initializer list of this type is excluded to avoid + ambiguities as these types implicitly convert to `std::string`. + + @return copy of the JSON value, converted to type @a ValueType + + @throw type_error.302 in case passed type @a ValueType is incompatible + to the JSON value type (e.g., the JSON value is of type boolean, but a + string is requested); see example below + + @complexity Linear in the size of the JSON value. + + @liveexample{The example below shows several conversions from JSON values + to other types. There a few things to note: (1) Floating-point numbers can + be converted to integers\, (2) A JSON array can be converted to a standard + `std::vector`\, (3) A JSON object can be converted to C++ + associative containers such as `std::unordered_map`.,operator__ValueType} + + @since version 1.0.0 + */ + template < typename ValueType, typename std::enable_if < + detail::conjunction < + detail::negation>, + detail::negation>>, + detail::negation>, + detail::negation>, + detail::negation>>, + +#if defined(JSON_HAS_CPP_17) && (defined(__GNUC__) || (defined(_MSC_VER) && _MSC_VER >= 1910 && _MSC_VER <= 1914)) + detail::negation>, +#endif + detail::is_detected_lazy + >::value, int >::type = 0 > + JSON_EXPLICIT operator ValueType() const + { + // delegate the call to get<>() const + return get(); + } + + /// @brief get a binary value + /// @sa https://json.nlohmann.me/api/basic_json/get_binary/ + binary_t& get_binary() + { + if (!is_binary()) + { + JSON_THROW(type_error::create(302, "type must be binary, but is " + std::string(type_name()), *this)); + } + + return *get_ptr(); + } + + /// @brief get a binary value + /// @sa https://json.nlohmann.me/api/basic_json/get_binary/ + const binary_t& get_binary() const + { + if (!is_binary()) + { + JSON_THROW(type_error::create(302, "type must be binary, but is " + std::string(type_name()), *this)); + } + + return *get_ptr(); + } + + /// @} + + + //////////////////// + // element access // + //////////////////// + + /// @name element access + /// Access to the JSON value. + /// @{ + + /// @brief access specified array element with bounds checking + /// @sa https://json.nlohmann.me/api/basic_json/at/ + reference at(size_type idx) + { + // at only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + JSON_TRY + { + return set_parent(m_value.array->at(idx)); + } + JSON_CATCH (std::out_of_range&) + { + // create better exception explanation + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range", *this)); + } + } + else + { + JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name()), *this)); + } + } + + /// @brief access specified array element with bounds checking + /// @sa https://json.nlohmann.me/api/basic_json/at/ + const_reference at(size_type idx) const + { + // at only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + JSON_TRY + { + return m_value.array->at(idx); + } + JSON_CATCH (std::out_of_range&) + { + // create better exception explanation + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range", *this)); + } + } + else + { + JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name()), *this)); + } + } + + /// @brief access specified object element with bounds checking + /// @sa https://json.nlohmann.me/api/basic_json/at/ + reference at(const typename object_t::key_type& key) + { + // at only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + JSON_TRY + { + return set_parent(m_value.object->at(key)); + } + JSON_CATCH (std::out_of_range&) + { + // create better exception explanation + JSON_THROW(out_of_range::create(403, "key '" + key + "' not found", *this)); + } + } + else + { + JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name()), *this)); + } + } + + /// @brief access specified object element with bounds checking + /// @sa https://json.nlohmann.me/api/basic_json/at/ + const_reference at(const typename object_t::key_type& key) const + { + // at only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + JSON_TRY + { + return m_value.object->at(key); + } + JSON_CATCH (std::out_of_range&) + { + // create better exception explanation + JSON_THROW(out_of_range::create(403, "key '" + key + "' not found", *this)); + } + } + else + { + JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name()), *this)); + } + } + + /// @brief access specified array element + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + reference operator[](size_type idx) + { + // implicitly convert null value to an empty array + if (is_null()) + { + m_type = value_t::array; + m_value.array = create(); + assert_invariant(); + } + + // operator[] only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + // fill up array with null values if given idx is outside range + if (idx >= m_value.array->size()) + { +#if JSON_DIAGNOSTICS + // remember array size & capacity before resizing + const auto old_size = m_value.array->size(); + const auto old_capacity = m_value.array->capacity(); +#endif + m_value.array->resize(idx + 1); + +#if JSON_DIAGNOSTICS + if (JSON_HEDLEY_UNLIKELY(m_value.array->capacity() != old_capacity)) + { + // capacity has changed: update all parents + set_parents(); + } + else + { + // set parent for values added above + set_parents(begin() + static_cast(old_size), static_cast(idx + 1 - old_size)); + } +#endif + assert_invariant(); + } + + return m_value.array->operator[](idx); + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a numeric argument with " + std::string(type_name()), *this)); + } + + /// @brief access specified array element + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + const_reference operator[](size_type idx) const + { + // const operator[] only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + return m_value.array->operator[](idx); + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a numeric argument with " + std::string(type_name()), *this)); + } + + /// @brief access specified object element + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + reference operator[](const typename object_t::key_type& key) + { + // implicitly convert null value to an empty object + if (is_null()) + { + m_type = value_t::object; + m_value.object = create(); + assert_invariant(); + } + + // operator[] only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + return set_parent(m_value.object->operator[](key)); + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()), *this)); + } + + /// @brief access specified object element + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + const_reference operator[](const typename object_t::key_type& key) const + { + // const operator[] only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + JSON_ASSERT(m_value.object->find(key) != m_value.object->end()); + return m_value.object->find(key)->second; + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()), *this)); + } + + /// @brief access specified object element + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + template + JSON_HEDLEY_NON_NULL(2) + reference operator[](T* key) + { + // implicitly convert null to object + if (is_null()) + { + m_type = value_t::object; + m_value = value_t::object; + assert_invariant(); + } + + // at only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + return set_parent(m_value.object->operator[](key)); + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()), *this)); + } + + /// @brief access specified object element + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + template + JSON_HEDLEY_NON_NULL(2) + const_reference operator[](T* key) const + { + // at only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + JSON_ASSERT(m_value.object->find(key) != m_value.object->end()); + return m_value.object->find(key)->second; + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()), *this)); + } + + /// @brief access specified object element with default value + /// @sa https://json.nlohmann.me/api/basic_json/value/ + /// using std::is_convertible in a std::enable_if will fail when using explicit conversions + template < class ValueType, typename std::enable_if < + detail::is_getable::value + && !std::is_same::value, int >::type = 0 > + ValueType value(const typename object_t::key_type& key, const ValueType& default_value) const + { + // at only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + // if key is found, return value and given default value otherwise + const auto it = find(key); + if (it != end()) + { + return it->template get(); + } + + return default_value; + } + + JSON_THROW(type_error::create(306, "cannot use value() with " + std::string(type_name()), *this)); + } + + /// @brief access specified object element with default value + /// @sa https://json.nlohmann.me/api/basic_json/value/ + /// overload for a default value of type const char* + string_t value(const typename object_t::key_type& key, const char* default_value) const + { + return value(key, string_t(default_value)); + } + + /// @brief access specified object element via JSON Pointer with default value + /// @sa https://json.nlohmann.me/api/basic_json/value/ + template::value, int>::type = 0> + ValueType value(const json_pointer& ptr, const ValueType& default_value) const + { + // at only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + // if pointer resolves a value, return it or use default value + JSON_TRY + { + return ptr.get_checked(this).template get(); + } + JSON_INTERNAL_CATCH (out_of_range&) + { + return default_value; + } + } + + JSON_THROW(type_error::create(306, "cannot use value() with " + std::string(type_name()), *this)); + } + + /// @brief access specified object element via JSON Pointer with default value + /// @sa https://json.nlohmann.me/api/basic_json/value/ + /// overload for a default value of type const char* + JSON_HEDLEY_NON_NULL(3) + string_t value(const json_pointer& ptr, const char* default_value) const + { + return value(ptr, string_t(default_value)); + } + + /// @brief access the first element + /// @sa https://json.nlohmann.me/api/basic_json/front/ + reference front() + { + return *begin(); + } + + /// @brief access the first element + /// @sa https://json.nlohmann.me/api/basic_json/front/ + const_reference front() const + { + return *cbegin(); + } + + /// @brief access the last element + /// @sa https://json.nlohmann.me/api/basic_json/back/ + reference back() + { + auto tmp = end(); + --tmp; + return *tmp; + } + + /// @brief access the last element + /// @sa https://json.nlohmann.me/api/basic_json/back/ + const_reference back() const + { + auto tmp = cend(); + --tmp; + return *tmp; + } + + /// @brief remove element given an iterator + /// @sa https://json.nlohmann.me/api/basic_json/erase/ + template < class IteratorType, typename std::enable_if < + std::is_same::value || + std::is_same::value, int >::type + = 0 > + IteratorType erase(IteratorType pos) + { + // make sure iterator fits the current value + if (JSON_HEDLEY_UNLIKELY(this != pos.m_object)) + { + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", *this)); + } + + IteratorType result = end(); + + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + case value_t::binary: + { + if (JSON_HEDLEY_UNLIKELY(!pos.m_it.primitive_iterator.is_begin())) + { + JSON_THROW(invalid_iterator::create(205, "iterator out of range", *this)); + } + + if (is_string()) + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, m_value.string); + std::allocator_traits::deallocate(alloc, m_value.string, 1); + m_value.string = nullptr; + } + else if (is_binary()) + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, m_value.binary); + std::allocator_traits::deallocate(alloc, m_value.binary, 1); + m_value.binary = nullptr; + } + + m_type = value_t::null; + assert_invariant(); + break; + } + + case value_t::object: + { + result.m_it.object_iterator = m_value.object->erase(pos.m_it.object_iterator); + break; + } + + case value_t::array: + { + result.m_it.array_iterator = m_value.array->erase(pos.m_it.array_iterator); + break; + } + + case value_t::null: + case value_t::discarded: + default: + JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name()), *this)); + } + + return result; + } + + /// @brief remove elements given an iterator range + /// @sa https://json.nlohmann.me/api/basic_json/erase/ + template < class IteratorType, typename std::enable_if < + std::is_same::value || + std::is_same::value, int >::type + = 0 > + IteratorType erase(IteratorType first, IteratorType last) + { + // make sure iterator fits the current value + if (JSON_HEDLEY_UNLIKELY(this != first.m_object || this != last.m_object)) + { + JSON_THROW(invalid_iterator::create(203, "iterators do not fit current value", *this)); + } + + IteratorType result = end(); + + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + case value_t::binary: + { + if (JSON_HEDLEY_LIKELY(!first.m_it.primitive_iterator.is_begin() + || !last.m_it.primitive_iterator.is_end())) + { + JSON_THROW(invalid_iterator::create(204, "iterators out of range", *this)); + } + + if (is_string()) + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, m_value.string); + std::allocator_traits::deallocate(alloc, m_value.string, 1); + m_value.string = nullptr; + } + else if (is_binary()) + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, m_value.binary); + std::allocator_traits::deallocate(alloc, m_value.binary, 1); + m_value.binary = nullptr; + } + + m_type = value_t::null; + assert_invariant(); + break; + } + + case value_t::object: + { + result.m_it.object_iterator = m_value.object->erase(first.m_it.object_iterator, + last.m_it.object_iterator); + break; + } + + case value_t::array: + { + result.m_it.array_iterator = m_value.array->erase(first.m_it.array_iterator, + last.m_it.array_iterator); + break; + } + + case value_t::null: + case value_t::discarded: + default: + JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name()), *this)); + } + + return result; + } + + /// @brief remove element from a JSON object given a key + /// @sa https://json.nlohmann.me/api/basic_json/erase/ + size_type erase(const typename object_t::key_type& key) + { + // this erase only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + return m_value.object->erase(key); + } + + JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name()), *this)); + } + + /// @brief remove element from a JSON array given an index + /// @sa https://json.nlohmann.me/api/basic_json/erase/ + void erase(const size_type idx) + { + // this erase only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + if (JSON_HEDLEY_UNLIKELY(idx >= size())) + { + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range", *this)); + } + + m_value.array->erase(m_value.array->begin() + static_cast(idx)); + } + else + { + JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name()), *this)); + } + } + + /// @} + + + //////////// + // lookup // + //////////// + + /// @name lookup + /// @{ + + /// @brief find an element in a JSON object + /// @sa https://json.nlohmann.me/api/basic_json/find/ + template + iterator find(KeyT&& key) + { + auto result = end(); + + if (is_object()) + { + result.m_it.object_iterator = m_value.object->find(std::forward(key)); + } + + return result; + } + + /// @brief find an element in a JSON object + /// @sa https://json.nlohmann.me/api/basic_json/find/ + template + const_iterator find(KeyT&& key) const + { + auto result = cend(); + + if (is_object()) + { + result.m_it.object_iterator = m_value.object->find(std::forward(key)); + } + + return result; + } + + /// @brief returns the number of occurrences of a key in a JSON object + /// @sa https://json.nlohmann.me/api/basic_json/count/ + template + size_type count(KeyT&& key) const + { + // return 0 for all nonobject types + return is_object() ? m_value.object->count(std::forward(key)) : 0; + } + + /// @brief check the existence of an element in a JSON object + /// @sa https://json.nlohmann.me/api/basic_json/contains/ + template < typename KeyT, typename std::enable_if < + !std::is_same::type, json_pointer>::value, int >::type = 0 > + bool contains(KeyT && key) const + { + return is_object() && m_value.object->find(std::forward(key)) != m_value.object->end(); + } + + /// @brief check the existence of an element in a JSON object given a JSON pointer + /// @sa https://json.nlohmann.me/api/basic_json/contains/ + bool contains(const json_pointer& ptr) const + { + return ptr.contains(this); + } + + /// @} + + + /////////////// + // iterators // + /////////////// + + /// @name iterators + /// @{ + + /// @brief returns an iterator to the first element + /// @sa https://json.nlohmann.me/api/basic_json/begin/ + iterator begin() noexcept + { + iterator result(this); + result.set_begin(); + return result; + } + + /// @brief returns an iterator to the first element + /// @sa https://json.nlohmann.me/api/basic_json/begin/ + const_iterator begin() const noexcept + { + return cbegin(); + } + + /// @brief returns a const iterator to the first element + /// @sa https://json.nlohmann.me/api/basic_json/cbegin/ + const_iterator cbegin() const noexcept + { + const_iterator result(this); + result.set_begin(); + return result; + } + + /// @brief returns an iterator to one past the last element + /// @sa https://json.nlohmann.me/api/basic_json/end/ + iterator end() noexcept + { + iterator result(this); + result.set_end(); + return result; + } + + /// @brief returns an iterator to one past the last element + /// @sa https://json.nlohmann.me/api/basic_json/end/ + const_iterator end() const noexcept + { + return cend(); + } + + /// @brief returns an iterator to one past the last element + /// @sa https://json.nlohmann.me/api/basic_json/cend/ + const_iterator cend() const noexcept + { + const_iterator result(this); + result.set_end(); + return result; + } + + /// @brief returns an iterator to the reverse-beginning + /// @sa https://json.nlohmann.me/api/basic_json/rbegin/ + reverse_iterator rbegin() noexcept + { + return reverse_iterator(end()); + } + + /// @brief returns an iterator to the reverse-beginning + /// @sa https://json.nlohmann.me/api/basic_json/rbegin/ + const_reverse_iterator rbegin() const noexcept + { + return crbegin(); + } + + /// @brief returns an iterator to the reverse-end + /// @sa https://json.nlohmann.me/api/basic_json/rend/ + reverse_iterator rend() noexcept + { + return reverse_iterator(begin()); + } + + /// @brief returns an iterator to the reverse-end + /// @sa https://json.nlohmann.me/api/basic_json/rend/ + const_reverse_iterator rend() const noexcept + { + return crend(); + } + + /// @brief returns a const reverse iterator to the last element + /// @sa https://json.nlohmann.me/api/basic_json/crbegin/ + const_reverse_iterator crbegin() const noexcept + { + return const_reverse_iterator(cend()); + } + + /// @brief returns a const reverse iterator to one before the first + /// @sa https://json.nlohmann.me/api/basic_json/crend/ + const_reverse_iterator crend() const noexcept + { + return const_reverse_iterator(cbegin()); + } + + public: + /// @brief wrapper to access iterator member functions in range-based for + /// @sa https://json.nlohmann.me/api/basic_json/items/ + /// @deprecated This function is deprecated since 3.1.0 and will be removed in + /// version 4.0.0 of the library. Please use @ref items() instead; + /// that is, replace `json::iterator_wrapper(j)` with `j.items()`. + JSON_HEDLEY_DEPRECATED_FOR(3.1.0, items()) + static iteration_proxy iterator_wrapper(reference ref) noexcept + { + return ref.items(); + } + + /// @brief wrapper to access iterator member functions in range-based for + /// @sa https://json.nlohmann.me/api/basic_json/items/ + /// @deprecated This function is deprecated since 3.1.0 and will be removed in + /// version 4.0.0 of the library. Please use @ref items() instead; + /// that is, replace `json::iterator_wrapper(j)` with `j.items()`. + JSON_HEDLEY_DEPRECATED_FOR(3.1.0, items()) + static iteration_proxy iterator_wrapper(const_reference ref) noexcept + { + return ref.items(); + } + + /// @brief helper to access iterator member functions in range-based for + /// @sa https://json.nlohmann.me/api/basic_json/items/ + iteration_proxy items() noexcept + { + return iteration_proxy(*this); + } + + /// @brief helper to access iterator member functions in range-based for + /// @sa https://json.nlohmann.me/api/basic_json/items/ + iteration_proxy items() const noexcept + { + return iteration_proxy(*this); + } + + /// @} + + + ////////////// + // capacity // + ////////////// + + /// @name capacity + /// @{ + + /// @brief checks whether the container is empty. + /// @sa https://json.nlohmann.me/api/basic_json/empty/ + bool empty() const noexcept + { + switch (m_type) + { + case value_t::null: + { + // null values are empty + return true; + } + + case value_t::array: + { + // delegate call to array_t::empty() + return m_value.array->empty(); + } + + case value_t::object: + { + // delegate call to object_t::empty() + return m_value.object->empty(); + } + + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + // all other types are nonempty + return false; + } + } + } + + /// @brief returns the number of elements + /// @sa https://json.nlohmann.me/api/basic_json/size/ + size_type size() const noexcept + { + switch (m_type) + { + case value_t::null: + { + // null values are empty + return 0; + } + + case value_t::array: + { + // delegate call to array_t::size() + return m_value.array->size(); + } + + case value_t::object: + { + // delegate call to object_t::size() + return m_value.object->size(); + } + + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + // all other types have size 1 + return 1; + } + } + } + + /// @brief returns the maximum possible number of elements + /// @sa https://json.nlohmann.me/api/basic_json/max_size/ + size_type max_size() const noexcept + { + switch (m_type) + { + case value_t::array: + { + // delegate call to array_t::max_size() + return m_value.array->max_size(); + } + + case value_t::object: + { + // delegate call to object_t::max_size() + return m_value.object->max_size(); + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + // all other types have max_size() == size() + return size(); + } + } + } + + /// @} + + + /////////////// + // modifiers // + /////////////// + + /// @name modifiers + /// @{ + + /// @brief clears the contents + /// @sa https://json.nlohmann.me/api/basic_json/clear/ + void clear() noexcept + { + switch (m_type) + { + case value_t::number_integer: + { + m_value.number_integer = 0; + break; + } + + case value_t::number_unsigned: + { + m_value.number_unsigned = 0; + break; + } + + case value_t::number_float: + { + m_value.number_float = 0.0; + break; + } + + case value_t::boolean: + { + m_value.boolean = false; + break; + } + + case value_t::string: + { + m_value.string->clear(); + break; + } + + case value_t::binary: + { + m_value.binary->clear(); + break; + } + + case value_t::array: + { + m_value.array->clear(); + break; + } + + case value_t::object: + { + m_value.object->clear(); + break; + } + + case value_t::null: + case value_t::discarded: + default: + break; + } + } + + /// @brief add an object to an array + /// @sa https://json.nlohmann.me/api/basic_json/push_back/ + void push_back(basic_json&& val) + { + // push_back only works for null objects or arrays + if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_array()))) + { + JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name()), *this)); + } + + // transform null object into an array + if (is_null()) + { + m_type = value_t::array; + m_value = value_t::array; + assert_invariant(); + } + + // add element to array (move semantics) + const auto old_capacity = m_value.array->capacity(); + m_value.array->push_back(std::move(val)); + set_parent(m_value.array->back(), old_capacity); + // if val is moved from, basic_json move constructor marks it null, so we do not call the destructor + } + + /// @brief add an object to an array + /// @sa https://json.nlohmann.me/api/basic_json/operator+=/ + reference operator+=(basic_json&& val) + { + push_back(std::move(val)); + return *this; + } + + /// @brief add an object to an array + /// @sa https://json.nlohmann.me/api/basic_json/push_back/ + void push_back(const basic_json& val) + { + // push_back only works for null objects or arrays + if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_array()))) + { + JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name()), *this)); + } + + // transform null object into an array + if (is_null()) + { + m_type = value_t::array; + m_value = value_t::array; + assert_invariant(); + } + + // add element to array + const auto old_capacity = m_value.array->capacity(); + m_value.array->push_back(val); + set_parent(m_value.array->back(), old_capacity); + } + + /// @brief add an object to an array + /// @sa https://json.nlohmann.me/api/basic_json/operator+=/ + reference operator+=(const basic_json& val) + { + push_back(val); + return *this; + } + + /// @brief add an object to an object + /// @sa https://json.nlohmann.me/api/basic_json/push_back/ + void push_back(const typename object_t::value_type& val) + { + // push_back only works for null objects or objects + if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_object()))) + { + JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name()), *this)); + } + + // transform null object into an object + if (is_null()) + { + m_type = value_t::object; + m_value = value_t::object; + assert_invariant(); + } + + // add element to object + auto res = m_value.object->insert(val); + set_parent(res.first->second); + } + + /// @brief add an object to an object + /// @sa https://json.nlohmann.me/api/basic_json/operator+=/ + reference operator+=(const typename object_t::value_type& val) + { + push_back(val); + return *this; + } + + /// @brief add an object to an object + /// @sa https://json.nlohmann.me/api/basic_json/push_back/ + void push_back(initializer_list_t init) + { + if (is_object() && init.size() == 2 && (*init.begin())->is_string()) + { + basic_json&& key = init.begin()->moved_or_copied(); + push_back(typename object_t::value_type( + std::move(key.get_ref()), (init.begin() + 1)->moved_or_copied())); + } + else + { + push_back(basic_json(init)); + } + } + + /// @brief add an object to an object + /// @sa https://json.nlohmann.me/api/basic_json/operator+=/ + reference operator+=(initializer_list_t init) + { + push_back(init); + return *this; + } + + /// @brief add an object to an array + /// @sa https://json.nlohmann.me/api/basic_json/emplace_back/ + template + reference emplace_back(Args&& ... args) + { + // emplace_back only works for null objects or arrays + if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_array()))) + { + JSON_THROW(type_error::create(311, "cannot use emplace_back() with " + std::string(type_name()), *this)); + } + + // transform null object into an array + if (is_null()) + { + m_type = value_t::array; + m_value = value_t::array; + assert_invariant(); + } + + // add element to array (perfect forwarding) + const auto old_capacity = m_value.array->capacity(); + m_value.array->emplace_back(std::forward(args)...); + return set_parent(m_value.array->back(), old_capacity); + } + + /// @brief add an object to an object if key does not exist + /// @sa https://json.nlohmann.me/api/basic_json/emplace/ + template + std::pair emplace(Args&& ... args) + { + // emplace only works for null objects or arrays + if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_object()))) + { + JSON_THROW(type_error::create(311, "cannot use emplace() with " + std::string(type_name()), *this)); + } + + // transform null object into an object + if (is_null()) + { + m_type = value_t::object; + m_value = value_t::object; + assert_invariant(); + } + + // add element to array (perfect forwarding) + auto res = m_value.object->emplace(std::forward(args)...); + set_parent(res.first->second); + + // create result iterator and set iterator to the result of emplace + auto it = begin(); + it.m_it.object_iterator = res.first; + + // return pair of iterator and boolean + return {it, res.second}; + } + + /// Helper for insertion of an iterator + /// @note: This uses std::distance to support GCC 4.8, + /// see https://github.com/nlohmann/json/pull/1257 + template + iterator insert_iterator(const_iterator pos, Args&& ... args) + { + iterator result(this); + JSON_ASSERT(m_value.array != nullptr); + + auto insert_pos = std::distance(m_value.array->begin(), pos.m_it.array_iterator); + m_value.array->insert(pos.m_it.array_iterator, std::forward(args)...); + result.m_it.array_iterator = m_value.array->begin() + insert_pos; + + // This could have been written as: + // result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val); + // but the return value of insert is missing in GCC 4.8, so it is written this way instead. + + set_parents(); + return result; + } + + /// @brief inserts element into array + /// @sa https://json.nlohmann.me/api/basic_json/insert/ + iterator insert(const_iterator pos, const basic_json& val) + { + // insert only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + // check if iterator pos fits to this JSON value + if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) + { + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", *this)); + } + + // insert to array and return iterator + return insert_iterator(pos, val); + } + + JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()), *this)); + } + + /// @brief inserts element into array + /// @sa https://json.nlohmann.me/api/basic_json/insert/ + iterator insert(const_iterator pos, basic_json&& val) + { + return insert(pos, val); + } + + /// @brief inserts copies of element into array + /// @sa https://json.nlohmann.me/api/basic_json/insert/ + iterator insert(const_iterator pos, size_type cnt, const basic_json& val) + { + // insert only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + // check if iterator pos fits to this JSON value + if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) + { + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", *this)); + } + + // insert to array and return iterator + return insert_iterator(pos, cnt, val); + } + + JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()), *this)); + } + + /// @brief inserts range of elements into array + /// @sa https://json.nlohmann.me/api/basic_json/insert/ + iterator insert(const_iterator pos, const_iterator first, const_iterator last) + { + // insert only works for arrays + if (JSON_HEDLEY_UNLIKELY(!is_array())) + { + JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()), *this)); + } + + // check if iterator pos fits to this JSON value + if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) + { + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", *this)); + } + + // check if range iterators belong to the same JSON object + if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) + { + JSON_THROW(invalid_iterator::create(210, "iterators do not fit", *this)); + } + + if (JSON_HEDLEY_UNLIKELY(first.m_object == this)) + { + JSON_THROW(invalid_iterator::create(211, "passed iterators may not belong to container", *this)); + } + + // insert to array and return iterator + return insert_iterator(pos, first.m_it.array_iterator, last.m_it.array_iterator); + } + + /// @brief inserts elements from initializer list into array + /// @sa https://json.nlohmann.me/api/basic_json/insert/ + iterator insert(const_iterator pos, initializer_list_t ilist) + { + // insert only works for arrays + if (JSON_HEDLEY_UNLIKELY(!is_array())) + { + JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()), *this)); + } + + // check if iterator pos fits to this JSON value + if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) + { + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", *this)); + } + + // insert to array and return iterator + return insert_iterator(pos, ilist.begin(), ilist.end()); + } + + /// @brief inserts range of elements into object + /// @sa https://json.nlohmann.me/api/basic_json/insert/ + void insert(const_iterator first, const_iterator last) + { + // insert only works for objects + if (JSON_HEDLEY_UNLIKELY(!is_object())) + { + JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()), *this)); + } + + // check if range iterators belong to the same JSON object + if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) + { + JSON_THROW(invalid_iterator::create(210, "iterators do not fit", *this)); + } + + // passed iterators must belong to objects + if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_object())) + { + JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects", *this)); + } + + m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator); + } + + /// @brief updates a JSON object from another object, overwriting existing keys + /// @sa https://json.nlohmann.me/api/basic_json/update/ + void update(const_reference j, bool merge_objects = false) + { + update(j.begin(), j.end(), merge_objects); + } + + /// @brief updates a JSON object from another object, overwriting existing keys + /// @sa https://json.nlohmann.me/api/basic_json/update/ + void update(const_iterator first, const_iterator last, bool merge_objects = false) + { + // implicitly convert null value to an empty object + if (is_null()) + { + m_type = value_t::object; + m_value.object = create(); + assert_invariant(); + } + + if (JSON_HEDLEY_UNLIKELY(!is_object())) + { + JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(type_name()), *this)); + } + + // check if range iterators belong to the same JSON object + if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) + { + JSON_THROW(invalid_iterator::create(210, "iterators do not fit", *this)); + } + + // passed iterators must belong to objects + if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_object())) + { + JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(first.m_object->type_name()), *first.m_object)); + } + + for (auto it = first; it != last; ++it) + { + if (merge_objects && it.value().is_object()) + { + auto it2 = m_value.object->find(it.key()); + if (it2 != m_value.object->end()) + { + it2->second.update(it.value(), true); + continue; + } + } + m_value.object->operator[](it.key()) = it.value(); +#if JSON_DIAGNOSTICS + m_value.object->operator[](it.key()).m_parent = this; +#endif + } + } + + /// @brief exchanges the values + /// @sa https://json.nlohmann.me/api/basic_json/swap/ + void swap(reference other) noexcept ( + std::is_nothrow_move_constructible::value&& + std::is_nothrow_move_assignable::value&& + std::is_nothrow_move_constructible::value&& + std::is_nothrow_move_assignable::value + ) + { + std::swap(m_type, other.m_type); + std::swap(m_value, other.m_value); + + set_parents(); + other.set_parents(); + assert_invariant(); + } + + /// @brief exchanges the values + /// @sa https://json.nlohmann.me/api/basic_json/swap/ + friend void swap(reference left, reference right) noexcept ( + std::is_nothrow_move_constructible::value&& + std::is_nothrow_move_assignable::value&& + std::is_nothrow_move_constructible::value&& + std::is_nothrow_move_assignable::value + ) + { + left.swap(right); + } + + /// @brief exchanges the values + /// @sa https://json.nlohmann.me/api/basic_json/swap/ + void swap(array_t& other) // NOLINT(bugprone-exception-escape) + { + // swap only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + std::swap(*(m_value.array), other); + } + else + { + JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name()), *this)); + } + } + + /// @brief exchanges the values + /// @sa https://json.nlohmann.me/api/basic_json/swap/ + void swap(object_t& other) // NOLINT(bugprone-exception-escape) + { + // swap only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + std::swap(*(m_value.object), other); + } + else + { + JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name()), *this)); + } + } + + /// @brief exchanges the values + /// @sa https://json.nlohmann.me/api/basic_json/swap/ + void swap(string_t& other) // NOLINT(bugprone-exception-escape) + { + // swap only works for strings + if (JSON_HEDLEY_LIKELY(is_string())) + { + std::swap(*(m_value.string), other); + } + else + { + JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name()), *this)); + } + } + + /// @brief exchanges the values + /// @sa https://json.nlohmann.me/api/basic_json/swap/ + void swap(binary_t& other) // NOLINT(bugprone-exception-escape) + { + // swap only works for strings + if (JSON_HEDLEY_LIKELY(is_binary())) + { + std::swap(*(m_value.binary), other); + } + else + { + JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name()), *this)); + } + } + + /// @brief exchanges the values + /// @sa https://json.nlohmann.me/api/basic_json/swap/ + void swap(typename binary_t::container_type& other) // NOLINT(bugprone-exception-escape) + { + // swap only works for strings + if (JSON_HEDLEY_LIKELY(is_binary())) + { + std::swap(*(m_value.binary), other); + } + else + { + JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name()), *this)); + } + } + + /// @} + + public: + ////////////////////////////////////////// + // lexicographical comparison operators // + ////////////////////////////////////////// + + /// @name lexicographical comparison operators + /// @{ + + /// @brief comparison: equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/ + friend bool operator==(const_reference lhs, const_reference rhs) noexcept + { +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + const auto lhs_type = lhs.type(); + const auto rhs_type = rhs.type(); + + if (lhs_type == rhs_type) + { + switch (lhs_type) + { + case value_t::array: + return *lhs.m_value.array == *rhs.m_value.array; + + case value_t::object: + return *lhs.m_value.object == *rhs.m_value.object; + + case value_t::null: + return true; + + case value_t::string: + return *lhs.m_value.string == *rhs.m_value.string; + + case value_t::boolean: + return lhs.m_value.boolean == rhs.m_value.boolean; + + case value_t::number_integer: + return lhs.m_value.number_integer == rhs.m_value.number_integer; + + case value_t::number_unsigned: + return lhs.m_value.number_unsigned == rhs.m_value.number_unsigned; + + case value_t::number_float: + return lhs.m_value.number_float == rhs.m_value.number_float; + + case value_t::binary: + return *lhs.m_value.binary == *rhs.m_value.binary; + + case value_t::discarded: + default: + return false; + } + } + else if (lhs_type == value_t::number_integer && rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_integer) == rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float && rhs_type == value_t::number_integer) + { + return lhs.m_value.number_float == static_cast(rhs.m_value.number_integer); + } + else if (lhs_type == value_t::number_unsigned && rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float && rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_float == static_cast(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_unsigned && rhs_type == value_t::number_integer) + { + return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_integer; + } + else if (lhs_type == value_t::number_integer && rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_integer == static_cast(rhs.m_value.number_unsigned); + } + + return false; +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + } + + /// @brief comparison: equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/ + template::value, int>::type = 0> + friend bool operator==(const_reference lhs, ScalarType rhs) noexcept + { + return lhs == basic_json(rhs); + } + + /// @brief comparison: equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/ + template::value, int>::type = 0> + friend bool operator==(ScalarType lhs, const_reference rhs) noexcept + { + return basic_json(lhs) == rhs; + } + + /// @brief comparison: not equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_ne/ + friend bool operator!=(const_reference lhs, const_reference rhs) noexcept + { + return !(lhs == rhs); + } + + /// @brief comparison: not equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_ne/ + template::value, int>::type = 0> + friend bool operator!=(const_reference lhs, ScalarType rhs) noexcept + { + return lhs != basic_json(rhs); + } + + /// @brief comparison: not equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_ne/ + template::value, int>::type = 0> + friend bool operator!=(ScalarType lhs, const_reference rhs) noexcept + { + return basic_json(lhs) != rhs; + } + + /// @brief comparison: less than + /// @sa https://json.nlohmann.me/api/basic_json/operator_lt/ + friend bool operator<(const_reference lhs, const_reference rhs) noexcept + { + const auto lhs_type = lhs.type(); + const auto rhs_type = rhs.type(); + + if (lhs_type == rhs_type) + { + switch (lhs_type) + { + case value_t::array: + // note parentheses are necessary, see + // https://github.com/nlohmann/json/issues/1530 + return (*lhs.m_value.array) < (*rhs.m_value.array); + + case value_t::object: + return (*lhs.m_value.object) < (*rhs.m_value.object); + + case value_t::null: + return false; + + case value_t::string: + return (*lhs.m_value.string) < (*rhs.m_value.string); + + case value_t::boolean: + return (lhs.m_value.boolean) < (rhs.m_value.boolean); + + case value_t::number_integer: + return (lhs.m_value.number_integer) < (rhs.m_value.number_integer); + + case value_t::number_unsigned: + return (lhs.m_value.number_unsigned) < (rhs.m_value.number_unsigned); + + case value_t::number_float: + return (lhs.m_value.number_float) < (rhs.m_value.number_float); + + case value_t::binary: + return (*lhs.m_value.binary) < (*rhs.m_value.binary); + + case value_t::discarded: + default: + return false; + } + } + else if (lhs_type == value_t::number_integer && rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_integer) < rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float && rhs_type == value_t::number_integer) + { + return lhs.m_value.number_float < static_cast(rhs.m_value.number_integer); + } + else if (lhs_type == value_t::number_unsigned && rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float && rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_float < static_cast(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_integer && rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_integer < static_cast(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_unsigned && rhs_type == value_t::number_integer) + { + return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_integer; + } + + // We only reach this line if we cannot compare values. In that case, + // we compare types. Note we have to call the operator explicitly, + // because MSVC has problems otherwise. + return operator<(lhs_type, rhs_type); + } + + /// @brief comparison: less than + /// @sa https://json.nlohmann.me/api/basic_json/operator_lt/ + template::value, int>::type = 0> + friend bool operator<(const_reference lhs, ScalarType rhs) noexcept + { + return lhs < basic_json(rhs); + } + + /// @brief comparison: less than + /// @sa https://json.nlohmann.me/api/basic_json/operator_lt/ + template::value, int>::type = 0> + friend bool operator<(ScalarType lhs, const_reference rhs) noexcept + { + return basic_json(lhs) < rhs; + } + + /// @brief comparison: less than or equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_le/ + friend bool operator<=(const_reference lhs, const_reference rhs) noexcept + { + return !(rhs < lhs); + } + + /// @brief comparison: less than or equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_le/ + template::value, int>::type = 0> + friend bool operator<=(const_reference lhs, ScalarType rhs) noexcept + { + return lhs <= basic_json(rhs); + } + + /// @brief comparison: less than or equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_le/ + template::value, int>::type = 0> + friend bool operator<=(ScalarType lhs, const_reference rhs) noexcept + { + return basic_json(lhs) <= rhs; + } + + /// @brief comparison: greater than + /// @sa https://json.nlohmann.me/api/basic_json/operator_gt/ + friend bool operator>(const_reference lhs, const_reference rhs) noexcept + { + return !(lhs <= rhs); + } + + /// @brief comparison: greater than + /// @sa https://json.nlohmann.me/api/basic_json/operator_gt/ + template::value, int>::type = 0> + friend bool operator>(const_reference lhs, ScalarType rhs) noexcept + { + return lhs > basic_json(rhs); + } + + /// @brief comparison: greater than + /// @sa https://json.nlohmann.me/api/basic_json/operator_gt/ + template::value, int>::type = 0> + friend bool operator>(ScalarType lhs, const_reference rhs) noexcept + { + return basic_json(lhs) > rhs; + } + + /// @brief comparison: greater than or equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/ + friend bool operator>=(const_reference lhs, const_reference rhs) noexcept + { + return !(lhs < rhs); + } + + /// @brief comparison: greater than or equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/ + template::value, int>::type = 0> + friend bool operator>=(const_reference lhs, ScalarType rhs) noexcept + { + return lhs >= basic_json(rhs); + } + + /// @brief comparison: greater than or equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/ + template::value, int>::type = 0> + friend bool operator>=(ScalarType lhs, const_reference rhs) noexcept + { + return basic_json(lhs) >= rhs; + } + + /// @} + + /////////////////// + // serialization // + /////////////////// + + /// @name serialization + /// @{ +#ifndef JSON_NO_IO + /// @brief serialize to stream + /// @sa https://json.nlohmann.me/api/basic_json/operator_ltlt/ + friend std::ostream& operator<<(std::ostream& o, const basic_json& j) + { + // read width member and use it as indentation parameter if nonzero + const bool pretty_print = o.width() > 0; + const auto indentation = pretty_print ? o.width() : 0; + + // reset width to 0 for subsequent calls to this stream + o.width(0); + + // do the actual serialization + serializer s(detail::output_adapter(o), o.fill()); + s.dump(j, pretty_print, false, static_cast(indentation)); + return o; + } + + /// @brief serialize to stream + /// @sa https://json.nlohmann.me/api/basic_json/operator_ltlt/ + /// @deprecated This function is deprecated since 3.0.0 and will be removed in + /// version 4.0.0 of the library. Please use + /// operator<<(std::ostream&, const basic_json&) instead; that is, + /// replace calls like `j >> o;` with `o << j;`. + JSON_HEDLEY_DEPRECATED_FOR(3.0.0, operator<<(std::ostream&, const basic_json&)) + friend std::ostream& operator>>(const basic_json& j, std::ostream& o) + { + return o << j; + } +#endif // JSON_NO_IO + /// @} + + + ///////////////////// + // deserialization // + ///////////////////// + + /// @name deserialization + /// @{ + + /// @brief deserialize from a compatible input + /// @sa https://json.nlohmann.me/api/basic_json/parse/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json parse(InputType&& i, + const parser_callback_t cb = nullptr, + const bool allow_exceptions = true, + const bool ignore_comments = false) + { + basic_json result; + parser(detail::input_adapter(std::forward(i)), cb, allow_exceptions, ignore_comments).parse(true, result); + return result; + } + + /// @brief deserialize from a pair of character iterators + /// @sa https://json.nlohmann.me/api/basic_json/parse/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json parse(IteratorType first, + IteratorType last, + const parser_callback_t cb = nullptr, + const bool allow_exceptions = true, + const bool ignore_comments = false) + { + basic_json result; + parser(detail::input_adapter(std::move(first), std::move(last)), cb, allow_exceptions, ignore_comments).parse(true, result); + return result; + } + + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, parse(ptr, ptr + len)) + static basic_json parse(detail::span_input_adapter&& i, + const parser_callback_t cb = nullptr, + const bool allow_exceptions = true, + const bool ignore_comments = false) + { + basic_json result; + parser(i.get(), cb, allow_exceptions, ignore_comments).parse(true, result); + return result; + } + + /// @brief check if the input is valid JSON + /// @sa https://json.nlohmann.me/api/basic_json/accept/ + template + static bool accept(InputType&& i, + const bool ignore_comments = false) + { + return parser(detail::input_adapter(std::forward(i)), nullptr, false, ignore_comments).accept(true); + } + + /// @brief check if the input is valid JSON + /// @sa https://json.nlohmann.me/api/basic_json/accept/ + template + static bool accept(IteratorType first, IteratorType last, + const bool ignore_comments = false) + { + return parser(detail::input_adapter(std::move(first), std::move(last)), nullptr, false, ignore_comments).accept(true); + } + + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, accept(ptr, ptr + len)) + static bool accept(detail::span_input_adapter&& i, + const bool ignore_comments = false) + { + return parser(i.get(), nullptr, false, ignore_comments).accept(true); + } + + /// @brief generate SAX events + /// @sa https://json.nlohmann.me/api/basic_json/sax_parse/ + template + JSON_HEDLEY_NON_NULL(2) + static bool sax_parse(InputType&& i, SAX* sax, + input_format_t format = input_format_t::json, + const bool strict = true, + const bool ignore_comments = false) + { + auto ia = detail::input_adapter(std::forward(i)); + return format == input_format_t::json + ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict) + : detail::binary_reader(std::move(ia)).sax_parse(format, sax, strict); + } + + /// @brief generate SAX events + /// @sa https://json.nlohmann.me/api/basic_json/sax_parse/ + template + JSON_HEDLEY_NON_NULL(3) + static bool sax_parse(IteratorType first, IteratorType last, SAX* sax, + input_format_t format = input_format_t::json, + const bool strict = true, + const bool ignore_comments = false) + { + auto ia = detail::input_adapter(std::move(first), std::move(last)); + return format == input_format_t::json + ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict) + : detail::binary_reader(std::move(ia)).sax_parse(format, sax, strict); + } + + /// @brief generate SAX events + /// @sa https://json.nlohmann.me/api/basic_json/sax_parse/ + /// @deprecated This function is deprecated since 3.8.0 and will be removed in + /// version 4.0.0 of the library. Please use + /// sax_parse(ptr, ptr + len) instead. + template + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, sax_parse(ptr, ptr + len, ...)) + JSON_HEDLEY_NON_NULL(2) + static bool sax_parse(detail::span_input_adapter&& i, SAX* sax, + input_format_t format = input_format_t::json, + const bool strict = true, + const bool ignore_comments = false) + { + auto ia = i.get(); + return format == input_format_t::json + // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) + ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict) + // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) + : detail::binary_reader(std::move(ia)).sax_parse(format, sax, strict); + } +#ifndef JSON_NO_IO + /// @brief deserialize from stream + /// @sa https://json.nlohmann.me/api/basic_json/operator_gtgt/ + /// @deprecated This stream operator is deprecated since 3.0.0 and will be removed in + /// version 4.0.0 of the library. Please use + /// operator>>(std::istream&, basic_json&) instead; that is, + /// replace calls like `j << i;` with `i >> j;`. + JSON_HEDLEY_DEPRECATED_FOR(3.0.0, operator>>(std::istream&, basic_json&)) + friend std::istream& operator<<(basic_json& j, std::istream& i) + { + return operator>>(i, j); + } + + /// @brief deserialize from stream + /// @sa https://json.nlohmann.me/api/basic_json/operator_gtgt/ + friend std::istream& operator>>(std::istream& i, basic_json& j) + { + parser(detail::input_adapter(i)).parse(false, j); + return i; + } +#endif // JSON_NO_IO + /// @} + + /////////////////////////// + // convenience functions // + /////////////////////////// + + /// @brief return the type as string + /// @sa https://json.nlohmann.me/api/basic_json/type_name/ + JSON_HEDLEY_RETURNS_NON_NULL + const char* type_name() const noexcept + { + switch (m_type) + { + case value_t::null: + return "null"; + case value_t::object: + return "object"; + case value_t::array: + return "array"; + case value_t::string: + return "string"; + case value_t::boolean: + return "boolean"; + case value_t::binary: + return "binary"; + case value_t::discarded: + return "discarded"; + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + default: + return "number"; + } + } + + + JSON_PRIVATE_UNLESS_TESTED: + ////////////////////// + // member variables // + ////////////////////// + + /// the type of the current element + value_t m_type = value_t::null; + + /// the value of the current element + json_value m_value = {}; + +#if JSON_DIAGNOSTICS + /// a pointer to a parent value (for debugging purposes) + basic_json* m_parent = nullptr; +#endif + + ////////////////////////////////////////// + // binary serialization/deserialization // + ////////////////////////////////////////// + + /// @name binary serialization/deserialization support + /// @{ + + public: + /// @brief create a CBOR serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_cbor/ + static std::vector to_cbor(const basic_json& j) + { + std::vector result; + to_cbor(j, result); + return result; + } + + /// @brief create a CBOR serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_cbor/ + static void to_cbor(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_cbor(j); + } + + /// @brief create a CBOR serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_cbor/ + static void to_cbor(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_cbor(j); + } + + /// @brief create a MessagePack serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_msgpack/ + static std::vector to_msgpack(const basic_json& j) + { + std::vector result; + to_msgpack(j, result); + return result; + } + + /// @brief create a MessagePack serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_msgpack/ + static void to_msgpack(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_msgpack(j); + } + + /// @brief create a MessagePack serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_msgpack/ + static void to_msgpack(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_msgpack(j); + } + + /// @brief create a UBJSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_ubjson/ + static std::vector to_ubjson(const basic_json& j, + const bool use_size = false, + const bool use_type = false) + { + std::vector result; + to_ubjson(j, result, use_size, use_type); + return result; + } + + /// @brief create a UBJSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_ubjson/ + static void to_ubjson(const basic_json& j, detail::output_adapter o, + const bool use_size = false, const bool use_type = false) + { + binary_writer(o).write_ubjson(j, use_size, use_type); + } + + /// @brief create a UBJSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_ubjson/ + static void to_ubjson(const basic_json& j, detail::output_adapter o, + const bool use_size = false, const bool use_type = false) + { + binary_writer(o).write_ubjson(j, use_size, use_type); + } + + /// @brief create a BSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bson/ + static std::vector to_bson(const basic_json& j) + { + std::vector result; + to_bson(j, result); + return result; + } + + /// @brief create a BSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bson/ + static void to_bson(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_bson(j); + } + + /// @brief create a BSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bson/ + static void to_bson(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_bson(j); + } + + /// @brief create a JSON value from an input in CBOR format + /// @sa https://json.nlohmann.me/api/basic_json/from_cbor/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_cbor(InputType&& i, + const bool strict = true, + const bool allow_exceptions = true, + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::forward(i)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in CBOR format + /// @sa https://json.nlohmann.me/api/basic_json/from_cbor/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_cbor(IteratorType first, IteratorType last, + const bool strict = true, + const bool allow_exceptions = true, + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::move(first), std::move(last)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); + return res ? result : basic_json(value_t::discarded); + } + + template + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_cbor(ptr, ptr + len)) + static basic_json from_cbor(const T* ptr, std::size_t len, + const bool strict = true, + const bool allow_exceptions = true, + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) + { + return from_cbor(ptr, ptr + len, strict, allow_exceptions, tag_handler); + } + + + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_cbor(ptr, ptr + len)) + static basic_json from_cbor(detail::span_input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true, + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = i.get(); + // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in MessagePack format + /// @sa https://json.nlohmann.me/api/basic_json/from_msgpack/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_msgpack(InputType&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::forward(i)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::msgpack, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in MessagePack format + /// @sa https://json.nlohmann.me/api/basic_json/from_msgpack/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_msgpack(IteratorType first, IteratorType last, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::move(first), std::move(last)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::msgpack, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + template + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_msgpack(ptr, ptr + len)) + static basic_json from_msgpack(const T* ptr, std::size_t len, + const bool strict = true, + const bool allow_exceptions = true) + { + return from_msgpack(ptr, ptr + len, strict, allow_exceptions); + } + + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_msgpack(ptr, ptr + len)) + static basic_json from_msgpack(detail::span_input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = i.get(); + // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::msgpack, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in UBJSON format + /// @sa https://json.nlohmann.me/api/basic_json/from_ubjson/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_ubjson(InputType&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::forward(i)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::ubjson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in UBJSON format + /// @sa https://json.nlohmann.me/api/basic_json/from_ubjson/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_ubjson(IteratorType first, IteratorType last, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::move(first), std::move(last)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::ubjson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + template + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_ubjson(ptr, ptr + len)) + static basic_json from_ubjson(const T* ptr, std::size_t len, + const bool strict = true, + const bool allow_exceptions = true) + { + return from_ubjson(ptr, ptr + len, strict, allow_exceptions); + } + + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_ubjson(ptr, ptr + len)) + static basic_json from_ubjson(detail::span_input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = i.get(); + // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::ubjson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in BSON format + /// @sa https://json.nlohmann.me/api/basic_json/from_bson/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_bson(InputType&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::forward(i)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::bson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in BSON format + /// @sa https://json.nlohmann.me/api/basic_json/from_bson/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_bson(IteratorType first, IteratorType last, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::move(first), std::move(last)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::bson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + template + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_bson(ptr, ptr + len)) + static basic_json from_bson(const T* ptr, std::size_t len, + const bool strict = true, + const bool allow_exceptions = true) + { + return from_bson(ptr, ptr + len, strict, allow_exceptions); + } + + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_bson(ptr, ptr + len)) + static basic_json from_bson(detail::span_input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = i.get(); + // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::bson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + /// @} + + ////////////////////////// + // JSON Pointer support // + ////////////////////////// + + /// @name JSON Pointer functions + /// @{ + + /// @brief access specified element via JSON Pointer + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + reference operator[](const json_pointer& ptr) + { + return ptr.get_unchecked(this); + } + + /// @brief access specified element via JSON Pointer + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + const_reference operator[](const json_pointer& ptr) const + { + return ptr.get_unchecked(this); + } + + /// @brief access specified element via JSON Pointer + /// @sa https://json.nlohmann.me/api/basic_json/at/ + reference at(const json_pointer& ptr) + { + return ptr.get_checked(this); + } + + /// @brief access specified element via JSON Pointer + /// @sa https://json.nlohmann.me/api/basic_json/at/ + const_reference at(const json_pointer& ptr) const + { + return ptr.get_checked(this); + } + + /// @brief return flattened JSON value + /// @sa https://json.nlohmann.me/api/basic_json/flatten/ + basic_json flatten() const + { + basic_json result(value_t::object); + json_pointer::flatten("", *this, result); + return result; + } + + /// @brief unflatten a previously flattened JSON value + /// @sa https://json.nlohmann.me/api/basic_json/unflatten/ + basic_json unflatten() const + { + return json_pointer::unflatten(*this); + } + + /// @} + + ////////////////////////// + // JSON Patch functions // + ////////////////////////// + + /// @name JSON Patch functions + /// @{ + + /// @brief applies a JSON patch + /// @sa https://json.nlohmann.me/api/basic_json/patch/ + basic_json patch(const basic_json& json_patch) const + { + // make a working copy to apply the patch to + basic_json result = *this; + + // the valid JSON Patch operations + enum class patch_operations {add, remove, replace, move, copy, test, invalid}; + + const auto get_op = [](const std::string & op) + { + if (op == "add") + { + return patch_operations::add; + } + if (op == "remove") + { + return patch_operations::remove; + } + if (op == "replace") + { + return patch_operations::replace; + } + if (op == "move") + { + return patch_operations::move; + } + if (op == "copy") + { + return patch_operations::copy; + } + if (op == "test") + { + return patch_operations::test; + } + + return patch_operations::invalid; + }; + + // wrapper for "add" operation; add value at ptr + const auto operation_add = [&result](json_pointer & ptr, basic_json val) + { + // adding to the root of the target document means replacing it + if (ptr.empty()) + { + result = val; + return; + } + + // make sure the top element of the pointer exists + json_pointer top_pointer = ptr.top(); + if (top_pointer != ptr) + { + result.at(top_pointer); + } + + // get reference to parent of JSON pointer ptr + const auto last_path = ptr.back(); + ptr.pop_back(); + basic_json& parent = result[ptr]; + + switch (parent.m_type) + { + case value_t::null: + case value_t::object: + { + // use operator[] to add value + parent[last_path] = val; + break; + } + + case value_t::array: + { + if (last_path == "-") + { + // special case: append to back + parent.push_back(val); + } + else + { + const auto idx = json_pointer::array_index(last_path); + if (JSON_HEDLEY_UNLIKELY(idx > parent.size())) + { + // avoid undefined behavior + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range", parent)); + } + + // default case: insert add offset + parent.insert(parent.begin() + static_cast(idx), val); + } + break; + } + + // if there exists a parent it cannot be primitive + case value_t::string: // LCOV_EXCL_LINE + case value_t::boolean: // LCOV_EXCL_LINE + case value_t::number_integer: // LCOV_EXCL_LINE + case value_t::number_unsigned: // LCOV_EXCL_LINE + case value_t::number_float: // LCOV_EXCL_LINE + case value_t::binary: // LCOV_EXCL_LINE + case value_t::discarded: // LCOV_EXCL_LINE + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + }; + + // wrapper for "remove" operation; remove value at ptr + const auto operation_remove = [this, &result](json_pointer & ptr) + { + // get reference to parent of JSON pointer ptr + const auto last_path = ptr.back(); + ptr.pop_back(); + basic_json& parent = result.at(ptr); + + // remove child + if (parent.is_object()) + { + // perform range check + auto it = parent.find(last_path); + if (JSON_HEDLEY_LIKELY(it != parent.end())) + { + parent.erase(it); + } + else + { + JSON_THROW(out_of_range::create(403, "key '" + last_path + "' not found", *this)); + } + } + else if (parent.is_array()) + { + // note erase performs range check + parent.erase(json_pointer::array_index(last_path)); + } + }; + + // type check: top level value must be an array + if (JSON_HEDLEY_UNLIKELY(!json_patch.is_array())) + { + JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects", json_patch)); + } + + // iterate and apply the operations + for (const auto& val : json_patch) + { + // wrapper to get a value for an operation + const auto get_value = [&val](const std::string & op, + const std::string & member, + bool string_type) -> basic_json & + { + // find value + auto it = val.m_value.object->find(member); + + // context-sensitive error message + const auto error_msg = (op == "op") ? "operation" : "operation '" + op + "'"; + + // check if desired value is present + if (JSON_HEDLEY_UNLIKELY(it == val.m_value.object->end())) + { + // NOLINTNEXTLINE(performance-inefficient-string-concatenation) + JSON_THROW(parse_error::create(105, 0, error_msg + " must have member '" + member + "'", val)); + } + + // check if result is of type string + if (JSON_HEDLEY_UNLIKELY(string_type && !it->second.is_string())) + { + // NOLINTNEXTLINE(performance-inefficient-string-concatenation) + JSON_THROW(parse_error::create(105, 0, error_msg + " must have string member '" + member + "'", val)); + } + + // no error: return value + return it->second; + }; + + // type check: every element of the array must be an object + if (JSON_HEDLEY_UNLIKELY(!val.is_object())) + { + JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects", val)); + } + + // collect mandatory members + const auto op = get_value("op", "op", true).template get(); + const auto path = get_value(op, "path", true).template get(); + json_pointer ptr(path); + + switch (get_op(op)) + { + case patch_operations::add: + { + operation_add(ptr, get_value("add", "value", false)); + break; + } + + case patch_operations::remove: + { + operation_remove(ptr); + break; + } + + case patch_operations::replace: + { + // the "path" location must exist - use at() + result.at(ptr) = get_value("replace", "value", false); + break; + } + + case patch_operations::move: + { + const auto from_path = get_value("move", "from", true).template get(); + json_pointer from_ptr(from_path); + + // the "from" location must exist - use at() + basic_json v = result.at(from_ptr); + + // The move operation is functionally identical to a + // "remove" operation on the "from" location, followed + // immediately by an "add" operation at the target + // location with the value that was just removed. + operation_remove(from_ptr); + operation_add(ptr, v); + break; + } + + case patch_operations::copy: + { + const auto from_path = get_value("copy", "from", true).template get(); + const json_pointer from_ptr(from_path); + + // the "from" location must exist - use at() + basic_json v = result.at(from_ptr); + + // The copy is functionally identical to an "add" + // operation at the target location using the value + // specified in the "from" member. + operation_add(ptr, v); + break; + } + + case patch_operations::test: + { + bool success = false; + JSON_TRY + { + // check if "value" matches the one at "path" + // the "path" location must exist - use at() + success = (result.at(ptr) == get_value("test", "value", false)); + } + JSON_INTERNAL_CATCH (out_of_range&) + { + // ignore out of range errors: success remains false + } + + // throw an exception if test fails + if (JSON_HEDLEY_UNLIKELY(!success)) + { + JSON_THROW(other_error::create(501, "unsuccessful: " + val.dump(), val)); + } + + break; + } + + case patch_operations::invalid: + default: + { + // op must be "add", "remove", "replace", "move", "copy", or + // "test" + JSON_THROW(parse_error::create(105, 0, "operation value '" + op + "' is invalid", val)); + } + } + } + + return result; + } + + /// @brief creates a diff as a JSON patch + /// @sa https://json.nlohmann.me/api/basic_json/diff/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json diff(const basic_json& source, const basic_json& target, + const std::string& path = "") + { + // the patch + basic_json result(value_t::array); + + // if the values are the same, return empty patch + if (source == target) + { + return result; + } + + if (source.type() != target.type()) + { + // different types: replace value + result.push_back( + { + {"op", "replace"}, {"path", path}, {"value", target} + }); + return result; + } + + switch (source.type()) + { + case value_t::array: + { + // first pass: traverse common elements + std::size_t i = 0; + while (i < source.size() && i < target.size()) + { + // recursive call to compare array values at index i + auto temp_diff = diff(source[i], target[i], path + "/" + std::to_string(i)); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + ++i; + } + + // We now reached the end of at least one array + // in a second pass, traverse the remaining elements + + // remove my remaining elements + const auto end_index = static_cast(result.size()); + while (i < source.size()) + { + // add operations in reverse order to avoid invalid + // indices + result.insert(result.begin() + end_index, object( + { + {"op", "remove"}, + {"path", path + "/" + std::to_string(i)} + })); + ++i; + } + + // add other remaining elements + while (i < target.size()) + { + result.push_back( + { + {"op", "add"}, + {"path", path + "/-"}, + {"value", target[i]} + }); + ++i; + } + + break; + } + + case value_t::object: + { + // first pass: traverse this object's elements + for (auto it = source.cbegin(); it != source.cend(); ++it) + { + // escape the key name to be used in a JSON patch + const auto path_key = path + "/" + detail::escape(it.key()); + + if (target.find(it.key()) != target.end()) + { + // recursive call to compare object values at key it + auto temp_diff = diff(it.value(), target[it.key()], path_key); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + } + else + { + // found a key that is not in o -> remove it + result.push_back(object( + { + {"op", "remove"}, {"path", path_key} + })); + } + } + + // second pass: traverse other object's elements + for (auto it = target.cbegin(); it != target.cend(); ++it) + { + if (source.find(it.key()) == source.end()) + { + // found a key that is not in this -> add it + const auto path_key = path + "/" + detail::escape(it.key()); + result.push_back( + { + {"op", "add"}, {"path", path_key}, + {"value", it.value()} + }); + } + } + + break; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + // both primitive type: replace value + result.push_back( + { + {"op", "replace"}, {"path", path}, {"value", target} + }); + break; + } + } + + return result; + } + + /// @} + + //////////////////////////////// + // JSON Merge Patch functions // + //////////////////////////////// + + /// @name JSON Merge Patch functions + /// @{ + + /// @brief applies a JSON Merge Patch + /// @sa https://json.nlohmann.me/api/basic_json/merge_patch/ + void merge_patch(const basic_json& apply_patch) + { + if (apply_patch.is_object()) + { + if (!is_object()) + { + *this = object(); + } + for (auto it = apply_patch.begin(); it != apply_patch.end(); ++it) + { + if (it.value().is_null()) + { + erase(it.key()); + } + else + { + operator[](it.key()).merge_patch(it.value()); + } + } + } + else + { + *this = apply_patch; + } + } + + /// @} +}; + +/// @brief user-defined to_string function for JSON values +/// @sa https://json.nlohmann.me/api/basic_json/to_string/ +NLOHMANN_BASIC_JSON_TPL_DECLARATION +std::string to_string(const NLOHMANN_BASIC_JSON_TPL& j) +{ + return j.dump(); +} + +} // namespace nlohmann + +/////////////////////// +// nonmember support // +/////////////////////// + +namespace std // NOLINT(cert-dcl58-cpp) +{ + +/// @brief hash value for JSON objects +/// @sa https://json.nlohmann.me/api/basic_json/std_hash/ +NLOHMANN_BASIC_JSON_TPL_DECLARATION +struct hash +{ + std::size_t operator()(const nlohmann::NLOHMANN_BASIC_JSON_TPL& j) const + { + return nlohmann::detail::hash(j); + } +}; + +// specialization for std::less +template<> +struct less< ::nlohmann::detail::value_t> // do not remove the space after '<', see https://github.com/nlohmann/json/pull/679 +{ + /*! + @brief compare two value_t enum values + @since version 3.0.0 + */ + bool operator()(nlohmann::detail::value_t lhs, + nlohmann::detail::value_t rhs) const noexcept + { + return nlohmann::detail::operator<(lhs, rhs); + } +}; + +// C++20 prohibit function specialization in the std namespace. +#ifndef JSON_HAS_CPP_20 + +/// @brief exchanges the values of two JSON objects +/// @sa https://json.nlohmann.me/api/basic_json/std_swap/ +NLOHMANN_BASIC_JSON_TPL_DECLARATION +inline void swap(nlohmann::NLOHMANN_BASIC_JSON_TPL& j1, nlohmann::NLOHMANN_BASIC_JSON_TPL& j2) noexcept( // NOLINT(readability-inconsistent-declaration-parameter-name) + is_nothrow_move_constructible::value&& // NOLINT(misc-redundant-expression) + is_nothrow_move_assignable::value) +{ + j1.swap(j2); +} + +#endif + +} // namespace std + +/// @brief user-defined string literal for JSON values +/// @sa https://json.nlohmann.me/api/basic_json/operator_literal_json/ +JSON_HEDLEY_NON_NULL(1) +inline nlohmann::json operator "" _json(const char* s, std::size_t n) +{ + return nlohmann::json::parse(s, s + n); +} + +/// @brief user-defined string literal for JSON pointer +/// @sa https://json.nlohmann.me/api/basic_json/operator_literal_json_pointer/ +JSON_HEDLEY_NON_NULL(1) +inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std::size_t n) +{ + return nlohmann::json::json_pointer(std::string(s, n)); +} + +// #include + + +// restore clang diagnostic settings +#if defined(__clang__) + #pragma clang diagnostic pop +#endif + +// clean up +#undef JSON_ASSERT +#undef JSON_INTERNAL_CATCH +#undef JSON_CATCH +#undef JSON_THROW +#undef JSON_TRY +#undef JSON_PRIVATE_UNLESS_TESTED +#undef JSON_HAS_CPP_11 +#undef JSON_HAS_CPP_14 +#undef JSON_HAS_CPP_17 +#undef JSON_HAS_CPP_20 +#undef JSON_HAS_FILESYSTEM +#undef JSON_HAS_EXPERIMENTAL_FILESYSTEM +#undef NLOHMANN_BASIC_JSON_TPL_DECLARATION +#undef NLOHMANN_BASIC_JSON_TPL +#undef JSON_EXPLICIT +#undef NLOHMANN_CAN_CALL_STD_FUNC_IMPL + +// #include + + +#undef JSON_HEDLEY_ALWAYS_INLINE +#undef JSON_HEDLEY_ARM_VERSION +#undef JSON_HEDLEY_ARM_VERSION_CHECK +#undef JSON_HEDLEY_ARRAY_PARAM +#undef JSON_HEDLEY_ASSUME +#undef JSON_HEDLEY_BEGIN_C_DECLS +#undef JSON_HEDLEY_CLANG_HAS_ATTRIBUTE +#undef JSON_HEDLEY_CLANG_HAS_BUILTIN +#undef JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE +#undef JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE +#undef JSON_HEDLEY_CLANG_HAS_EXTENSION +#undef JSON_HEDLEY_CLANG_HAS_FEATURE +#undef JSON_HEDLEY_CLANG_HAS_WARNING +#undef JSON_HEDLEY_COMPCERT_VERSION +#undef JSON_HEDLEY_COMPCERT_VERSION_CHECK +#undef JSON_HEDLEY_CONCAT +#undef JSON_HEDLEY_CONCAT3 +#undef JSON_HEDLEY_CONCAT3_EX +#undef JSON_HEDLEY_CONCAT_EX +#undef JSON_HEDLEY_CONST +#undef JSON_HEDLEY_CONSTEXPR +#undef JSON_HEDLEY_CONST_CAST +#undef JSON_HEDLEY_CPP_CAST +#undef JSON_HEDLEY_CRAY_VERSION +#undef JSON_HEDLEY_CRAY_VERSION_CHECK +#undef JSON_HEDLEY_C_DECL +#undef JSON_HEDLEY_DEPRECATED +#undef JSON_HEDLEY_DEPRECATED_FOR +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_ +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION +#undef JSON_HEDLEY_DIAGNOSTIC_POP +#undef JSON_HEDLEY_DIAGNOSTIC_PUSH +#undef JSON_HEDLEY_DMC_VERSION +#undef JSON_HEDLEY_DMC_VERSION_CHECK +#undef JSON_HEDLEY_EMPTY_BASES +#undef JSON_HEDLEY_EMSCRIPTEN_VERSION +#undef JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK +#undef JSON_HEDLEY_END_C_DECLS +#undef JSON_HEDLEY_FLAGS +#undef JSON_HEDLEY_FLAGS_CAST +#undef JSON_HEDLEY_GCC_HAS_ATTRIBUTE +#undef JSON_HEDLEY_GCC_HAS_BUILTIN +#undef JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE +#undef JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE +#undef JSON_HEDLEY_GCC_HAS_EXTENSION +#undef JSON_HEDLEY_GCC_HAS_FEATURE +#undef JSON_HEDLEY_GCC_HAS_WARNING +#undef JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK +#undef JSON_HEDLEY_GCC_VERSION +#undef JSON_HEDLEY_GCC_VERSION_CHECK +#undef JSON_HEDLEY_GNUC_HAS_ATTRIBUTE +#undef JSON_HEDLEY_GNUC_HAS_BUILTIN +#undef JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE +#undef JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE +#undef JSON_HEDLEY_GNUC_HAS_EXTENSION +#undef JSON_HEDLEY_GNUC_HAS_FEATURE +#undef JSON_HEDLEY_GNUC_HAS_WARNING +#undef JSON_HEDLEY_GNUC_VERSION +#undef JSON_HEDLEY_GNUC_VERSION_CHECK +#undef JSON_HEDLEY_HAS_ATTRIBUTE +#undef JSON_HEDLEY_HAS_BUILTIN +#undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE +#undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS +#undef JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE +#undef JSON_HEDLEY_HAS_EXTENSION +#undef JSON_HEDLEY_HAS_FEATURE +#undef JSON_HEDLEY_HAS_WARNING +#undef JSON_HEDLEY_IAR_VERSION +#undef JSON_HEDLEY_IAR_VERSION_CHECK +#undef JSON_HEDLEY_IBM_VERSION +#undef JSON_HEDLEY_IBM_VERSION_CHECK +#undef JSON_HEDLEY_IMPORT +#undef JSON_HEDLEY_INLINE +#undef JSON_HEDLEY_INTEL_CL_VERSION +#undef JSON_HEDLEY_INTEL_CL_VERSION_CHECK +#undef JSON_HEDLEY_INTEL_VERSION +#undef JSON_HEDLEY_INTEL_VERSION_CHECK +#undef JSON_HEDLEY_IS_CONSTANT +#undef JSON_HEDLEY_IS_CONSTEXPR_ +#undef JSON_HEDLEY_LIKELY +#undef JSON_HEDLEY_MALLOC +#undef JSON_HEDLEY_MCST_LCC_VERSION +#undef JSON_HEDLEY_MCST_LCC_VERSION_CHECK +#undef JSON_HEDLEY_MESSAGE +#undef JSON_HEDLEY_MSVC_VERSION +#undef JSON_HEDLEY_MSVC_VERSION_CHECK +#undef JSON_HEDLEY_NEVER_INLINE +#undef JSON_HEDLEY_NON_NULL +#undef JSON_HEDLEY_NO_ESCAPE +#undef JSON_HEDLEY_NO_RETURN +#undef JSON_HEDLEY_NO_THROW +#undef JSON_HEDLEY_NULL +#undef JSON_HEDLEY_PELLES_VERSION +#undef JSON_HEDLEY_PELLES_VERSION_CHECK +#undef JSON_HEDLEY_PGI_VERSION +#undef JSON_HEDLEY_PGI_VERSION_CHECK +#undef JSON_HEDLEY_PREDICT +#undef JSON_HEDLEY_PRINTF_FORMAT +#undef JSON_HEDLEY_PRIVATE +#undef JSON_HEDLEY_PUBLIC +#undef JSON_HEDLEY_PURE +#undef JSON_HEDLEY_REINTERPRET_CAST +#undef JSON_HEDLEY_REQUIRE +#undef JSON_HEDLEY_REQUIRE_CONSTEXPR +#undef JSON_HEDLEY_REQUIRE_MSG +#undef JSON_HEDLEY_RESTRICT +#undef JSON_HEDLEY_RETURNS_NON_NULL +#undef JSON_HEDLEY_SENTINEL +#undef JSON_HEDLEY_STATIC_ASSERT +#undef JSON_HEDLEY_STATIC_CAST +#undef JSON_HEDLEY_STRINGIFY +#undef JSON_HEDLEY_STRINGIFY_EX +#undef JSON_HEDLEY_SUNPRO_VERSION +#undef JSON_HEDLEY_SUNPRO_VERSION_CHECK +#undef JSON_HEDLEY_TINYC_VERSION +#undef JSON_HEDLEY_TINYC_VERSION_CHECK +#undef JSON_HEDLEY_TI_ARMCL_VERSION +#undef JSON_HEDLEY_TI_ARMCL_VERSION_CHECK +#undef JSON_HEDLEY_TI_CL2000_VERSION +#undef JSON_HEDLEY_TI_CL2000_VERSION_CHECK +#undef JSON_HEDLEY_TI_CL430_VERSION +#undef JSON_HEDLEY_TI_CL430_VERSION_CHECK +#undef JSON_HEDLEY_TI_CL6X_VERSION +#undef JSON_HEDLEY_TI_CL6X_VERSION_CHECK +#undef JSON_HEDLEY_TI_CL7X_VERSION +#undef JSON_HEDLEY_TI_CL7X_VERSION_CHECK +#undef JSON_HEDLEY_TI_CLPRU_VERSION +#undef JSON_HEDLEY_TI_CLPRU_VERSION_CHECK +#undef JSON_HEDLEY_TI_VERSION +#undef JSON_HEDLEY_TI_VERSION_CHECK +#undef JSON_HEDLEY_UNAVAILABLE +#undef JSON_HEDLEY_UNLIKELY +#undef JSON_HEDLEY_UNPREDICTABLE +#undef JSON_HEDLEY_UNREACHABLE +#undef JSON_HEDLEY_UNREACHABLE_RETURN +#undef JSON_HEDLEY_VERSION +#undef JSON_HEDLEY_VERSION_DECODE_MAJOR +#undef JSON_HEDLEY_VERSION_DECODE_MINOR +#undef JSON_HEDLEY_VERSION_DECODE_REVISION +#undef JSON_HEDLEY_VERSION_ENCODE +#undef JSON_HEDLEY_WARNING +#undef JSON_HEDLEY_WARN_UNUSED_RESULT +#undef JSON_HEDLEY_WARN_UNUSED_RESULT_MSG +#undef JSON_HEDLEY_FALL_THROUGH + + + +#endif // INCLUDE_NLOHMANN_JSON_HPP_ diff --git a/utils/libmysql_client/libmysql_client.h b/utils/libmysql_client/libmysql_client.h index 4207a7054..931886b78 100644 --- a/utils/libmysql_client/libmysql_client.h +++ b/utils/libmysql_client/libmysql_client.h @@ -91,4 +91,3 @@ class LibMySQL #endif // UTILS_LIBMYSQL_CL_H -// vim:ts=4 sw=4: diff --git a/utils/loggingcpp/errorcodes.cpp b/utils/loggingcpp/errorcodes.cpp index e52a5e32d..c8e879c02 100644 --- a/utils/loggingcpp/errorcodes.cpp +++ b/utils/loggingcpp/errorcodes.cpp @@ -103,4 +103,3 @@ string ErrorCodes::errorString(uint16_t code) const return (fPreamble + msg); } } // namespace logging -// vim:ts=4 sw=4: diff --git a/utils/loggingcpp/exceptclasses.h b/utils/loggingcpp/exceptclasses.h index d1139e89b..456436066 100644 --- a/utils/loggingcpp/exceptclasses.h +++ b/utils/loggingcpp/exceptclasses.h @@ -305,5 +305,3 @@ class ProtocolError : public std::logic_error } while (0) } // namespace logging - -// vim:ts=4 sw=4: diff --git a/utils/loggingcpp/format.h b/utils/loggingcpp/format.h new file mode 100644 index 000000000..f12d019ce --- /dev/null +++ b/utils/loggingcpp/format.h @@ -0,0 +1,92 @@ +/* + Copyright (c) 2021 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. +*/ + +#pragma once + +#include +#include +#include +#include +#include + +namespace logging +{ +template +void formatOne(std::string& errMsg, Iter iter, uint32_t position) +{ + T arg = boost::any_cast(*iter); + std::string token = std::string("%") + std::to_string(position) + std::string("%"); + size_t index = 0; + + while (true) + { + index = errMsg.find(token, index); + if (index == std::string::npos) + break; + + if constexpr (std::is_same_v) + { + errMsg.replace(index, token.length(), arg); + } + else + { + errMsg.replace(index, token.length(), std::to_string(arg)); + } + + index += token.length(); + } +} + +template +void formatMany(std::string& errMsg, const T& args) +{ + auto iter = args.begin(); + auto end = args.end(); + uint32_t position = 1; + + while (iter != end) + { + if (iter->type() == typeid(long)) + { + formatOne(errMsg, iter, position); + } + else if (iter->type() == typeid(uint64_t)) + { + formatOne(errMsg, iter, position); + } + else if (iter->type() == typeid(double)) + { + formatOne(errMsg, iter, position); + } + else if (iter->type() == typeid(std::string)) + { + formatOne(errMsg, iter, position); + } + else + { + throw std::logic_error("logggin::format: unexpected type in argslist"); + } + ++iter; + ++position; + } + static std::regex restToken("%[0-9]%"); + errMsg = std::regex_replace(errMsg, restToken, ""); +} + +} // namespace logging \ No newline at end of file diff --git a/utils/loggingcpp/idberrorinfo.cpp b/utils/loggingcpp/idberrorinfo.cpp index 2b0054881..dbcd5c920 100644 --- a/utils/loggingcpp/idberrorinfo.cpp +++ b/utils/loggingcpp/idberrorinfo.cpp @@ -29,7 +29,6 @@ #include using namespace std; -#include #include #include using namespace boost; @@ -43,6 +42,8 @@ using namespace config; #include "installdir.h" +#include "format.h" + namespace logging { IDBErrorInfo* IDBErrorInfo::fInstance = 0; @@ -156,43 +157,7 @@ string IDBErrorInfo::logError(const logging::LOG_TYPE logLevel, const logging::L void IDBErrorInfo::format(string& errMsg, const Message::Args& args) { - Message::Args::AnyVec::const_iterator iter = args.args().begin(); - Message::Args::AnyVec::const_iterator end = args.args().end(); - - boost::format fmt(errMsg); - fmt.exceptions(boost::io::no_error_bits); - - while (iter != end) - { - if (iter->type() == typeid(long)) - { - long l = any_cast(*iter); - fmt % l; - } - else if (iter->type() == typeid(uint64_t)) - { - uint64_t u64 = any_cast(*iter); - fmt % u64; - } - else if (iter->type() == typeid(double)) - { - double d = any_cast(*iter); - fmt % d; - } - else if (iter->type() == typeid(string)) - { - string s = any_cast(*iter); - fmt % s; - } - else - { - throw logic_error("IDBErrorInfo::format: unexpected type in argslist"); - } - - ++iter; - } - - errMsg = fmt.str(); + formatMany(errMsg, args.args()); } /* static */ diff --git a/utils/loggingcpp/message.cpp b/utils/loggingcpp/message.cpp index 316923f6f..f8edf9703 100644 --- a/utils/loggingcpp/message.cpp +++ b/utils/loggingcpp/message.cpp @@ -29,7 +29,7 @@ #include using namespace std; -#include + #include #include using namespace boost; @@ -41,6 +41,7 @@ using namespace config; #include "installdir.h" +#include "format.h" namespace { boost::mutex mx; @@ -143,43 +144,7 @@ void Message::Args::reset() void Message::format(const Args& args) { - Args::AnyVec::const_iterator iter = args.args().begin(); - Args::AnyVec::const_iterator end = args.args().end(); - - boost::format fmt(fMsg); - fmt.exceptions(boost::io::no_error_bits); - - while (iter != end) - { - if (iter->type() == typeid(long)) - { - long l = any_cast(*iter); - fmt % l; - } - else if (iter->type() == typeid(uint64_t)) - { - uint64_t u64 = any_cast(*iter); - fmt % u64; - } - else if (iter->type() == typeid(double)) - { - double d = any_cast(*iter); - fmt % d; - } - else if (iter->type() == typeid(string)) - { - string s = any_cast(*iter); - fmt % s; - } - else - { - throw logic_error("Message::format: unexpected type in argslist"); - } - - ++iter; - } - - fMsg = fmt.str(); + formatMany(fMsg, args.args()); } /* static */ diff --git a/utils/messageqcpp/inetstreamsocket.cpp b/utils/messageqcpp/inetstreamsocket.cpp index 81ed82f6d..45f605bcb 100644 --- a/utils/messageqcpp/inetstreamsocket.cpp +++ b/utils/messageqcpp/inetstreamsocket.cpp @@ -101,8 +101,6 @@ using boost::scoped_array; // some static functions namespace { -using messageqcpp::ByteStream; - // @bug 2441 - Retry after 512 read() error. // ERESTARTSYS (512) is a kernal I/O errno that is similar to a EINTR, except // that it is not supposed to "leak" out into the user space. But we are @@ -937,16 +935,8 @@ void InetStreamSocket::connect(const sockaddr* serv_addr) char buf = '\0'; (void)::recv(socketParms().sd(), &buf, 1, 0); #else -#if defined(__GNUC__) && __GNUC__ >= 5 -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-result" char buf = '\0'; - ::read(socketParms().sd(), &buf, 1); // we know 1 byte is in the recv buffer -#pragma GCC diagnostic pop -#else - char buf = '\0'; - ::read(socketParms().sd(), &buf, 1); // we know 1 byte is in the recv buffer -#endif // pragma + std::ignore = ::read(socketParms().sd(), &buf, 1); // we know 1 byte is in the recv buffer #endif return; } diff --git a/utils/messageqcpp/messagequeue.h b/utils/messageqcpp/messagequeue.h index 5dfd02f94..99a63cbdf 100644 --- a/utils/messageqcpp/messagequeue.h +++ b/utils/messageqcpp/messagequeue.h @@ -336,4 +336,3 @@ inline void MessageQueueClient::syncProto(bool use) #undef EXPORT -// vim:ts=4 sw=4: diff --git a/utils/multicast/multicast.cpp b/utils/multicast/multicast.cpp index 116f0382c..b471c170f 100644 --- a/utils/multicast/multicast.cpp +++ b/utils/multicast/multicast.cpp @@ -93,4 +93,3 @@ void MulticastSender::send(const ByteStream& msg) } // namespace multicast -// vim:ts=4 sw=4: diff --git a/utils/regr/regrmysql.cpp b/utils/regr/regrmysql.cpp index 1b480aa84..d9e9a51b6 100644 --- a/utils/regr/regrmysql.cpp +++ b/utils/regr/regrmysql.cpp @@ -1405,4 +1405,3 @@ extern "C" return valOut; } } -// vim:ts=4 sw=4: diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index b27cd91a8..1faf18211 100644 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -1,6 +1,6 @@ /* Copyright (C) 2014 InfiniDB, Inc. - Copyright (c) 2019-2020 MariaDB Corporation + Copyright (c) 2019-2021 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -1567,7 +1567,8 @@ void RowAggregation::serialize(messageqcpp::ByteStream& bs) const for (uint64_t i = 0; i < functionCount; i++) fFunctionCols[i]->serialize(bs); - bs << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + bs << timeZone; } //------------------------------------------------------------------------------ @@ -1612,7 +1613,9 @@ void RowAggregation::deserialize(messageqcpp::ByteStream& bs) fFunctionCols.push_back(funct); } - bs >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + bs >> timeZone; + fTimeZone = timeZone; } //------------------------------------------------------------------------------ diff --git a/utils/rowgroup/rowaggregation.h b/utils/rowgroup/rowaggregation.h index 360f08128..a5abc4b40 100644 --- a/utils/rowgroup/rowaggregation.h +++ b/utils/rowgroup/rowaggregation.h @@ -338,7 +338,7 @@ struct GroupConcat std::vector> fOrderCond; // position to order by [asc/desc] joblist::ResourceManager* fRm; // resource manager boost::shared_ptr fSessionMemLimit; - std::string fTimeZone; + long fTimeZone; GroupConcat() : fRm(nullptr) { @@ -505,11 +505,11 @@ class RowAggregation : public messageqcpp::Serializeable return fAggMapKeyCount; } - inline void timeZone(const std::string& timeZone) + inline void timeZone(long timeZone) { fTimeZone = timeZone; } - inline const std::string& timeZone() const + inline long timeZone() const { return fTimeZone; } @@ -598,7 +598,7 @@ class RowAggregation : public messageqcpp::Serializeable bool fKeyOnHeap = false; - std::string fTimeZone; + long fTimeZone; // We need a separate copy for each thread. mcsv1sdk::mcsv1Context fRGContext; diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 4e767e1d8..7aa41477b 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -800,7 +800,8 @@ template <> inline bool Row::isNullValue_offset(uint32_t offset) const { const int128_t* intPtr = reinterpret_cast(&data[offset]); - return datatypes::Decimal::isWideDecimalNullValue(*intPtr); + const datatypes::TSInt128 value(intPtr); + return datatypes::Decimal::isWideDecimalNullValue(value.getValue()); } template <> @@ -1751,4 +1752,3 @@ RowGroup RowGroup::truncate(uint32_t cols) } // namespace rowgroup -// vim:ts=4 sw=4: diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 087371214..c3ecab709 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -2183,4 +2183,3 @@ inline void RGData::getRow(uint32_t num, Row* row) } // namespace rowgroup -// vim:ts=4 sw=4: diff --git a/utils/rwlock/rwlock.cpp b/utils/rwlock/rwlock.cpp index 73a9c5db2..ca7b82e9d 100644 --- a/utils/rwlock/rwlock.cpp +++ b/utils/rwlock/rwlock.cpp @@ -746,4 +746,3 @@ LockState RWLock::getLockState() } } // namespace rwlock -// vim:ts=4 sw=4: diff --git a/utils/rwlock/rwlock.h b/utils/rwlock/rwlock.h index ec47a0c70..a0beaf572 100644 --- a/utils/rwlock/rwlock.h +++ b/utils/rwlock/rwlock.h @@ -1,4 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2016-2022 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -26,6 +27,8 @@ #pragma once +#include + #include #include @@ -42,20 +45,17 @@ namespace rwlock { +const std::array RWLockNames = { + "all", "VSS", "ExtentMap", "FreeList", "VBBM", "CopyLocks", "ExtentMapIndex", +}; + /// the layout of the shmseg struct State { -#ifdef _MSC_VER - volatile LONG writerswaiting; - volatile LONG writing; - volatile LONG readerswaiting; - volatile LONG reading; -#else - volatile int writerswaiting; - volatile int writing; - volatile int readerswaiting; - volatile int reading; -#endif + int writerswaiting; + int writing; + int readerswaiting; + int reading; boost::interprocess::interprocess_semaphore sems[3]; }; @@ -275,4 +275,3 @@ class RWLock #undef EXPORT -// vim:ts=4 sw=4: diff --git a/utils/startup/installdir.cpp b/utils/startup/installdir.cpp index 65437a9c1..2db2128db 100644 --- a/utils/startup/installdir.cpp +++ b/utils/startup/installdir.cpp @@ -112,4 +112,3 @@ const string StartUp::tmpDir() } } // namespace startup -// vim:ts=4 sw=4: diff --git a/utils/startup/installdir.h b/utils/startup/installdir.h index bee02fdb4..8195601a9 100644 --- a/utils/startup/installdir.h +++ b/utils/startup/installdir.h @@ -52,4 +52,3 @@ class StartUp } // namespace startup -// vim:ts=4 sw=4: diff --git a/utils/testbc/iomanager.cpp b/utils/testbc/iomanager.cpp index 7c3cc4d6c..b2944e5b7 100644 --- a/utils/testbc/iomanager.cpp +++ b/utils/testbc/iomanager.cpp @@ -325,4 +325,3 @@ fileRequest* ioManager::getNextRequest() } } // namespace dbbc -// vim:ts=4 sw=4: diff --git a/utils/testbc/iomanager.h b/utils/testbc/iomanager.h index 3ad953ded..f9dae5b5e 100644 --- a/utils/testbc/iomanager.h +++ b/utils/testbc/iomanager.h @@ -98,4 +98,3 @@ class ioManager }; } // namespace dbbc -// vim:ts=4 sw=4: diff --git a/utils/threadpool/CMakeLists.txt b/utils/threadpool/CMakeLists.txt index d20887ed6..519ba3b5c 100644 --- a/utils/threadpool/CMakeLists.txt +++ b/utils/threadpool/CMakeLists.txt @@ -9,5 +9,6 @@ set(threadpool_LIB_SRCS weightedthreadpool.cpp threadpool.cpp prioritythreadpool add_library(threadpool SHARED ${threadpool_LIB_SRCS}) add_dependencies(threadpool loggingcpp) +target_link_libraries(threadpool Boost::chrono) install(TARGETS threadpool DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) diff --git a/utils/threadpool/prioritythreadpool.cpp b/utils/threadpool/prioritythreadpool.cpp index b26144b62..908c0ba97 100644 --- a/utils/threadpool/prioritythreadpool.cpp +++ b/utils/threadpool/prioritythreadpool.cpp @@ -322,4 +322,3 @@ void PriorityThreadPool::stop() } } // namespace threadpool -// vim:ts=4 sw=4: diff --git a/utils/udfsdk/udfmysql.cpp b/utils/udfsdk/udfmysql.cpp index 552729421..f32fc6b3a 100644 --- a/utils/udfsdk/udfmysql.cpp +++ b/utils/udfsdk/udfmysql.cpp @@ -535,4 +535,3 @@ extern "C" return 0; } } -// vim:ts=4 sw=4: diff --git a/utils/udfsdk/udfsdk.cpp b/utils/udfsdk/udfsdk.cpp index 0cf806181..dc5e4fa86 100644 --- a/utils/udfsdk/udfsdk.cpp +++ b/utils/udfsdk/udfsdk.cpp @@ -375,4 +375,3 @@ int64_t MCS_isnull::getDatetimeIntVal(Row& row, FunctionParm& parm, bool& isNull } } // namespace udfsdk -// vim:ts=4 sw=4: diff --git a/utils/udfsdk/udfsdk.h b/utils/udfsdk/udfsdk.h index 9e53601b3..6c0ee013b 100644 --- a/utils/udfsdk/udfsdk.h +++ b/utils/udfsdk/udfsdk.h @@ -314,4 +314,3 @@ class MCS_isnull : public funcexp::Func #undef EXPORT -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/framebound.cpp b/utils/windowfunction/framebound.cpp index 043620f53..56b50d2e0 100644 --- a/utils/windowfunction/framebound.cpp +++ b/utils/windowfunction/framebound.cpp @@ -72,4 +72,3 @@ const string FrameBound::toString() const } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/framebound.h b/utils/windowfunction/framebound.h index 1664f9c8b..7dd4ad179 100644 --- a/utils/windowfunction/framebound.h +++ b/utils/windowfunction/framebound.h @@ -145,4 +145,3 @@ extern std::map colType2String; } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/frameboundrange.cpp b/utils/windowfunction/frameboundrange.cpp index 0b356bce7..e884d9303 100644 --- a/utils/windowfunction/frameboundrange.cpp +++ b/utils/windowfunction/frameboundrange.cpp @@ -411,4 +411,3 @@ template class FrameBoundExpressionRange; template class FrameBoundExpressionRange; } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/frameboundrange.h b/utils/windowfunction/frameboundrange.h index d221e59fd..5526a0b04 100644 --- a/utils/windowfunction/frameboundrange.h +++ b/utils/windowfunction/frameboundrange.h @@ -213,4 +213,3 @@ class FrameBoundExpressionRange : public FrameBoundConstantRange } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/frameboundrow.cpp b/utils/windowfunction/frameboundrow.cpp index 616b1e0eb..3d4427413 100644 --- a/utils/windowfunction/frameboundrow.cpp +++ b/utils/windowfunction/frameboundrow.cpp @@ -139,4 +139,3 @@ template class FrameBoundExpressionRow; template class FrameBoundExpressionRow; } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/frameboundrow.h b/utils/windowfunction/frameboundrow.h index f7d4ec26c..79c071e49 100644 --- a/utils/windowfunction/frameboundrow.h +++ b/utils/windowfunction/frameboundrow.h @@ -146,4 +146,3 @@ class FrameBoundExpressionRow : public FrameBoundConstantRow } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/idborderby.cpp b/utils/windowfunction/idborderby.cpp index 30d442e74..5160eed26 100644 --- a/utils/windowfunction/idborderby.cpp +++ b/utils/windowfunction/idborderby.cpp @@ -898,4 +898,3 @@ bool IdbOrderBy::Eq::operator()(const Row::Pointer& d1, const Row::Pointer& d2) } } // namespace ordering -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_count.cpp b/utils/windowfunction/wf_count.cpp index c45cd2978..bc4694762 100644 --- a/utils/windowfunction/wf_count.cpp +++ b/utils/windowfunction/wf_count.cpp @@ -184,4 +184,3 @@ template boost::shared_ptr WF_count::makeFunction(i WindowFunctionColumn*); } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_count.h b/utils/windowfunction/wf_count.h index 34b880589..cb0844788 100644 --- a/utils/windowfunction/wf_count.h +++ b/utils/windowfunction/wf_count.h @@ -47,4 +47,3 @@ class WF_count : public WindowFunctionType } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_lead_lag.cpp b/utils/windowfunction/wf_lead_lag.cpp index 4e7de53ff..b3babe8b8 100644 --- a/utils/windowfunction/wf_lead_lag.cpp +++ b/utils/windowfunction/wf_lead_lag.cpp @@ -306,4 +306,3 @@ template void WF_lead_lag::parseParms(const std::vector& template void WF_lead_lag::parseParms(const std::vector&); } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_lead_lag.h b/utils/windowfunction/wf_lead_lag.h index 2aac7265d..7ccb41062 100644 --- a/utils/windowfunction/wf_lead_lag.h +++ b/utils/windowfunction/wf_lead_lag.h @@ -52,4 +52,3 @@ class WF_lead_lag : public WindowFunctionType } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_min_max.cpp b/utils/windowfunction/wf_min_max.cpp index 4120fe32a..b5b63bde7 100644 --- a/utils/windowfunction/wf_min_max.cpp +++ b/utils/windowfunction/wf_min_max.cpp @@ -185,4 +185,3 @@ template boost::shared_ptr WF_min_max::makeFunction WindowFunctionColumn*); } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_min_max.h b/utils/windowfunction/wf_min_max.h index a296b8f19..caf615191 100644 --- a/utils/windowfunction/wf_min_max.h +++ b/utils/windowfunction/wf_min_max.h @@ -46,4 +46,3 @@ class WF_min_max : public WindowFunctionType } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_nth_value.cpp b/utils/windowfunction/wf_nth_value.cpp index 338984a04..f3c9e2a1b 100644 --- a/utils/windowfunction/wf_nth_value.cpp +++ b/utils/windowfunction/wf_nth_value.cpp @@ -271,4 +271,3 @@ template boost::shared_ptr WF_nth_value::makeFuncti WindowFunctionColumn*); } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_nth_value.h b/utils/windowfunction/wf_nth_value.h index a4bf2c5f3..d4a2247ff 100644 --- a/utils/windowfunction/wf_nth_value.h +++ b/utils/windowfunction/wf_nth_value.h @@ -50,4 +50,3 @@ class WF_nth_value : public WindowFunctionType } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_ntile.cpp b/utils/windowfunction/wf_ntile.cpp index 375fd81d2..4b6634603 100644 --- a/utils/windowfunction/wf_ntile.cpp +++ b/utils/windowfunction/wf_ntile.cpp @@ -154,4 +154,3 @@ void WF_ntile::operator()(int64_t b, int64_t e, int64_t c) } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_ntile.h b/utils/windowfunction/wf_ntile.h index e5c7278b0..ed5ee6b77 100644 --- a/utils/windowfunction/wf_ntile.h +++ b/utils/windowfunction/wf_ntile.h @@ -47,4 +47,3 @@ class WF_ntile : public WindowFunctionType } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_percentile.cpp b/utils/windowfunction/wf_percentile.cpp index 4d0dfce07..ef2c12548 100644 --- a/utils/windowfunction/wf_percentile.cpp +++ b/utils/windowfunction/wf_percentile.cpp @@ -335,7 +335,10 @@ void WF_percentile::operator()(int64_t b, int64_t e, int64_t c) vd = (crn - rn) * fv + (rn - frn) * cv; } - v = *(reinterpret_cast(&vd)); + double tempvd[2]; + tempvd[0] = vd; + tempvd[1] = 0; + v = *(reinterpret_cast(&tempvd[0])); // old code that referred to 'vd' var triggered partial out-of-bounds warning. p = &v; } else // (fFunctionId == WF__PERCENTILE_DISC) @@ -382,4 +385,3 @@ template boost::shared_ptr WF_percentile::makeFunct WindowFunctionColumn*); } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_percentile.h b/utils/windowfunction/wf_percentile.h index da50a04e9..db08c08d2 100644 --- a/utils/windowfunction/wf_percentile.h +++ b/utils/windowfunction/wf_percentile.h @@ -48,4 +48,3 @@ class WF_percentile : public WindowFunctionType } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_ranking.cpp b/utils/windowfunction/wf_ranking.cpp index f0459b7fe..540ca8082 100644 --- a/utils/windowfunction/wf_ranking.cpp +++ b/utils/windowfunction/wf_ranking.cpp @@ -145,4 +145,3 @@ void WF_ranking::operator()(int64_t b, int64_t e, int64_t c) } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_ranking.h b/utils/windowfunction/wf_ranking.h index 748c9607e..ecfdae97a 100644 --- a/utils/windowfunction/wf_ranking.h +++ b/utils/windowfunction/wf_ranking.h @@ -46,4 +46,3 @@ class WF_ranking : public WindowFunctionType } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_row_number.cpp b/utils/windowfunction/wf_row_number.cpp index 6453c0a08..6dd58ce31 100644 --- a/utils/windowfunction/wf_row_number.cpp +++ b/utils/windowfunction/wf_row_number.cpp @@ -86,4 +86,3 @@ void WF_row_number::operator()(int64_t b, int64_t e, int64_t c) } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_row_number.h b/utils/windowfunction/wf_row_number.h index f7c4e4ca3..953a68c35 100644 --- a/utils/windowfunction/wf_row_number.h +++ b/utils/windowfunction/wf_row_number.h @@ -45,4 +45,3 @@ class WF_row_number : public WindowFunctionType } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_stats.cpp b/utils/windowfunction/wf_stats.cpp index b7a01de79..d4df1b995 100644 --- a/utils/windowfunction/wf_stats.cpp +++ b/utils/windowfunction/wf_stats.cpp @@ -239,4 +239,3 @@ template boost::shared_ptr WF_stats::makeFunction(i WindowFunctionColumn*); } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_stats.h b/utils/windowfunction/wf_stats.h index fd72c7b3d..a443ad322 100644 --- a/utils/windowfunction/wf_stats.h +++ b/utils/windowfunction/wf_stats.h @@ -48,4 +48,3 @@ class WF_stats : public WindowFunctionType } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_sum_avg.cpp b/utils/windowfunction/wf_sum_avg.cpp index d988f0a14..573b6e097 100644 --- a/utils/windowfunction/wf_sum_avg.cpp +++ b/utils/windowfunction/wf_sum_avg.cpp @@ -306,4 +306,3 @@ template boost::shared_ptr WF_sum_avg: int, const string&, int, WindowFunctionColumn*); } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_sum_avg.h b/utils/windowfunction/wf_sum_avg.h index 2ad329b9b..c423e7617 100644 --- a/utils/windowfunction/wf_sum_avg.h +++ b/utils/windowfunction/wf_sum_avg.h @@ -60,4 +60,3 @@ class WF_sum_avg : public WindowFunctionType } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_udaf.cpp b/utils/windowfunction/wf_udaf.cpp index 274c97018..0894c6ad1 100644 --- a/utils/windowfunction/wf_udaf.cpp +++ b/utils/windowfunction/wf_udaf.cpp @@ -1191,4 +1191,3 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) fPrev = c; } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_udaf.h b/utils/windowfunction/wf_udaf.h index e3c28e317..54b2085eb 100644 --- a/utils/windowfunction/wf_udaf.h +++ b/utils/windowfunction/wf_udaf.h @@ -112,4 +112,3 @@ class WF_udaf : public WindowFunctionType } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/windowframe.cpp b/utils/windowfunction/windowframe.cpp index cbc9352c6..491e0a0fb 100644 --- a/utils/windowfunction/windowframe.cpp +++ b/utils/windowfunction/windowframe.cpp @@ -81,4 +81,3 @@ const string WindowFrame::toString() const } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/windowframe.h b/utils/windowfunction/windowframe.h index 6b3356538..2dd3c23d7 100644 --- a/utils/windowfunction/windowframe.h +++ b/utils/windowfunction/windowframe.h @@ -119,4 +119,3 @@ class WindowFrame } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/windowfunction.cpp b/utils/windowfunction/windowfunction.cpp index a0e844933..941fff911 100644 --- a/utils/windowfunction/windowfunction.cpp +++ b/utils/windowfunction/windowfunction.cpp @@ -259,4 +259,3 @@ void WindowFunction::sort(std::vector::iterator v, uint64_t n) } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/windowfunction.h b/utils/windowfunction/windowfunction.h index fc6569b27..6bdd41dde 100644 --- a/utils/windowfunction/windowfunction.h +++ b/utils/windowfunction/windowfunction.h @@ -111,4 +111,3 @@ class WindowFunction } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/windowfunctiontype.cpp b/utils/windowfunction/windowfunctiontype.cpp index ccf74609d..43f05f4bc 100644 --- a/utils/windowfunction/windowfunctiontype.cpp +++ b/utils/windowfunction/windowfunctiontype.cpp @@ -797,4 +797,3 @@ void WindowFunctionType::constParms(const std::vector& functionParms) } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/windowfunctiontype.h b/utils/windowfunction/windowfunctiontype.h index 40b7a54a8..ef59444cc 100644 --- a/utils/windowfunction/windowfunctiontype.h +++ b/utils/windowfunction/windowfunctiontype.h @@ -303,4 +303,3 @@ extern std::map colType2String; } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/winport/fixup.cpp b/utils/winport/fixup.cpp index dfba53a50..2e909d564 100644 --- a/utils/winport/fixup.cpp +++ b/utils/winport/fixup.cpp @@ -258,22 +258,6 @@ int fixupCalpontXML() cout << "Fixing " << section << "." << parm << " = " << val << endl; } - // Fixup WES - section = "pm1_WriteEngineServer"; - parm = "IPAddr"; - val = cf->getConfig(section, parm); - - if (val.empty()) - { - val = "127.0.0.1"; - cf->setConfig(section, parm, val); - cout << "Adding " << section << "." << parm << " = " << val << endl; - parm = "Port"; - val = "8630"; - cf->setConfig(section, parm, val); - cout << "Adding " << section << "." << parm << " = " << val << endl; - } - // Fixup TableLockSaveFile section = "SystemConfig"; parm = "TableLockSaveFile"; diff --git a/utils/winport/grepit.cpp b/utils/winport/grepit.cpp index f20182efa..d6948fd0e 100644 --- a/utils/winport/grepit.cpp +++ b/utils/winport/grepit.cpp @@ -19,7 +19,7 @@ #include using namespace std; -#include +#include #include "grepit.h" @@ -32,13 +32,13 @@ namespace winport { bool grepit(istream& is, const string& pattern) { - boost::regex pat(pattern); + std::regex pat(pattern); string cInput; getline(is, cInput); while (is.good()) { - if (boost::regex_match(cInput, pat)) + if (std::regex_match(cInput, pat)) return true; getline(is, cInput); diff --git a/versioning/BRM/brmshmimpl.cpp b/versioning/BRM/brmshmimpl.cpp index 2c25caf8b..44750c630 100644 --- a/versioning/BRM/brmshmimpl.cpp +++ b/versioning/BRM/brmshmimpl.cpp @@ -40,7 +40,15 @@ namespace bi = boost::interprocess; namespace BRM { -BRMShmImpl::BRMShmImpl(unsigned key, off_t size, bool readOnly) : fKey(key), fSize(size), fReadOnly(readOnly) +const constexpr uint32_t ShmCreateMaxRetries = 10; +const constexpr unsigned int NapTimer = 500000; + +BRMShmImplParent::BRMShmImplParent(unsigned key, off_t size, bool readOnly) + : fKey(key), fSize(size), fReadOnly(readOnly){}; + +BRMShmImplParent::~BRMShmImplParent(){}; + +BRMShmImpl::BRMShmImpl(unsigned key, off_t size, bool readOnly) : BRMShmImplParent(key, size, readOnly) { string keyName = ShmKeys::keyToName(fKey); @@ -238,6 +246,167 @@ void BRMShmImpl::destroy() bi::shared_memory_object::remove(oldName.c_str()); } +BRMManagedShmImpl::BRMManagedShmImpl(unsigned key, off_t size, bool readOnly) + : BRMShmImplParent(key, size, readOnly) +{ + string keyName = ShmKeys::keyToName(fKey); + off_t curSize = 0; + + for (uint32_t tries = 0; fSize == 0 && tries <= ShmCreateMaxRetries; ++tries) + { + try + { + auto* shmSegment = new boost::interprocess::managed_shared_memory(bi::open_only, keyName.c_str()); + curSize = shmSegment->get_size(); + + if (curSize == 0) + { + delete shmSegment; + throw bi::interprocess_exception("shared memory segment size is 0."); + } + else + { + fShmSegment = shmSegment; + fSize = curSize; + return; + } + } + catch (bi::interprocess_exception&) + { + if (tries == ShmCreateMaxRetries) + { + log("BRMManagedShmImpl::BRMManagedShmImpl(): re-creating shared memory segment\ + b/c of its size == 0. Re-throw."); + throw; + } + + cerr << "BRMManagedShmImpl::BRMManagedShmImpl(): re-creating shared memory segment\ + b/c of its size == 0" + << endl; + usleep(NapTimer); + } + } + + try + { + bi::permissions perms; + perms.set_unrestricted(); + fShmSegment = new bi::managed_shared_memory(bi::create_only, keyName.c_str(), fSize, + 0, // use a default address to map the segment + perms); + // fSize == 0 on any process startup but managed_shared_memory ctor throws + // so control flow doesn't get here. + idbassert(fSize > 0); + } + catch (bi::interprocess_exception& b) + { + if (b.get_error_code() != bi::already_exists_error) + { + ostringstream o; + o << "BRM caught an exception creating a shared memory segment: " << b.what(); + log(o.str()); + throw; + } + bi::managed_shared_memory* shmSegment = nullptr; + try + { + if (fReadOnly) + shmSegment = new bi::managed_shared_memory(bi::open_read_only, keyName.c_str()); + else + shmSegment = new bi::managed_shared_memory(bi::open_only, keyName.c_str()); + } + catch (exception& e) + { + ostringstream o; + o << "BRM caught an exception attaching to a shared memory segment (" << keyName << "): " << b.what(); + log(o.str()); + throw; + } + off_t curSize = shmSegment->get_size(); + + idbassert(curSize > 0); + idbassert(curSize >= fSize); + fShmSegment = shmSegment; + fSize = curSize; + } +} + +int BRMManagedShmImpl::grow(off_t newSize) +{ + auto keyName = ShmKeys::keyToName(fKey); + + if (newSize > fSize) + { + const auto incSize = newSize - fSize; + if (fShmSegment) + { + // Call destructor to unmap the segment. + delete fShmSegment; + // Grow the segment. + bi::managed_shared_memory::grow(keyName.c_str(), incSize); + // Open only with the assumption ::grow() can be called on read-write shmem. + fShmSegment = new bi::managed_shared_memory(bi::open_only, keyName.c_str()); + // Update size. + fSize = newSize; + } + } + + return 0; +} + +// Dummy method that has no references in the code. +int BRMManagedShmImpl::clear(unsigned newKey, off_t newSize) +{ + return 0; +} + +// This method calls for all related shmem pointers to be refreshed. +void BRMManagedShmImpl::setReadOnly() +{ + if (fReadOnly) + return; + const bool readOnly = true; + remap(readOnly); + fReadOnly = true; +} + +void BRMManagedShmImpl::swap(BRMManagedShmImpl& rhs) +{ + fShmSegment->swap(*rhs.fShmSegment); + std::swap(fKey, rhs.fKey); + std::swap(fSize, rhs.fSize); + std::swap(fReadOnly, rhs.fReadOnly); +} + +// The method was copied from non-managed shmem impl class +// and it has no refences in MCS 6.x code. +void BRMManagedShmImpl::destroy() +{ + string keyName = ShmKeys::keyToName(fKey); + try + { + bi::shared_memory_object::remove(keyName.c_str()); + } + catch (bi::interprocess_exception& b) + { + std::ostringstream o; + o << "BRMManagedShmImpl::destroy caught an exception removing a managed shared memory segment: " + << b.what(); + log(o.str()); + throw; + } +} + +void BRMManagedShmImpl::remap(const bool readOnly) +{ + delete fShmSegment; + fShmSegment = nullptr; + string keyName = ShmKeys::keyToName(fKey); + if (readOnly) + fShmSegment = new bi::managed_shared_memory(bi::open_read_only, keyName.c_str()); + else + fShmSegment = new bi::managed_shared_memory(bi::open_only, keyName.c_str()); +} + } // namespace BRM -// vim:ts=4 sw=4: diff --git a/versioning/BRM/brmshmimpl.h b/versioning/BRM/brmshmimpl.h index a34d40c5f..fa1f0cfee 100644 --- a/versioning/BRM/brmshmimpl.h +++ b/versioning/BRM/brmshmimpl.h @@ -30,17 +30,18 @@ //#define NDEBUG #include #include +#include #include +namespace bi = boost::interprocess; + namespace BRM { -class BRMShmImpl +class BRMShmImplParent { public: - BRMShmImpl(unsigned key, off_t size, bool readOnly = false); - ~BRMShmImpl() - { - } + BRMShmImplParent(unsigned key, off_t size, bool readOnly = false); + virtual ~BRMShmImplParent(); inline unsigned key() const { @@ -55,23 +56,63 @@ class BRMShmImpl return fReadOnly; } - void setReadOnly(); - int grow(unsigned newKey, off_t newSize); - int clear(unsigned newKey, off_t newSize); - - void swap(BRMShmImpl& rhs); - void destroy(); - - boost::interprocess::shared_memory_object fShmobj; - boost::interprocess::mapped_region fMapreg; - - private: - BRMShmImpl(const BRMShmImpl& rhs); - BRMShmImpl& operator=(const BRMShmImpl& rhs); + virtual void setReadOnly() = 0; + virtual int clear(unsigned newKey, off_t newSize) = 0; + virtual void destroy() = 0; + protected: unsigned fKey; off_t fSize; bool fReadOnly; }; +class BRMShmImpl : public BRMShmImplParent +{ + public: + BRMShmImpl(unsigned key, off_t size, bool readOnly = false); + BRMShmImpl(const BRMShmImpl& rhs) = delete; + BRMShmImpl& operator=(const BRMShmImpl& rhs) = delete; + ~BRMShmImpl() + { + } + + int clear(unsigned newKey, off_t newSize) override; + void destroy() override; + void setReadOnly() override; + + int grow(unsigned newKey, off_t newSize); + void swap(BRMShmImpl& rhs); + + bi::shared_memory_object fShmobj; + bi::mapped_region fMapreg; +}; + +class BRMManagedShmImpl : public BRMShmImplParent +{ + public: + BRMManagedShmImpl(unsigned key, off_t size, bool readOnly = false); + BRMManagedShmImpl(const BRMManagedShmImpl& rhs) = delete; + BRMManagedShmImpl& operator=(const BRMManagedShmImpl& rhs) = delete; + ~BRMManagedShmImpl() + { + delete fShmSegment; + } + + int clear(unsigned newKey, off_t newSize) override; + void destroy() override; + void setReadOnly() override; + + int grow(off_t newSize); + void remap(const bool readOnly = false); + void swap(BRMManagedShmImpl& rhs); + bi::managed_shared_memory* getManagedSegment() + { + assert(fShmSegment); + return fShmSegment; + } + + private: + bi::managed_shared_memory* fShmSegment; +}; + } // namespace BRM diff --git a/versioning/BRM/dbrm.cpp b/versioning/BRM/dbrm.cpp index 1cc3fb983..2ab47d8ca 100644 --- a/versioning/BRM/dbrm.cpp +++ b/versioning/BRM/dbrm.cpp @@ -98,7 +98,7 @@ DBRM::DBRM(const DBRM& brm) throw logic_error("DBRM: Don't use the copy constructor."); } -DBRM::~DBRM() throw() +DBRM::~DBRM() { if (msgClient != NULL) MessageQueueClientPool::releaseInstance(msgClient); @@ -461,7 +461,7 @@ int DBRM::markExtentsInvalid(const vector& lbids, } template -int DBRM::getExtentMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum) throw() +int DBRM::getExtentMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum) { #ifdef BRM_INFO @@ -489,7 +489,7 @@ int DBRM::getExtentMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum) th } } -int DBRM::getExtentCPMaxMin(const LBID_t lbid, CPMaxMin& cpMaxMin) throw() +int DBRM::getExtentCPMaxMin(const LBID_t lbid, CPMaxMin& cpMaxMin) { try { @@ -4555,10 +4555,19 @@ void DBRM::invalidateUncommittedExtentLBIDs(execplan::CalpontSystemCatalog::SCN setExtentsMaxMin(cpInfos); } -template int DBRM::getExtentMaxMin(const LBID_t lbid, int128_t& max, int128_t& min, - int32_t& seqNum) throw(); +size_t DBRM::EMIndexShmemSize() +{ + return em->EMIndexShmemSize(); +} -template int DBRM::getExtentMaxMin(const LBID_t lbid, int64_t& max, int64_t& min, - int32_t& seqNum) throw(); +size_t DBRM::EMIndexShmemFree() +{ + return em->EMIndexShmemFree(); +} + +template int DBRM::getExtentMaxMin(const LBID_t lbid, int128_t& max, int128_t& min, + int32_t& seqNum); + +template int DBRM::getExtentMaxMin(const LBID_t lbid, int64_t& max, int64_t& min, int32_t& seqNum); } // namespace BRM diff --git a/versioning/BRM/dbrm.h b/versioning/BRM/dbrm.h index 38f350781..f9eaf54a0 100644 --- a/versioning/BRM/dbrm.h +++ b/versioning/BRM/dbrm.h @@ -103,7 +103,7 @@ class DBRM // The param noBRMFcns suppresses init of the ExtentMap, VSS, VBBM, and CopyLocks. // It can speed up init if the caller only needs the other structures. EXPORT DBRM(bool noBRMFcns = false); - EXPORT ~DBRM() throw(); + EXPORT ~DBRM(); EXPORT static void refreshShm() { @@ -780,12 +780,12 @@ class DBRM const std::vector& colDataTypes) DBRM_THROW; template - EXPORT int getExtentMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum) throw(); + EXPORT int getExtentMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum); EXPORT int setExtentMaxMin(const LBID_t lbid, const int64_t max, const int64_t min, const int32_t seqNum) DBRM_THROW; - EXPORT int getExtentCPMaxMin(const LBID_t lbid, CPMaxMin& cpMaxMin) throw(); + EXPORT int getExtentCPMaxMin(const LBID_t lbid, CPMaxMin& cpMaxMin); /** @brief Updates the max and min casual partitioning info for the passed extents. * @@ -984,6 +984,9 @@ class DBRM EXPORT void invalidateUncommittedExtentLBIDs(execplan::CalpontSystemCatalog::SCN txnid, bool allExtents, std::vector* plbidList = NULL); + size_t EMIndexShmemSize(); + size_t EMIndexShmemFree(); + private: DBRM(const DBRM& brm); DBRM& operator=(const DBRM& brm); diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index 6235f3017..fab238286 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -1,19 +1,21 @@ + /* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2016-2022 MariaDB Corporation - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ /***************************************************************************** * $Id: extentmap.cpp 1936 2013-07-09 22:10:29Z dhall $ @@ -45,7 +47,6 @@ #include #include -namespace bi = boost::interprocess; #include "liboamcpp.h" #include "brmtypes.h" @@ -113,6 +114,7 @@ inline void incSeqNum(int32_t& seqNum) namespace BRM { +static const char* EmIndexObjectName = "i"; //------------------------------------------------------------------------------ // EMCasualPartition_struct methods //------------------------------------------------------------------------------ @@ -217,9 +219,10 @@ bool EMEntry::operator<(const EMEntry& e) const /*static*/ boost::mutex ExtentMapImpl::fInstanceMutex; boost::mutex ExtentMap::mutex; +boost::mutex ExtentMap::emIndexMutex; /*static*/ -ExtentMapImpl* ExtentMapImpl::fInstance = 0; +ExtentMapImpl* ExtentMapImpl::fInstance = nullptr; /*static*/ ExtentMapImpl* ExtentMapImpl::makeExtentMapImpl(unsigned key, off_t size, bool readOnly) @@ -279,19 +282,316 @@ FreeListImpl::FreeListImpl(unsigned key, off_t size, bool readOnly) : fFreeList( { } +/*static*/ +std::mutex ExtentMapIndexImpl::fInstanceMutex_; + +/*static*/ +ExtentMapIndexImpl* ExtentMapIndexImpl::fInstance_ = nullptr; + +/*static*/ +ExtentMapIndexImpl* ExtentMapIndexImpl::makeExtentMapIndexImpl(unsigned key, off_t size, bool readOnly) +{ + std::lock_guard lock(fInstanceMutex_); + + if (fInstance_) + { + if (size != fInstance_->getShmemSize()) + { + fInstance_->fBRMManagedShmMemImpl_.remap(); + } + + return fInstance_; + } + + fInstance_ = new ExtentMapIndexImpl(key, size, readOnly); + fInstance_->createExtentMapIndexIfNeeded(); + + return fInstance_; +} + +ExtentMapIndexImpl::ExtentMapIndexImpl(unsigned key, off_t size, bool readOnly) + : fBRMManagedShmMemImpl_(key, size, readOnly) +{ +} + +void ExtentMapIndexImpl::createExtentMapIndexIfNeeded() +{ + // pair + auto managedShmemSearchPair = + fBRMManagedShmMemImpl_.getManagedSegment()->find(EmIndexObjectName); + if (!managedShmemSearchPair.first || managedShmemSearchPair.second == 0) + { + ShmVoidAllocator alloc(fBRMManagedShmMemImpl_.getManagedSegment()->get_segment_manager()); + fBRMManagedShmMemImpl_.getManagedSegment()->construct(EmIndexObjectName)(alloc); + } +} + +ExtentMapIndex* ExtentMapIndexImpl::get() +{ + // pair + auto managedShmemSearchPair = + fBRMManagedShmMemImpl_.getManagedSegment()->find(EmIndexObjectName); + assert(managedShmemSearchPair.first && managedShmemSearchPair.second > 0); + return managedShmemSearchPair.first; +} + +bool ExtentMapIndexImpl::growIfNeeded(const size_t memoryNeeded) +{ + auto freeShmem = getShmemFree(); + // Worst case managed segment can't get continues buffer with len = memoryNeeded + if (freeShmem < memoryNeeded) + { + const size_t currentShmemSize = getShmemSize(); + constexpr static const size_t minAllowance = 16 * 1024 * 1024; + const size_t newShmemSize = std::max(minAllowance, memoryNeeded) + currentShmemSize; + grow(newShmemSize); + return true; + } + return false; +} + +InsertUpdateShmemKeyPair ExtentMapIndexImpl::insert(const EMEntry& emEntry, const size_t emIdx) +{ + auto dbRoot = emEntry.dbRoot; + auto* extentMapIndexPtr = get(); + bool shmemHasGrown = false; + + while (dbRoot >= extentMapIndexPtr->size()) + { + const size_t memNeeded = (extentMapIndexPtr->capacity() + extraUnits_) * dbRootContainerUnitSize_; + shmemHasGrown = growIfNeeded(memNeeded); + // Need to refresh all refs and iterators b/c the local address range changed. + extentMapIndexPtr = get(); + assert(extentMapIndexPtr); + ShmVoidAllocator alloc(fBRMManagedShmMemImpl_.getManagedSegment()->get_segment_manager()); + OIDIndexContainerT oidIndices(alloc); + extentMapIndexPtr->push_back(std::move(oidIndices)); + } + auto& extentMapIndex = *extentMapIndexPtr; + return insert2ndLayerWrapper(extentMapIndex[dbRoot], emEntry, emIdx, shmemHasGrown); +} + +InsertUpdateShmemKeyPair ExtentMapIndexImpl::insert2ndLayer(OIDIndexContainerT& oids, const EMEntry& emEntry, + const size_t emIdx, const bool aShmemHasGrown) +{ + OID_t oid = emEntry.fileID; + ShmVoidAllocator alloc(fBRMManagedShmMemImpl_.getManagedSegment()->get_segment_manager()); + + PartitionIndexContainerT partitionIndex(alloc); + auto iterAndResult = oids.insert({oid, std::move(partitionIndex)}); + + if (iterAndResult.second) + { + PartitionIndexContainerT& partitionsContainer = (*iterAndResult.first).second; + return insert3dLayerWrapper(partitionsContainer, emEntry, emIdx, aShmemHasGrown); + } + else + return {false, aShmemHasGrown}; +} + +InsertUpdateShmemKeyPair ExtentMapIndexImpl::insert2ndLayerWrapper(OIDIndexContainerT& oids, + const EMEntry& emEntry, const size_t emIdx, + const bool aShmemHasGrown) +{ + OID_t oid = emEntry.fileID; + auto oidsIter = oids.find(oid); + bool shmemHasGrown = aShmemHasGrown; + if (oidsIter == oids.end()) + { + const size_t freeShmem = fBRMManagedShmMemImpl_.getManagedSegment()->get_free_memory(); + const size_t memNeeded = (oids.size() + extraUnits_) * oidContainerUnitSize_; + if (oids.load_factor() >= oids.max_load_factor() || freeShmem <= freeSpaceThreshold_) + { + // Need to refresh all refs and iterators b/c the local address range changed. + shmemHasGrown = growIfNeeded(memNeeded); + auto* extMapIndexPtr = get(); + assert(extMapIndexPtr); + auto& extMapIndex = *extMapIndexPtr; + shmemHasGrown = shmemHasGrown || aShmemHasGrown; + // The dbroot must be here b/c it was already found once in the upper insert(). + OIDIndexContainerT& refreshedOidsRef = extMapIndex[emEntry.dbRoot]; + return insert2ndLayer(refreshedOidsRef, emEntry, emIdx, shmemHasGrown); + } + return insert2ndLayer(oids, emEntry, emIdx, shmemHasGrown); + } + PartitionIndexContainerT& partitions = (*oidsIter).second; + return insert3dLayerWrapper(partitions, emEntry, emIdx, shmemHasGrown); +} + +InsertUpdateShmemKeyPair ExtentMapIndexImpl::insert3dLayer(PartitionIndexContainerT& partitions, + const EMEntry& emEntry, const size_t emIdx, + const bool aShmemHasGrown) +{ + auto partitionNumber = emEntry.partitionNum; + ShmVoidAllocator alloc(fBRMManagedShmMemImpl_.getManagedSegment()->get_segment_manager()); + ExtentMapIndicesT emIndices(alloc); + emIndices.push_back(emIdx); + auto iterAndResult = partitions.insert({partitionNumber, std::move(emIndices)}); + return {iterAndResult.second, aShmemHasGrown}; +} + +InsertUpdateShmemKeyPair ExtentMapIndexImpl::insert3dLayerWrapper(PartitionIndexContainerT& partitions, + const EMEntry& emEntry, const size_t emIdx, + const bool aShmemHasGrown) +{ + auto partitionNumber = emEntry.partitionNum; + auto partitionsIter = partitions.find(partitionNumber); + bool shmemHasGrown = aShmemHasGrown; + if (partitionsIter == partitions.end()) + { + const size_t freeShmem = fBRMManagedShmMemImpl_.getManagedSegment()->get_free_memory(); + const size_t memNeeded = + (partitions.size() + extraUnits_) * partitionContainerUnitSize_ + emIdentUnitSize_; + if (partitions.load_factor() >= partitions.max_load_factor() || freeShmem <= freeSpaceThreshold_) + { + // Need to refresh all refs and iterators b/c the local address range changed. + shmemHasGrown = growIfNeeded(memNeeded); + auto* extMapIndexPtr = get(); + assert(extMapIndexPtr); + auto& extMapIndex = *extMapIndexPtr; + shmemHasGrown = shmemHasGrown || aShmemHasGrown; + // The dbroot must be here b/c we found it once in insert(). + OIDIndexContainerT& refreshedOidsRef = extMapIndex[emEntry.dbRoot]; + auto oidsIter = refreshedOidsRef.find(emEntry.fileID); + PartitionIndexContainerT& refreshedPartitionsRef = (*oidsIter).second; + return insert3dLayer(refreshedPartitionsRef, emEntry, emIdx, shmemHasGrown); + } + return insert3dLayer(partitions, emEntry, emIdx, shmemHasGrown); + } + + ExtentMapIndicesT& emIndices = (*partitionsIter).second; + emIndices.push_back(emIdx); + return {true, shmemHasGrown}; +} + +ExtentMapIndexFindResult ExtentMapIndexImpl::find(const DBRootT dbroot, const OID_t oid, + const PartitionNumberT partitionNumber) +{ + ExtentMapIndex& emIndex = *get(); + if (dbroot >= emIndex.size()) + return {}; + return search2ndLayer(emIndex[dbroot], oid, partitionNumber); +} + +ExtentMapIndexFindResult ExtentMapIndexImpl::find(const DBRootT dbroot, const OID_t oid) +{ + ExtentMapIndex& emIndex = *get(); + if (dbroot >= emIndex.size()) + return {}; + return search2ndLayer(emIndex[dbroot], oid); +} + +ExtentMapIndexFindResult ExtentMapIndexImpl::search2ndLayer(OIDIndexContainerT& oids, const OID_t oid, + const PartitionNumberT partitionNumber) +{ + auto oidsIter = oids.find(oid); + if (oidsIter == oids.end()) + return {}; + + PartitionIndexContainerT& partitions = (*oidsIter).second; + return search3dLayer(partitions, partitionNumber); +} + +ExtentMapIndexFindResult ExtentMapIndexImpl::search2ndLayer(OIDIndexContainerT& oids, const OID_t oid) +{ + auto oidsIter = oids.find(oid); + if (oidsIter == oids.end()) + return {}; + + ExtentMapIndexFindResult result; + PartitionIndexContainerT& partitions = (*oidsIter).second; + for (auto& partKeyValue : partitions) + { + ExtentMapIndicesT& emIdentifiers = partKeyValue.second; + for (auto& emIdent : emIdentifiers) + result.push_back(emIdent); + } + + return result; +} + +ExtentMapIndexFindResult ExtentMapIndexImpl::search3dLayer(PartitionIndexContainerT& partitions, + const PartitionNumberT partitionNumber) +{ + auto partitionsIter = partitions.find(partitionNumber); + if (partitionsIter == partitions.end()) + return {}; + + ExtentMapIndexFindResult result; + ExtentMapIndicesT& emIndicesVec = (*partitionsIter).second; + for (auto& emIndex : emIndicesVec) + result.push_back(emIndex); + return result; +} + +void ExtentMapIndexImpl::deleteDbRoot(const DBRootT dbroot) +{ + auto& extMapIndex = *get(); + if (dbroot >= extMapIndex.size()) + return; // nothing to delete + extMapIndex[dbroot].clear(); +} + +void ExtentMapIndexImpl::deleteOID(const DBRootT dbroot, const OID_t oid) +{ + auto& extMapIndex = *get(); + // nothing to delete + if (dbroot >= extMapIndex.size()) + return; + if (extMapIndex[dbroot].empty()) + return; + auto oidsIter = extMapIndex[dbroot].find(oid); + // Nothing to delete. Might be a sign of a problem. + if (oidsIter == extMapIndex[dbroot].end()) + return; + extMapIndex[dbroot].erase(oidsIter); +} + +void ExtentMapIndexImpl::deleteEMEntry(const EMEntry& emEntry, const ExtentMapIdxT emIdent) +{ + // find partition + auto& extMapIndex = *get(); + if (emEntry.dbRoot >= extMapIndex.size()) + return; + if (extMapIndex[emEntry.dbRoot].empty()) + return; + auto oidsIter = extMapIndex[emEntry.dbRoot].find(emEntry.fileID); + if (oidsIter == extMapIndex[emEntry.dbRoot].end()) + return; + PartitionIndexContainerT& partitions = (*oidsIter).second; + auto partitionsIter = partitions.find(emEntry.partitionNum); + if (partitionsIter == partitions.end()) + return; + ExtentMapIndicesT& emIdentifiers = (*partitionsIter).second; + // pop the identifier + if (emIdentifiers.size() > 1) + { + auto emIdentifiersTargetIter = std::find(emIdentifiers.begin(), emIdentifiers.end(), emIdent); + std::swap(*emIdentifiersTargetIter, emIdentifiers.back()); + emIdentifiers.pop_back(); + } + else // only 1 ident in this partition + { + partitions.erase(partitionsIter); + } +} + ExtentMap::ExtentMap() { - fExtentMap = NULL; - fFreeList = NULL; + fExtentMap = nullptr; + fFreeList = nullptr; + fPExtMapImpl = nullptr; fCurrentEMShmkey = -1; fCurrentFLShmkey = -1; - fEMShminfo = NULL; - fFLShminfo = NULL; + fEMShminfo = nullptr; + fFLShminfo = nullptr; + fEMIndexShminfo = nullptr; r_only = false; flLocked = false; emLocked = false; - fPExtMapImpl = 0; - fPFreeListImpl = 0; + emIndexLocked = false; + fPFreeListImpl = nullptr; + fPExtMapIndexImpl_ = nullptr; #ifdef BRM_INFO fDebug = ("Y" == config::Config::makeConfig()->getConfig("DBRM", "Debug")); @@ -416,6 +716,7 @@ int ExtentMap::markInvalid(const LBID_t lbid, const execplan::CalpontSystemCatal #endif grabEMEntryTable(WRITE); + grabEMIndex(WRITE); return _markInvalid(lbid, colDataType); } @@ -448,6 +749,7 @@ int ExtentMap::markInvalid(const vector& lbids, #endif grabEMEntryTable(WRITE); + grabEMIndex(WRITE); // XXXPAT: what's the proper return code when one and only one fails? for (i = 0; i < size; ++i) @@ -514,6 +816,7 @@ int ExtentMap::setMaxMin(const LBID_t lbid, const int64_t max, const int64_t min #endif grabEMEntryTable(WRITE); + grabEMIndex(WRITE); entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); for (i = 0; i < entries; i++) @@ -571,8 +874,10 @@ int ExtentMap::setMaxMin(const LBID_t lbid, const int64_t max, const int64_t min if (emLocked) releaseEMEntryTable(WRITE); + if (emIndexLocked) + releaseEMIndex(WRITE); + throw logic_error("ExtentMap::setMaxMin(): lbid isn't allocated"); - // return -1; } // @bug 1970. Added updateExtentsMaxMin function. @@ -625,7 +930,10 @@ void ExtentMap::setExtentsMaxMin(const CPMaxMinMap_t& cpMap, bool firstNode, boo #endif if (useLock) + { grabEMEntryTable(WRITE); + grabEMIndex(WRITE); + } entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); @@ -759,7 +1067,7 @@ void ExtentMap::setExtentsMaxMin(const CPMaxMinMap_t& cpMap, bool firstNode, boo // @bug 1970. Added mergeExtentsMaxMin to merge CP info for list of extents. // @note - The key passed in the map must the starting LBID in the extent. // Used by cpimport to update extentmap casual partition min/max. -// NULL or empty values should not be passed in as min/max values. +// nullptr or empty values should not be passed in as min/max values. // seqNum in the input struct is not currently used. // // Note that DML calls markInvalid() to flag an extent as CP_UPDATING and incre- @@ -823,7 +1131,10 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) #endif if (useLock) + { grabEMEntryTable(WRITE); + grabEMIndex(WRITE); + } int entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); @@ -1127,6 +1438,7 @@ int ExtentMap::getMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum) #endif grabEMEntryTable(READ); + grabEMIndex(READ); entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); for (i = 0; i < entries; i++) @@ -1149,12 +1461,14 @@ int ExtentMap::getMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum) } seqNum = fExtentMap[i].partition.cprange.sequenceNum; isValid = fExtentMap[i].partition.cprange.isValid; + releaseEMIndex(READ); releaseEMEntryTable(READ); return isValid; } } } + releaseEMIndex(READ); releaseEMEntryTable(READ); throw logic_error("ExtentMap::getMaxMin(): that lbid isn't allocated"); // return -1; @@ -1323,7 +1637,6 @@ void ExtentMap::loadVersion4or5(T* in, bool upgradeV4ToV5) void* fExtentMapPtr = static_cast(fExtentMap); memset(fExtentMapPtr, 0, fEMShminfo->allocdSize); fEMShminfo->currentSize = 0; - // init the free list memset(fFreeList, 0, fFLShminfo->allocdSize); fFreeList[0].size = (1 << 26); // 2^36 LBIDs @@ -1346,13 +1659,12 @@ void ExtentMap::loadVersion4or5(T* in, bool upgradeV4ToV5) growEMShmseg(nrows); } + size_t progress = 0, writeSize = emNumElements * sizeof(EMEntry); int err; char* writePos; - size_t progress, writeSize; if (!upgradeV4ToV5) { - progress = 0; writeSize = emNumElements * sizeof(EMEntry); writePos = (char*)fExtentMap; @@ -1414,6 +1726,14 @@ void ExtentMap::loadVersion4or5(T* in, bool upgradeV4ToV5) //@bug 1911 - verify status value is valid if (fExtentMap[i].status < EXTENTSTATUSMIN || fExtentMap[i].status > EXTENTSTATUSMAX) fExtentMap[i].status = EXTENTAVAILABLE; + + auto resShmemHasGrownPair = fPExtMapIndexImpl_->insert(fExtentMap[i], i); + + if (resShmemHasGrownPair.second) + fEMIndexShminfo->allocdSize = fPExtMapIndexImpl_->getShmemSize(); + + if (!resShmemHasGrownPair.first) + logAndSetEMIndexReadOnly("loadVersion4or5"); } fEMShminfo->currentSize = emNumElements * sizeof(EMEntry); @@ -1454,6 +1774,7 @@ void ExtentMap::load(const string& filename, bool fixFL) #endif grabEMEntryTable(WRITE); + grabEMIndex(WRITE); try { @@ -1461,6 +1782,7 @@ void ExtentMap::load(const string& filename, bool fixFL) } catch (...) { + releaseEMIndex(WRITE); releaseEMEntryTable(WRITE); throw; } @@ -1473,6 +1795,7 @@ void ExtentMap::load(const string& filename, bool fixFL) { log_errno("ExtentMap::load(): open"); releaseFreeList(WRITE); + releaseEMIndex(WRITE); releaseEMEntryTable(WRITE); throw ios_base::failure("ExtentMap::load(): open failed. Check the error log."); } @@ -1485,13 +1808,14 @@ void ExtentMap::load(const string& filename, bool fixFL) catch (...) { releaseFreeList(WRITE); + releaseEMIndex(WRITE); releaseEMEntryTable(WRITE); throw; } releaseFreeList(WRITE); + releaseEMIndex(WRITE); releaseEMEntryTable(WRITE); - // checkConsistency(); } // This is a quick workaround, to be able to initialize initial system tables @@ -1535,11 +1859,13 @@ void ExtentMap::loadFromBinaryBlob(const char* blob) catch (...) { releaseFreeList(WRITE); + releaseEMIndex(WRITE); releaseEMEntryTable(WRITE); throw; } releaseFreeList(WRITE); + releaseEMIndex(WRITE); releaseEMEntryTable(WRITE); } @@ -1586,6 +1912,7 @@ void ExtentMap::save(const string& filename) int allocdSize, loadSize[3], i; grabEMEntryTable(READ); + grabEMIndex(READ); try { @@ -1593,6 +1920,7 @@ void ExtentMap::save(const string& filename) } catch (...) { + releaseEMIndex(READ); releaseEMEntryTable(READ); throw; } @@ -1601,6 +1929,7 @@ void ExtentMap::save(const string& filename) { log("ExtentMap::save(): got request to save an empty BRM"); releaseFreeList(READ); + releaseEMIndex(READ); releaseEMEntryTable(READ); throw runtime_error("ExtentMap::save(): got request to save an empty BRM"); } @@ -1613,6 +1942,7 @@ void ExtentMap::save(const string& filename) { log_errno("ExtentMap::save(): open"); releaseFreeList(READ); + releaseEMIndex(READ); releaseEMEntryTable(READ); throw ios_base::failure("ExtentMap::save(): open failed. Check the error log."); } @@ -1634,6 +1964,7 @@ void ExtentMap::save(const string& filename) catch (...) { releaseFreeList(READ); + releaseEMIndex(READ); releaseEMEntryTable(READ); throw; } @@ -1659,6 +1990,7 @@ void ExtentMap::save(const string& filename) if (err < 0) { releaseFreeList(READ); + releaseEMIndex(READ); releaseEMEntryTable(READ); throw ios_base::failure("ExtentMap::save(): write failed. Check the error log."); } @@ -1678,6 +2010,7 @@ void ExtentMap::save(const string& filename) if (err < 0) { releaseFreeList(READ); + releaseEMIndex(READ); releaseEMEntryTable(READ); throw ios_base::failure("ExtentMap::save(): write failed. Check the error log."); } @@ -1685,9 +2018,6 @@ void ExtentMap::save(const string& filename) } } - // allocdSize = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); - // const int inlineLbidRangeSize = sizeof(InlineLBIDRange); - progress = 0; writeSize = fFLShminfo->allocdSize; char* writePos = (char*)fFreeList; @@ -1697,6 +2027,7 @@ void ExtentMap::save(const string& filename) if (err < 0) { releaseFreeList(READ); + releaseEMIndex(READ); releaseEMEntryTable(READ); throw ios_base::failure("ExtentMap::save(): write failed. Check the error log."); } @@ -1705,6 +2036,7 @@ void ExtentMap::save(const string& filename) } releaseFreeList(READ); + releaseEMIndex(READ); releaseEMEntryTable(READ); } @@ -1714,7 +2046,9 @@ void ExtentMap::grabEMEntryTable(OPS op) boost::mutex::scoped_lock lk(mutex); if (op == READ) + { fEMShminfo = fMST.getTable_read(MasterSegmentTable::EMTable); + } else { fEMShminfo = fMST.getTable_write(MasterSegmentTable::EMTable); @@ -1723,9 +2057,9 @@ void ExtentMap::grabEMEntryTable(OPS op) if (!fPExtMapImpl || fPExtMapImpl->key() != (unsigned)fEMShminfo->tableShmkey) { - if (fExtentMap != NULL) + if (fExtentMap != nullptr) { - fExtentMap = NULL; + fExtentMap = nullptr; } if (fEMShminfo->allocdSize == 0) @@ -1736,17 +2070,22 @@ void ExtentMap::grabEMEntryTable(OPS op) emLocked = true; if (fEMShminfo->allocdSize == 0) + { growEMShmseg(); + } emLocked = false; // has to be done holding the write lock fMST.getTable_downgrade(MasterSegmentTable::EMTable); } else + { growEMShmseg(); + } } else { fPExtMapImpl = ExtentMapImpl::makeExtentMapImpl(fEMShminfo->tableShmkey, 0); + ASSERT(fPExtMapImpl); if (r_only) @@ -1754,7 +2093,7 @@ void ExtentMap::grabEMEntryTable(OPS op) fExtentMap = fPExtMapImpl->get(); - if (fExtentMap == NULL) + if (fExtentMap == nullptr) { log_errno("ExtentMap::grabEMEntryTable(): shmat"); throw runtime_error("ExtentMap::grabEMEntryTable(): shmat failed. Check the error log."); @@ -1762,7 +2101,9 @@ void ExtentMap::grabEMEntryTable(OPS op) } } else + { fExtentMap = fPExtMapImpl->get(); + } } /* always returns holding the FL lock */ @@ -1783,9 +2124,9 @@ void ExtentMap::grabFreeList(OPS op) if (!fPFreeListImpl || fPFreeListImpl->key() != (unsigned)fFLShminfo->tableShmkey) { - if (fFreeList != NULL) + if (fFreeList != nullptr) { - fFreeList = NULL; + fFreeList = nullptr; } if (fFLShminfo->allocdSize == 0) @@ -1815,7 +2156,7 @@ void ExtentMap::grabFreeList(OPS op) fFreeList = fPFreeListImpl->get(); - if (fFreeList == NULL) + if (fFreeList == nullptr) { log_errno("ExtentMap::grabFreeList(): shmat"); throw runtime_error("ExtentMap::grabFreeList(): shmat failed. Check the error log."); @@ -1834,10 +2175,66 @@ void ExtentMap::grabFreeList(OPS op) } } +void ExtentMap::grabEMIndex(OPS op) +{ + boost::mutex::scoped_lock lk(emIndexMutex); + + if (op == READ) + { + fEMIndexShminfo = fMST.getTable_read(MasterSegmentTable::EMIndex); + } + else + { + fEMIndexShminfo = fMST.getTable_write(MasterSegmentTable::EMIndex); + emIndexLocked = true; + } + + if (!fPExtMapIndexImpl_) + { + if (fEMIndexShminfo->allocdSize == 0) + { + if (op == READ) + { + fMST.getTable_upgrade(MasterSegmentTable::EMIndex); + emIndexLocked = true; + + // Checking race conditions + if (fEMIndexShminfo->allocdSize == 0) + growEMIndexShmseg(); + + emIndexLocked = false; + fMST.getTable_downgrade(MasterSegmentTable::EMIndex); + } + else + { + growEMIndexShmseg(); + } + } + else + { + // Sending down current Managed Shmem size. If EMIndexImpl instance size doesn't match + // fEMIndexShminfo->allocdSize makeExtentMapIndexImpl will remap managed shmem segment. + fPExtMapIndexImpl_ = + ExtentMapIndexImpl::makeExtentMapIndexImpl(getInitialEMIndexShmkey(), fEMIndexShminfo->allocdSize); + + if (r_only) + fPExtMapIndexImpl_->makeReadOnly(); + } + } + else if (fPExtMapIndexImpl_->getShmemImplSize() != (unsigned)fEMIndexShminfo->allocdSize) + { + fPExtMapIndexImpl_->refreshShm(); + fPExtMapIndexImpl_ = + ExtentMapIndexImpl::makeExtentMapIndexImpl(getInitialEMIndexShmkey(), fEMIndexShminfo->allocdSize); + } +} + void ExtentMap::releaseEMEntryTable(OPS op) { if (op == READ) + { fMST.releaseTable_read(MasterSegmentTable::EMTable); + } else { /* @@ -1863,32 +2260,50 @@ void ExtentMap::releaseFreeList(OPS op) } } +void ExtentMap::releaseEMIndex(OPS op) +{ + if (op == READ) + { + fMST.releaseTable_read(MasterSegmentTable::EMIndex); + } + else + { + emIndexLocked = false; + fMST.releaseTable_write(MasterSegmentTable::EMIndex); + } +} + key_t ExtentMap::chooseEMShmkey() { - int fixedKeys = 1; - key_t ret; - - if (fEMShminfo->tableShmkey + 1 == (key_t)(fShmKeys.KEYRANGE_EXTENTMAP_BASE + fShmKeys.KEYRANGE_SIZE - 1) || - (unsigned)fEMShminfo->tableShmkey < fShmKeys.KEYRANGE_EXTENTMAP_BASE) - ret = fShmKeys.KEYRANGE_EXTENTMAP_BASE + fixedKeys; - else - ret = fEMShminfo->tableShmkey + 1; - - return ret; + return chooseShmkey(fEMShminfo, fShmKeys.KEYRANGE_EXTENTMAP_BASE); } key_t ExtentMap::chooseFLShmkey() { - int fixedKeys = 1, ret; + return chooseShmkey(fFLShminfo, fShmKeys.KEYRANGE_EMFREELIST_BASE); +} - if (fFLShminfo->tableShmkey + 1 == - (key_t)(fShmKeys.KEYRANGE_EMFREELIST_BASE + fShmKeys.KEYRANGE_SIZE - 1) || - (unsigned)fFLShminfo->tableShmkey < fShmKeys.KEYRANGE_EMFREELIST_BASE) - ret = fShmKeys.KEYRANGE_EMFREELIST_BASE + fixedKeys; - else - ret = fFLShminfo->tableShmkey + 1; +// The key values is fixed b/c MCS doesn't need to increase a segment id number +key_t ExtentMap::chooseEMIndexShmkey() +{ + return chooseShmkey(fEMIndexShminfo, fShmKeys.KEYRANGE_EXTENTMAP_INDEX_BASE); +} - return ret; +key_t ExtentMap::getInitialEMIndexShmkey() const +{ + return fShmKeys.KEYRANGE_EXTENTMAP_INDEX_BASE + 1; +} + +key_t ExtentMap::chooseShmkey(const MSTEntry* masterTableEntry, const uint32_t keyRangeBase) const +{ + int fixedKeys = 1; + + if (masterTableEntry->tableShmkey + 1 == (key_t)(keyRangeBase + fShmKeys.KEYRANGE_SIZE - 1) || + (unsigned)masterTableEntry->tableShmkey < keyRangeBase) + { + return keyRangeBase + fixedKeys; + } + return masterTableEntry->tableShmkey + 1; } /* Must be called holding the EM write lock @@ -1927,6 +2342,31 @@ void ExtentMap::growEMShmseg(size_t nrows) fExtentMap = fPExtMapImpl->get(); } +void ExtentMap::growEMIndexShmseg(const size_t suggestedSize) +{ + static const constexpr int InitEMIndexSize_ = 16 * 1024 * 1024; + size_t allocSize = std::max(InitEMIndexSize_, fEMIndexShminfo->allocdSize); + key_t newshmkey = chooseEMIndexShmkey(); + key_t fixedManagedSegmentKey = getInitialEMIndexShmkey(); + + allocSize = std::max(allocSize, suggestedSize); + if (!fPExtMapIndexImpl_) + { + fPExtMapIndexImpl_ = + ExtentMapIndexImpl::makeExtentMapIndexImpl(fixedManagedSegmentKey, allocSize, r_only); + } + else + { + fPExtMapIndexImpl_->growIfNeeded(allocSize); + } + + if (r_only) + fPExtMapIndexImpl_->makeReadOnly(); + + fEMIndexShminfo->tableShmkey = newshmkey; + fEMIndexShminfo->allocdSize = allocSize; +} + /* Must be called holding the FL lock Returns with the new shmseg mapped */ void ExtentMap::growFLShmseg() @@ -1986,7 +2426,6 @@ int ExtentMap::lookup(LBID_t lbid, LBID_t& firstLbid, LBID_t& lastLbid) #ifdef BRM_DEBUG - // printEM(); if (lbid < 0) { log("ExtentMap::lookup(): lbid must be >= 0", logging::LOG_TYPE_DEBUG); @@ -1997,6 +2436,7 @@ int ExtentMap::lookup(LBID_t lbid, LBID_t& firstLbid, LBID_t& lastLbid) #endif grabEMEntryTable(READ); + grabEMIndex(READ); entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); for (i = 0; i < entries; i++) @@ -2009,12 +2449,13 @@ int ExtentMap::lookup(LBID_t lbid, LBID_t& firstLbid, LBID_t& lastLbid) { firstLbid = fExtentMap[i].range.start; lastLbid = lastBlock; + releaseEMIndex(READ); releaseEMEntryTable(READ); return 0; } } } - + releaseEMIndex(READ); releaseEMEntryTable(READ); return -1; } @@ -2063,6 +2504,7 @@ int ExtentMap::lookupLocal(LBID_t lbid, int& OID, uint16_t& dbRoot, uint32_t& pa } grabEMEntryTable(READ); + grabEMIndex(READ); entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); @@ -2083,12 +2525,13 @@ int ExtentMap::lookupLocal(LBID_t lbid, int& OID, uint16_t& dbRoot, uint32_t& pa offset = lbid - fExtentMap[i].range.start; fileBlockOffset = fExtentMap[i].blockOffset + offset; + releaseEMIndex(READ); releaseEMEntryTable(READ); return 0; } } } - + releaseEMIndex(READ); releaseEMEntryTable(READ); return -1; } @@ -2110,7 +2553,7 @@ int ExtentMap::lookupLocal(int OID, uint32_t partitionNum, uint16_t segmentNum, } #endif - int entries, i, offset; + int offset; if (OID < 0) { @@ -2119,25 +2562,31 @@ int ExtentMap::lookupLocal(int OID, uint32_t partitionNum, uint16_t segmentNum, } grabEMEntryTable(READ); + grabEMIndex(READ); - entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + DBRootVec dbRootVec(getAllDbRoots()); - for (i = 0; i < entries; i++) + for (auto dbRoot : dbRootVec) { - // TODO: Blockoffset logic. - if (fExtentMap[i].range.size != 0 && fExtentMap[i].fileID == OID && - fExtentMap[i].partitionNum == partitionNum && fExtentMap[i].segmentNum == segmentNum && - fExtentMap[i].blockOffset <= fileBlockOffset && - fileBlockOffset <= - (fExtentMap[i].blockOffset + (static_cast(fExtentMap[i].range.size) * 1024) - 1)) + auto emIdents = fPExtMapIndexImpl_->find(dbRoot, OID, partitionNum); + for (auto i : emIdents) { - offset = fileBlockOffset - fExtentMap[i].blockOffset; - LBID = fExtentMap[i].range.start + offset; - releaseEMEntryTable(READ); - return 0; + // TODO: Blockoffset logic. + if (fExtentMap[i].range.size != 0 && fExtentMap[i].segmentNum == segmentNum && + fExtentMap[i].blockOffset <= fileBlockOffset && + fileBlockOffset <= + (fExtentMap[i].blockOffset + (static_cast(fExtentMap[i].range.size) * 1024) - 1)) + { + offset = fileBlockOffset - fExtentMap[i].blockOffset; + LBID = fExtentMap[i].range.start + offset; + releaseEMIndex(READ); + releaseEMEntryTable(READ); + return 0; + } } } + releaseEMIndex(READ); releaseEMEntryTable(READ); return -1; } @@ -2168,6 +2617,7 @@ int ExtentMap::lookupLocal_DBroot(int OID, uint16_t dbroot, uint32_t partitionNu } grabEMEntryTable(READ); + grabEMIndex(READ); entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); @@ -2182,11 +2632,13 @@ int ExtentMap::lookupLocal_DBroot(int OID, uint16_t dbroot, uint32_t partitionNu { offset = fileBlockOffset - fExtentMap[i].blockOffset; LBID = fExtentMap[i].range.start + offset; + releaseEMIndex(READ); releaseEMEntryTable(READ); return 0; } } + releaseEMIndex(READ); releaseEMEntryTable(READ); return -1; } @@ -2214,7 +2666,6 @@ int ExtentMap::lookupLocalStartLbid(int OID, uint32_t partitionNum, uint16_t seg } #endif - int entries, i; if (OID < 0) { @@ -2225,22 +2676,29 @@ int ExtentMap::lookupLocalStartLbid(int OID, uint32_t partitionNum, uint16_t seg } grabEMEntryTable(READ); - entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + grabEMIndex(READ); - for (i = 0; i < entries; i++) + DBRootVec dbRootVec(getAllDbRoots()); + + for (auto dbRoot : dbRootVec) { - if (fExtentMap[i].range.size != 0 && fExtentMap[i].fileID == OID && - fExtentMap[i].partitionNum == partitionNum && fExtentMap[i].segmentNum == segmentNum && - fExtentMap[i].blockOffset <= fileBlockOffset && - fileBlockOffset <= - (fExtentMap[i].blockOffset + (static_cast(fExtentMap[i].range.size) * 1024) - 1)) + auto emIdents = fPExtMapIndexImpl_->find(dbRoot, OID, partitionNum); + for (auto i : emIdents) { - LBID = fExtentMap[i].range.start; - releaseEMEntryTable(READ); - return 0; + if (fExtentMap[i].range.size != 0 && fExtentMap[i].segmentNum == segmentNum && + fExtentMap[i].blockOffset <= fileBlockOffset && + fileBlockOffset <= + (fExtentMap[i].blockOffset + (static_cast(fExtentMap[i].range.size) * 1024) - 1)) + { + LBID = fExtentMap[i].range.start; + releaseEMIndex(READ); + releaseEMEntryTable(READ); + return 0; + } } } + releaseEMIndex(READ); releaseEMEntryTable(READ); return -1; @@ -2270,6 +2728,7 @@ void ExtentMap::createStripeColumnExtents(const vectorcurrentSize == fEMShminfo->allocdSize) + { growEMShmseg(); + } // size is the number of multiples of 1024 blocks. // ex: size=1 --> 1024 blocks @@ -2414,11 +2876,11 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co uint32_t& partitionNum, uint16_t& segmentNum, uint32_t& startBlockOffset) { - int emptyEMEntry = -1; - int lastExtentIndex = -1; - uint32_t highestOffset = 0; - uint32_t highestPartNum = 0; - uint16_t highestSegNum = 0; + EmptyEMEntry emptyEMEntry = -1; + LastExtentIndexT lastExtentIndex = -1; + HighestOffset highestOffset = 0; + PartitionNumberT highestPartNum = 0; + SegmentT highestSegNum = 0; const unsigned FILES_PER_COL_PART = getFilesPerColumnPartition(); const unsigned EXTENT_ROWS = getExtentRows(); const unsigned EXTENTS_PER_SEGFILE = getExtentsPerSegmentFile(); @@ -2426,9 +2888,9 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co // Variables that track list of segfiles in target (HWM) DBRoot & partition. // Map segment number to the highest fbo extent in each file - typedef tr1::unordered_map TargetDbRootSegsMap; - typedef TargetDbRootSegsMap::iterator TargetDbRootSegsMapIter; - typedef TargetDbRootSegsMap::const_iterator TargetDbRootSegsMapConstIter; + using TargetDbRootSegsMap = tr1::unordered_map; + using TargetDbRootSegsMapIter = TargetDbRootSegsMap::iterator; + using TargetDbRootSegsMapConstIter = TargetDbRootSegsMap::const_iterator; TargetDbRootSegsMap targetDbRootSegs; uint32_t highEmptySegNum = 0; // high seg num for user specified partition; @@ -2441,50 +2903,74 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co // 2. if DBRoot is empty, track highest seg num in user specified partition // 3. Find first unused extent map entry //-------------------------------------------------------------------------- - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - LBID_t startLBID = getLBIDsFromFreeList(size); + auto emIdents = fPExtMapIndexImpl_->find(dbRoot, OID); // Find the first empty Entry; and find last extent for this OID and dbRoot - for (int i = 0; i < emEntries; i++) + + for (auto i : emIdents) { if (fExtentMap[i].range.size != 0) { - if (fExtentMap[i].fileID == OID) + // 1. Find HWM extent in relevant DBRoot + if ((fExtentMap[i].partitionNum > highestPartNum) || + ((fExtentMap[i].partitionNum == highestPartNum) && (fExtentMap[i].blockOffset > highestOffset)) || + ((fExtentMap[i].partitionNum == highestPartNum) && (fExtentMap[i].blockOffset == highestOffset) && + (fExtentMap[i].segmentNum >= highestSegNum))) { - // 1. Find HWM extent in relevant DBRoot - if (fExtentMap[i].dbRoot == dbRoot) - { - if ((fExtentMap[i].partitionNum > highestPartNum) || - ((fExtentMap[i].partitionNum == highestPartNum) && - (fExtentMap[i].blockOffset > highestOffset)) || - ((fExtentMap[i].partitionNum == highestPartNum) && - (fExtentMap[i].blockOffset == highestOffset) && (fExtentMap[i].segmentNum >= highestSegNum))) - { - lastExtentIndex = i; - highestPartNum = fExtentMap[i].partitionNum; - highestSegNum = fExtentMap[i].segmentNum; - highestOffset = fExtentMap[i].blockOffset; - } - } - - // 2. for empty DBRoot track hi seg# in user specified part# - if ((lastExtentIndex == -1) && (fExtentMap[i].partitionNum == partitionNum)) - { - if ((fExtentMap[i].segmentNum > highEmptySegNum) || (!bHighEmptySegNumSet)) - { - highEmptySegNum = fExtentMap[i].segmentNum; - bHighEmptySegNumSet = true; - } - } - } // found extentmap entry for specified OID - } // found valid extentmap entry + lastExtentIndex = i; + highestPartNum = fExtentMap[i].partitionNum; + highestSegNum = fExtentMap[i].segmentNum; + highestOffset = fExtentMap[i].blockOffset; + } + } // found valid extentmap entry // 3. Find first available extent map entry that can be reused else if (emptyEMEntry < 0) emptyEMEntry = i; } // Loop through extent map entries + DBRootVec dbRootVec(getAllDbRoots()); + // 2. for empty DBRoot track hi seg# in user specified part# + if (lastExtentIndex == -1) + { + // loop over all extents that doesn't belong to the target dbroot + for (auto dbRootFromList : dbRootVec) + { + if (dbRootFromList == dbRoot) + continue; + + auto emIdentsLocal = fPExtMapIndexImpl_->find(dbRootFromList, OID, partitionNum); + for (auto i : emIdentsLocal) + { + if ((fExtentMap[i].range.size != 0) && + ((fExtentMap[i].segmentNum > highEmptySegNum) || (!bHighEmptySegNumSet))) + { + highEmptySegNum = fExtentMap[i].segmentNum; + bHighEmptySegNumSet = true; + } + + // Search for the first empty Entry + if (fExtentMap[i].range.size == 0) + { + emptyEMEntry = i; + break; + } + } + } + } + + size_t emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + // Search for the first empty Entry + for (size_t i = 0; emptyEMEntry < 0 && i < emEntries; ++i) + { + if (fExtentMap[i].range.size == 0) + { + emptyEMEntry = i; + break; + } + } + if (emptyEMEntry == -1) { ostringstream oss; @@ -2516,64 +3002,73 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co int partHighSeg = -1; // hi seg num for last partition int partHighSegNext = -1; // hi seg num for next partition + // Target dbroot has extents for the OID if (lastExtentIndex >= 0) { - uint32_t targetDbRootPart = fExtentMap[lastExtentIndex].partitionNum; - uint32_t targetDbRootPartNext = targetDbRootPart + 1; + PartitionNumberT targetDbRootPart = fExtentMap[lastExtentIndex].partitionNum; + PartitionNumberT targetDbRootPartNext = targetDbRootPart + 1; partHighSeg = fExtentMap[lastExtentIndex].segmentNum; targetDbRootSegs.insert(TargetDbRootSegsMap::value_type(fExtentMap[lastExtentIndex].segmentNum, fExtentMap[lastExtentIndex].blockOffset)); - for (int i = 0; i < emEntries; i++) + for (auto dbRootFromList : dbRootVec) { - if (fExtentMap[i].range.size != 0) + if (dbRootFromList == dbRoot) { - if (fExtentMap[i].fileID == OID) + auto emIdents = fPExtMapIndexImpl_->find(dbRootFromList, OID, targetDbRootPart); + for (auto i : emIdents) { - // 4. Track hi seg for hwm+1 partition - if (fExtentMap[i].partitionNum == targetDbRootPartNext) - { - if (fExtentMap[i].segmentNum > partHighSegNext) - { - partHighSegNext = fExtentMap[i].segmentNum; - } - } - // 5. Track hi seg for hwm partition - else if (fExtentMap[i].partitionNum == targetDbRootPart) + if (fExtentMap[i].segmentNum > partHighSeg) { - if (fExtentMap[i].segmentNum > partHighSeg) + partHighSeg = fExtentMap[i].segmentNum; + } + + // 6. Save list of seg files in target DBRoot/Partition, + // along with the highest fbo for each seg file + if (fExtentMap[i].status == EXTENTOUTOFSERVICE) + bSegsOutOfService = true; + + TargetDbRootSegsMapIter iter = targetDbRootSegs.find(fExtentMap[i].segmentNum); + + if (iter == targetDbRootSegs.end()) + { + targetDbRootSegs.insert( + TargetDbRootSegsMap::value_type(fExtentMap[i].segmentNum, fExtentMap[i].blockOffset)); + } + else + { + if (fExtentMap[i].blockOffset > iter->second) { - partHighSeg = fExtentMap[i].segmentNum; - } - - // 6. Save list of seg files in target DBRoot/Partition, - // along with the highest fbo for each seg file - if (fExtentMap[i].dbRoot == dbRoot) - { - if (fExtentMap[i].status == EXTENTOUTOFSERVICE) - bSegsOutOfService = true; - - TargetDbRootSegsMapIter iter = targetDbRootSegs.find(fExtentMap[i].segmentNum); - - if (iter == targetDbRootSegs.end()) - { - targetDbRootSegs.insert( - TargetDbRootSegsMap::value_type(fExtentMap[i].segmentNum, fExtentMap[i].blockOffset)); - } - else - { - if (fExtentMap[i].blockOffset > iter->second) - { - iter->second = fExtentMap[i].blockOffset; - } - } + iter->second = fExtentMap[i].blockOffset; } } - } // found extentmap entry for specified OID - } // found valid extentmap entry - } // loop through extent map entries - } // (lastExtentIndex >= 0) + } // loop over em idents + } // current dbroot == target dbroot + else + { + // 4. Track hi seg for hwm+1 partition + auto emIdentsNext = fPExtMapIndexImpl_->find(dbRootFromList, OID, targetDbRootPartNext); + for (auto i : emIdentsNext) + { + if (fExtentMap[i].segmentNum > partHighSegNext) + { + partHighSegNext = fExtentMap[i].segmentNum; + } + } + + // 5. Track hi seg for hwm partition + auto emIdents = fPExtMapIndexImpl_->find(dbRootFromList, OID, targetDbRootPart); + for (auto i : emIdents) + { + if (fExtentMap[i].segmentNum > partHighSeg) + { + partHighSeg = fExtentMap[i].segmentNum; + } + } + } // current dbroot != target dbroot + } // loop over dbroots + } // (lastExtentIndex >= 0) //-------------------------------------------------------------------------- // Third Step: Select partition and segment number for new extent @@ -2820,16 +3315,24 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co e->HWM = 0; e->status = EXTENTUNAVAILABLE; // mark extent as in process - // Partition, segment, and blockOffset 0 represents new table or column. - // When DDL creates a table, we can mark the first extent as VALID, since - // the table has no data. Marking as VALID enables cpimport to update - // the CP min/max for the first import. - // If DDL is adding a column to an existing table, setting to VALID won't - // hurt, because DDL resets to INVALID after the extent is created. - if ((e->partitionNum == 0) && (e->segmentNum == 0) && (e->blockOffset == 0)) - e->partition.cprange.isValid = CP_VALID; - else - e->partition.cprange.isValid = CP_INVALID; +#if 0 // XXX: sergueyz: I'll leave these under conditional flag for a while because it appears a huge change. + // Partition, segment, and blockOffset 0 represents new table or column. + // When DDL creates a table, we can mark the first extent as VALID, since + // the table has no data. Marking as VALID enables cpimport to update + // the CP min/max for the first import. + // If DDL is adding a column to an existing table, setting to VALID won't + // hurt, because DDL resets to INVALID after the extent is created. + // XXX: the comment above is out of date. bulk set of extents ranges + // works differently right now. + if ((e->partitionNum == 0) && + (e->segmentNum == 0) && + (e->blockOffset == 0)) + e->partition.cprange.isValid = CP_VALID; + else + e->partition.cprange.isValid = CP_INVALID; +#else + e->partition.cprange.isValid = CP_INVALID; +#endif partitionNum = e->partitionNum; segmentNum = e->segmentNum; @@ -2837,6 +3340,13 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co makeUndoRecord(fEMShminfo, sizeof(MSTEntry)); fEMShminfo->currentSize += sizeof(struct EMEntry); + auto resShmemHasGrownPair = fPExtMapIndexImpl_->insert(fExtentMap[emptyEMEntry], emptyEMEntry); + + if (resShmemHasGrownPair.second) + fEMIndexShminfo->allocdSize = fPExtMapIndexImpl_->getShmemSize(); + + if (!resShmemHasGrownPair.first) + logAndSetEMIndexReadOnly("_createColumnExtent_DBroot"); return startLBID; } @@ -2893,10 +3403,13 @@ void ExtentMap::createColumnExtentExactFile(int OID, uint32_t colWidth, uint16_t // extentRows should be multiple of blocksize (8192). const unsigned EXTENT_SIZE = (getExtentRows() * colWidth) / BLOCK_SIZE; grabEMEntryTable(WRITE); + grabEMIndex(WRITE); grabFreeList(WRITE); if (fEMShminfo->currentSize == fEMShminfo->allocdSize) + { growEMShmseg(); + } // size is the number of multiples of 1024 blocks. // ex: size=1 --> 1024 blocks @@ -2910,6 +3423,59 @@ void ExtentMap::createColumnExtentExactFile(int OID, uint32_t colWidth, uint16_t allocdsize = EXTENT_SIZE; } +LastIndEmptyIndEmptyInd ExtentMap::_createExtentCommonSearch(const OID_t OID, const DBRootT dbRoot, + const PartitionNumberT partitionNum, + const SegmentT segmentNum) +{ + EmptyEMEntry emptyEMEntry = -1; + LastExtentIndexT lastExtentIndex = -1; + HighestOffset highestOffset = 0; + + size_t emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + + auto emIdents = fPExtMapIndexImpl_->find(dbRoot, OID, partitionNum); + // DRRTUY we might need to use cache preload here. + // Search of the last extent idx and the highest offset + for (auto i : emIdents) + { + if (fExtentMap[i].range.size != 0) + { + if ((fExtentMap[i].segmentNum == segmentNum) && (fExtentMap[i].blockOffset >= highestOffset)) + { + lastExtentIndex = i; + highestOffset = fExtentMap[i].blockOffset; + } + } + // Search for the first empty Entry + else if (emptyEMEntry < 0) + emptyEMEntry = i; + } + + // Search for the first empty Entry + // DRRTUY We might need to support empty EM ids vector + for (size_t i = 0; emptyEMEntry < 0 && i < emEntries; ++i) + { + if (fExtentMap[i].range.size == 0) + { + emptyEMEntry = i; + break; + } + } + return {lastExtentIndex, emptyEMEntry}; +} + +void ExtentMap::logAndSetEMIndexReadOnly(const std::string& funcName) +{ + fPExtMapIndexImpl_->makeReadOnly(); + ostringstream os; + os << "ExtentMap::" << funcName << ": " + << "Can not update EM Index. EM Index shmem segment is set to" + << " readonly. Please restart Columnstore."; + log(os.str(), logging::LOG_TYPE_CRITICAL); + + throw logic_error(os.str()); +} + //------------------------------------------------------------------------------ // Creates an extent for the exact segment file specified by the requested // OID, DBRoot, partition, and segment. This is the internal implementation @@ -2933,32 +3499,9 @@ LBID_t ExtentMap::_createColumnExtentExactFile(uint32_t size, int OID, uint32_t execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t& startBlockOffset) { - int emptyEMEntry = -1; - int lastExtentIndex = -1; - uint32_t highestOffset = 0; - - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - LBID_t startLBID = getLBIDsFromFreeList(size); - - // Find the first empty Entry; and find the last extent for this - // combination of OID, partition, and segment. - for (int i = 0; i < emEntries; i++) - { - if (fExtentMap[i].range.size != 0) - { - if (fExtentMap[i].fileID == OID) - { - if ((fExtentMap[i].dbRoot == dbRoot) && (fExtentMap[i].partitionNum == partitionNum) && - (fExtentMap[i].segmentNum == segmentNum) && (fExtentMap[i].blockOffset >= highestOffset)) - { - lastExtentIndex = i; - highestOffset = fExtentMap[i].blockOffset; - } - } - } - else if (emptyEMEntry < 0) - emptyEMEntry = i; - } // Loop through extent map entries + auto lastIndEmptyIndEmptyInd = _createExtentCommonSearch(OID, dbRoot, partitionNum, segmentNum); + LastExtentIndexT lastExtentIndex = lastIndEmptyIndEmptyInd.first; + EmptyEMEntry emptyEMEntry = lastIndEmptyIndEmptyInd.second; if (emptyEMEntry == -1) { @@ -2973,6 +3516,7 @@ LBID_t ExtentMap::_createColumnExtentExactFile(uint32_t size, int OID, uint32_t makeUndoRecord(&fExtentMap[emptyEMEntry], sizeof(EMEntry)); EMEntry* e = &fExtentMap[emptyEMEntry]; + LBID_t startLBID = getLBIDsFromFreeList(size); e->range.start = startLBID; e->range.size = size; e->fileID = OID; @@ -3029,21 +3573,34 @@ LBID_t ExtentMap::_createColumnExtentExactFile(uint32_t size, int OID, uint32_t e->HWM = 0; } - // Partition, segment, and blockOffset 0 represents new table or column. - // When DDL creates a table, we can mark the first extent as VALID, since - // the table has no data. Marking as VALID enables cpimport to update - // the CP min/max for the first import. - // If DDL is adding a column to an existing table, setting to VALID won't - // hurt, because DDL resets to INVALID after the extent is created. - if ((e->partitionNum == 0) && (e->segmentNum == 0) && (e->blockOffset == 0)) - e->partition.cprange.isValid = CP_VALID; - else - e->partition.cprange.isValid = CP_INVALID; +#if 0 // XXX: sergueyz: I'll leave these under conditional flag for a while because it appears a huge change. + // Partition, segment, and blockOffset 0 represents new table or column. + // When DDL creates a table, we can mark the first extent as VALID, since + // the table has no data. Marking as VALID enables cpimport to update + // the CP min/max for the first import. + // If DDL is adding a column to an existing table, setting to VALID won't + // hurt, because DDL resets to INVALID after the extent is created. + if ((e->partitionNum == 0) && + (e->segmentNum == 0) && + (e->blockOffset == 0)) + e->partition.cprange.isValid = CP_VALID; + else + e->partition.cprange.isValid = CP_INVALID; +#else + e->partition.cprange.isValid = CP_INVALID; +#endif startBlockOffset = e->blockOffset; makeUndoRecord(fEMShminfo, sizeof(MSTEntry)); fEMShminfo->currentSize += sizeof(struct EMEntry); + auto resShmemHasGrownPair = fPExtMapIndexImpl_->insert(fExtentMap[emptyEMEntry], emptyEMEntry); + + if (resShmemHasGrownPair.second) + fEMIndexShminfo->allocdSize = fPExtMapIndexImpl_->getShmemSize(); + + if (!resShmemHasGrownPair.first) + logAndSetEMIndexReadOnly("_createColumnExtentExactFile"); return startLBID; } @@ -3094,10 +3651,13 @@ void ExtentMap::createDictStoreExtent(int OID, uint16_t dbRoot, uint32_t partiti const unsigned EXTENT_SIZE = (getExtentRows() * DICT_COL_WIDTH) / BLOCK_SIZE; grabEMEntryTable(WRITE); + grabEMIndex(WRITE); grabFreeList(WRITE); if (fEMShminfo->currentSize == fEMShminfo->allocdSize) + { growEMShmseg(); + } // size is the number of multiples of 1024 blocks. // ex: size=1 --> 1024 blocks @@ -3127,30 +3687,9 @@ void ExtentMap::createDictStoreExtent(int OID, uint16_t dbRoot, uint32_t partiti LBID_t ExtentMap::_createDictStoreExtent(uint32_t size, int OID, uint16_t dbRoot, uint32_t partitionNum, uint16_t segmentNum) { - int emptyEMEntry = -1; - int lastExtentIndex = -1; - uint32_t highestOffset = 0; - - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - - LBID_t startLBID = getLBIDsFromFreeList(size); - - // Find the first empty Entry; and find the last extent for this - // combination of OID, partition, and segment. - for (int i = 0; i < emEntries; i++) - { - if (fExtentMap[i].range.size != 0) - { - if ((fExtentMap[i].fileID == OID) && (fExtentMap[i].partitionNum == partitionNum) && - (fExtentMap[i].segmentNum == segmentNum) && (fExtentMap[i].blockOffset >= highestOffset)) - { - lastExtentIndex = i; - highestOffset = fExtentMap[i].blockOffset; - } - } - else if (emptyEMEntry < 0) - emptyEMEntry = i; - } // Loop through extent map entries + auto lastIndEmptyIndEmptyInd = _createExtentCommonSearch(OID, dbRoot, partitionNum, segmentNum); + LastExtentIndexT lastExtentIndex = lastIndEmptyIndEmptyInd.first; + EmptyEMEntry emptyEMEntry = lastIndEmptyIndEmptyInd.second; if (emptyEMEntry == -1) { @@ -3165,6 +3704,7 @@ LBID_t ExtentMap::_createDictStoreExtent(uint32_t size, int OID, uint16_t dbRoot makeUndoRecord(&fExtentMap[emptyEMEntry], sizeof(EMEntry)); EMEntry* e = &fExtentMap[emptyEMEntry]; + LBID_t startLBID = getLBIDsFromFreeList(size); e->range.start = startLBID; e->range.size = size; e->fileID = OID; @@ -3201,6 +3741,13 @@ LBID_t ExtentMap::_createDictStoreExtent(uint32_t size, int OID, uint16_t dbRoot makeUndoRecord(fEMShminfo, sizeof(MSTEntry)); fEMShminfo->currentSize += sizeof(struct EMEntry); + auto resShmemHasGrownPair = fPExtMapIndexImpl_->insert(fExtentMap[emptyEMEntry], emptyEMEntry); + + if (resShmemHasGrownPair.second) + fEMIndexShminfo->allocdSize = fPExtMapIndexImpl_->getShmemSize(); + + if (!resShmemHasGrownPair.first) + logAndSetEMIndexReadOnly("_createDictStoreExtent"); return startLBID; } @@ -3364,6 +3911,7 @@ void ExtentMap::rollbackColumnExtents_DBroot(int oid, bool bDeleteAll, uint16_t uint32_t fboLoPreviousStripe = 0; grabEMEntryTable(WRITE); + grabEMIndex(WRITE); grabFreeList(WRITE); int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); @@ -3562,6 +4110,7 @@ void ExtentMap::rollbackDictStoreExtents_DBroot(int oid, uint16_t dbRoot, uint32 tr1::unordered_map >::const_iterator segToHwmMapIter; grabEMEntryTable(WRITE); + grabEMIndex(WRITE); grabFreeList(WRITE); int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); @@ -3688,6 +4237,7 @@ void ExtentMap::deleteEmptyColExtents(const ExtentsInfoMap_t& extentsInfo) #endif grabEMEntryTable(WRITE); + grabEMIndex(WRITE); grabFreeList(WRITE); uint32_t fboLo = 0; @@ -3823,6 +4373,7 @@ void ExtentMap::deleteEmptyDictStoreExtents(const ExtentsInfoMap_t& extentsInfo) #endif grabEMEntryTable(WRITE); + grabEMIndex(WRITE); grabFreeList(WRITE); ExtentsInfoMap_t::const_iterator it; @@ -3953,8 +4504,15 @@ void ExtentMap::deleteOID(int OID) #endif grabEMEntryTable(WRITE); + grabEMIndex(WRITE); grabFreeList(WRITE); + // Clean up the index and tell deleteExtent to skip the clean-up. + DBRootVec dbRootVec(getAllDbRoots()); + for (auto dbRoot : dbRootVec) + fPExtMapIndexImpl_->deleteOID(dbRoot, OID); + const bool clearEMIndex = false; + int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); for (int emIndex = 0; emIndex < emEntries; emIndex++) @@ -3962,8 +4520,7 @@ void ExtentMap::deleteOID(int OID) if (fExtentMap[emIndex].range.size > 0 && fExtentMap[emIndex].fileID == OID) { OIDExists = true; - - deleteExtent(emIndex); + deleteExtent(emIndex, clearEMIndex); } } @@ -3991,10 +4548,20 @@ void ExtentMap::deleteOIDs(const OidsMap_t& OIDs) #endif grabEMEntryTable(WRITE); + grabEMIndex(WRITE); grabFreeList(WRITE); + OidsMap_t::const_iterator it; int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + const bool clearEMIndex = false; + DBRootVec dbRootVec(getAllDbRoots()); + for (auto dbRoot : dbRootVec) + { + for (auto& oidOidPair : OIDs) + fPExtMapIndexImpl_->deleteOID(dbRoot, oidOidPair.first); + } + for (int emIndex = 0; emIndex < emEntries; emIndex++) { if (fExtentMap[emIndex].range.size > 0) @@ -4002,7 +4569,7 @@ void ExtentMap::deleteOIDs(const OidsMap_t& OIDs) it = OIDs.find(fExtentMap[emIndex].fileID); if (it != OIDs.end()) - deleteExtent(emIndex); + deleteExtent(emIndex, clearEMIndex); } } } @@ -4011,7 +4578,7 @@ void ExtentMap::deleteOIDs(const OidsMap_t& OIDs) // Delete the specified extent from the extentmap and return to the free list. // emIndex - the index (from the extent map) of the extent to be deleted //------------------------------------------------------------------------------ -void ExtentMap::deleteExtent(int emIndex) +void ExtentMap::deleteExtent(const int emIndex, const bool clearEMIndex) { int flIndex, freeFLIndex, flEntries, preceedingExtent, succeedingExtent; LBID_t flBlockEnd, emBlockEnd; @@ -4134,6 +4701,8 @@ void ExtentMap::deleteExtent(int emIndex) // invalidate the entry in the Extent Map makeUndoRecord(&fExtentMap[emIndex], sizeof(EMEntry)); fExtentMap[emIndex].range.size = 0; + if (clearEMIndex) + fPExtMapIndexImpl_->deleteEMEntry(fExtentMap[emIndex], emIndex); makeUndoRecord(&fEMShminfo, sizeof(MSTEntry)); fEMShminfo->currentSize -= sizeof(struct EMEntry); } @@ -4183,6 +4752,7 @@ HWM_t ExtentMap::getLastHWM_DBroot(int OID, uint16_t dbRoot, uint32_t& partition } grabEMEntryTable(READ); + grabEMIndex(READ); // Searching the array in reverse order should be faster since the last // extent is usually at the bottom. We still have to search the entire @@ -4217,6 +4787,7 @@ HWM_t ExtentMap::getLastHWM_DBroot(int OID, uint16_t dbRoot, uint32_t& partition bFound = true; } + releaseEMIndex(READ); releaseEMEntryTable(READ); return hwm; @@ -4252,14 +4823,14 @@ void ExtentMap::getDbRootHWMInfo(int OID, uint16_t pmNumber, EmDbRootHWMInfo_v& // Determine List of DBRoots for specified PM, and construct map of // EmDbRootHWMInfo objects. tr1::unordered_map emDbRootMap; - vector dbRootList; - getPmDbRoots(pmNumber, dbRootList); + vector dbRootVec; + getPmDbRoots(pmNumber, dbRootVec); - if (dbRootList.size() > 0) + if (dbRootVec.size() > 0) { - for (unsigned int iroot = 0; iroot < dbRootList.size(); iroot++) + for (unsigned int iroot = 0; iroot < dbRootVec.size(); iroot++) { - uint16_t rootID = dbRootList[iroot]; + uint16_t rootID = dbRootVec[iroot]; EmDbRootHWMInfo emDbRootInfo(rootID); emDbRootMap[rootID] = emDbRootInfo; } @@ -4275,47 +4846,52 @@ void ExtentMap::getDbRootHWMInfo(int OID, uint16_t pmNumber, EmDbRootHWMInfo_v& } grabEMEntryTable(READ); + grabEMIndex(READ); tr1::unordered_map::iterator emIter; // Searching the array in reverse order should be faster since the last // extent is usually at the bottom. We still have to search the entire // array (just in case), but the number of operations per loop iteration // will be less. - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - - for (int i = emEntries - 1; i >= 0; i--) + for (auto dbRoot : dbRootVec) { - if ((fExtentMap[i].range.size != 0) && (fExtentMap[i].fileID == OID)) + auto emIdents = fPExtMapIndexImpl_->find(dbRoot, OID); + for (auto i : emIdents) { - // Include this extent in the search, only if the extent's - // DBRoot falls in the list of DBRoots for this PM. - emIter = emDbRootMap.find(fExtentMap[i].dbRoot); - - if (emIter == emDbRootMap.end()) - continue; - - EmDbRootHWMInfo& emDbRoot = emIter->second; - - if ((fExtentMap[i].status != EXTENTOUTOFSERVICE) && (fExtentMap[i].HWM != 0)) - emDbRoot.totalBlocks += (fExtentMap[i].HWM + 1); - - if ((fExtentMap[i].partitionNum > emDbRoot.partitionNum) || - ((fExtentMap[i].partitionNum == emDbRoot.partitionNum) && - (fExtentMap[i].blockOffset > emDbRoot.fbo)) || - ((fExtentMap[i].partitionNum == emDbRoot.partitionNum) && - (fExtentMap[i].blockOffset == emDbRoot.fbo) && (fExtentMap[i].segmentNum >= emDbRoot.segmentNum))) + if ((fExtentMap[i].range.size != 0) && (fExtentMap[i].fileID == OID)) { - emDbRoot.fbo = fExtentMap[i].blockOffset; - emDbRoot.partitionNum = fExtentMap[i].partitionNum; - emDbRoot.segmentNum = fExtentMap[i].segmentNum; - emDbRoot.localHWM = fExtentMap[i].HWM; - emDbRoot.startLbid = fExtentMap[i].range.start; - emDbRoot.status = fExtentMap[i].status; - emDbRoot.hwmExtentIndex = i; + // Include this extent in the search, only if the extent's + // DBRoot falls in the list of DBRoots for this PM. + emIter = emDbRootMap.find(fExtentMap[i].dbRoot); + + if (emIter == emDbRootMap.end()) + continue; + + EmDbRootHWMInfo& emDbRoot = emIter->second; + + if ((fExtentMap[i].status != EXTENTOUTOFSERVICE) && (fExtentMap[i].HWM != 0)) + emDbRoot.totalBlocks += (fExtentMap[i].HWM + 1); + + if ((fExtentMap[i].partitionNum > emDbRoot.partitionNum) || + ((fExtentMap[i].partitionNum == emDbRoot.partitionNum) && + (fExtentMap[i].blockOffset > emDbRoot.fbo)) || + ((fExtentMap[i].partitionNum == emDbRoot.partitionNum) && + (fExtentMap[i].blockOffset == emDbRoot.fbo) && + (fExtentMap[i].segmentNum >= emDbRoot.segmentNum))) + { + emDbRoot.fbo = fExtentMap[i].blockOffset; + emDbRoot.partitionNum = fExtentMap[i].partitionNum; + emDbRoot.segmentNum = fExtentMap[i].segmentNum; + emDbRoot.localHWM = fExtentMap[i].HWM; + emDbRoot.startLbid = fExtentMap[i].range.start; + emDbRoot.status = fExtentMap[i].status; + emDbRoot.hwmExtentIndex = i; + } } } } + releaseEMIndex(READ); releaseEMEntryTable(READ); for (tr1::unordered_map::iterator iter = emDbRootMap.begin(); @@ -4395,6 +4971,7 @@ void ExtentMap::getExtentState(int OID, uint32_t partitionNum, uint16_t segmentN } grabEMEntryTable(READ); + grabEMIndex(READ); emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); @@ -4409,6 +4986,7 @@ void ExtentMap::getExtentState(int OID, uint32_t partitionNum, uint16_t segmentN } } + releaseEMIndex(READ); releaseEMEntryTable(READ); } @@ -4441,7 +5019,6 @@ HWM_t ExtentMap::getLocalHWM(int OID, uint32_t partitionNum, uint16_t segmentNum #endif - int i, emEntries; HWM_t ret = 0; bool OIDPartSegExists = false; @@ -4454,26 +5031,31 @@ HWM_t ExtentMap::getLocalHWM(int OID, uint32_t partitionNum, uint16_t segmentNum } grabEMEntryTable(READ); + grabEMIndex(READ); - emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - - for (i = 0; i < emEntries; i++) + DBRootVec dbRootVec(getAllDbRoots()); + for (auto dbRoot : dbRootVec) { - if ((fExtentMap[i].range.size != 0) && (fExtentMap[i].fileID == OID) && - (fExtentMap[i].partitionNum == partitionNum) && (fExtentMap[i].segmentNum == segmentNum)) + auto emIdents = fPExtMapIndexImpl_->find(dbRoot, OID, partitionNum); + for (auto i : emIdents) { - OIDPartSegExists = true; - status = fExtentMap[i].status; - - if (fExtentMap[i].HWM != 0) + if ((fExtentMap[i].range.size != 0) && (fExtentMap[i].segmentNum == segmentNum)) { - ret = fExtentMap[i].HWM; - releaseEMEntryTable(READ); - return ret; + OIDPartSegExists = true; + status = fExtentMap[i].status; + + if (fExtentMap[i].HWM != 0) + { + ret = fExtentMap[i].HWM; + releaseEMIndex(READ); + releaseEMEntryTable(READ); + return ret; + } } } } + releaseEMIndex(READ); releaseEMEntryTable(READ); if (OIDPartSegExists) @@ -4520,31 +5102,37 @@ void ExtentMap::setLocalHWM(int OID, uint32_t partitionNum, uint16_t segmentNum, #endif - int lastExtentIndex = -1; + LastExtentIndexT lastExtentIndex = -1; int oldHWMExtentIndex = -1; - uint32_t highestOffset = 0; + HighestOffset highestOffset = 0; if (uselock) - grabEMEntryTable(WRITE); - - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - - for (int i = 0; i < emEntries; i++) { - if ((fExtentMap[i].range.size != 0) && (fExtentMap[i].fileID == OID) && - (fExtentMap[i].partitionNum == partitionNum) && (fExtentMap[i].segmentNum == segmentNum)) - { - // Find current HWM extent - if (fExtentMap[i].blockOffset >= highestOffset) - { - highestOffset = fExtentMap[i].blockOffset; - lastExtentIndex = i; - } + grabEMEntryTable(WRITE); + grabEMIndex(WRITE); + } - // Find previous HWM extent - if (fExtentMap[i].HWM != 0) + DBRootVec dbRootVec(getAllDbRoots()); + + for (auto dbRoot : dbRootVec) + { + auto emIdents = fPExtMapIndexImpl_->find(dbRoot, OID, partitionNum); + for (auto i : emIdents) + { + if ((fExtentMap[i].range.size != 0) && (fExtentMap[i].segmentNum == segmentNum)) { - oldHWMExtentIndex = i; + // Find current HWM extent + if (fExtentMap[i].blockOffset >= highestOffset) + { + highestOffset = fExtentMap[i].blockOffset; + lastExtentIndex = i; + } + + // Find previous HWM extent + if (fExtentMap[i].HWM != 0) + { + oldHWMExtentIndex = i; + } } } } @@ -4619,6 +5207,7 @@ void ExtentMap::setLocalHWM(int OID, uint32_t partitionNum, uint16_t segmentNum, void ExtentMap::bulkSetHWM(const vector& v, bool firstNode) { grabEMEntryTable(WRITE); + grabEMIndex(WRITE); for (uint32_t i = 0; i < v.size(); i++) setLocalHWM(v[i].oid, v[i].partNum, v[i].segNum, v[i].hwm, firstNode, false); @@ -4653,6 +5242,7 @@ void ExtentMap::bulkUpdateDBRoot(const vector& args) sArgs.insert(args[i]); grabEMEntryTable(WRITE); + grabEMIndex(WRITE); emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); @@ -4692,10 +5282,10 @@ void ExtentMap::getExtents(int OID, vector& entries, bool sorted } grabEMEntryTable(READ); + grabEMIndex(READ); emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); // Pre-expand entries to stop lots of small allocs entries.reserve(emEntries); - if (incOutOfService) { for (i = 0; i < emEntries; i++) @@ -4710,6 +5300,7 @@ void ExtentMap::getExtents(int OID, vector& entries, bool sorted entries.push_back(fExtentMap[i]); } + releaseEMIndex(READ); releaseEMEntryTable(READ); if (sorted) @@ -4767,12 +5358,14 @@ void ExtentMap::getExtents_dbroot(int OID, vector& entries, cons } grabEMEntryTable(READ); + grabEMIndex(READ); emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); for (i = 0; i < emEntries; i++) if ((fExtentMap[i].fileID == OID) && (fExtentMap[i].range.size != 0) && (fExtentMap[i].dbRoot == dbroot)) entries.push_back(fExtentMap[i]); + releaseEMIndex(READ); releaseEMEntryTable(READ); } @@ -4794,6 +5387,7 @@ void ExtentMap::getExtentCount_dbroot(int OID, uint16_t dbroot, bool incOutOfSer } grabEMEntryTable(READ); + grabEMIndex(READ); emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); numExtents = 0; @@ -4817,6 +5411,7 @@ void ExtentMap::getExtentCount_dbroot(int OID, uint16_t dbroot, bool incOutOfSer } } + releaseEMIndex(READ); releaseEMEntryTable(READ); } @@ -4842,6 +5437,7 @@ void ExtentMap::getSysCatDBRoot(OID_t oid, uint16_t& dbRoot) bool bFound = false; grabEMEntryTable(READ); + grabEMIndex(READ); int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); for (int i = 0; i < emEntries; i++) @@ -4854,6 +5450,7 @@ void ExtentMap::getSysCatDBRoot(OID_t oid, uint16_t& dbRoot) } } + releaseEMIndex(READ); releaseEMEntryTable(READ); if (!bFound) @@ -4904,6 +5501,7 @@ void ExtentMap::deletePartition(const set& oids, const set foundPartitions; int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); @@ -5005,6 +5603,7 @@ void ExtentMap::markPartitionForDeletion(const set& oids, const setallocdSize / sizeof(struct EMEntry); set foundPartitions; vector extents; @@ -5112,6 +5711,7 @@ void ExtentMap::markAllPartitionForDeletion(const set& oids) set::const_iterator it; grabEMEntryTable(WRITE); + grabEMIndex(WRITE); int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); for (int i = 0; i < emEntries; i++) @@ -5165,6 +5765,7 @@ void ExtentMap::restorePartition(const set& oids, const set::const_iterator it; grabEMEntryTable(WRITE); + grabEMIndex(WRITE); int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); vector extents; @@ -5257,6 +5858,7 @@ void ExtentMap::getOutOfServicePartitions(OID_t oid, set& part } grabEMEntryTable(READ); + grabEMIndex(READ); int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); for (int i = 0; i < emEntries; i++) @@ -5270,6 +5872,7 @@ void ExtentMap::getOutOfServicePartitions(OID_t oid, set& part } } + releaseEMIndex(READ); releaseEMEntryTable(READ); } @@ -5291,11 +5894,13 @@ void ExtentMap::deleteDBRoot(uint16_t dbroot) #endif grabEMEntryTable(WRITE); + grabEMIndex(WRITE); grabFreeList(WRITE); for (unsigned i = 0; i < fEMShminfo->allocdSize / sizeof(struct EMEntry); i++) if (fExtentMap[i].range.size != 0 && fExtentMap[i].dbRoot == dbroot) deleteExtent(i); + fPExtMapIndexImpl_->deleteDbRoot(dbroot); } //------------------------------------------------------------------------------ @@ -5318,6 +5923,7 @@ bool ExtentMap::isDBRootEmpty(uint16_t dbroot) bool bEmpty = true; int i, emEntries; grabEMEntryTable(READ); + grabEMIndex(READ); emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); if (fEMShminfo->currentSize == 0) @@ -5334,6 +5940,7 @@ bool ExtentMap::isDBRootEmpty(uint16_t dbroot) } } + releaseEMIndex(READ); releaseEMEntryTable(READ); return bEmpty; @@ -5393,6 +6000,7 @@ void ExtentMap::lookup(OID_t OID, LBIDRange_v& ranges) } grabEMEntryTable(READ); + grabEMIndex(READ); emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); for (i = 0; i < emEntries; i++) @@ -5404,6 +6012,7 @@ void ExtentMap::lookup(OID_t OID, LBIDRange_v& ranges) ranges.push_back(tmp); } + releaseEMIndex(READ); releaseEMEntryTable(READ); } @@ -5435,6 +6044,7 @@ int ExtentMap::checkConsistency() uint32_t usedEntries; grabEMEntryTable(READ); + grabEMIndex(READ); try { @@ -5442,6 +6052,7 @@ int ExtentMap::checkConsistency() } catch (...) { + releaseEMIndex(READ); releaseEMEntryTable(READ); throw; } @@ -5655,6 +6266,7 @@ int ExtentMap::checkConsistency() cout << "test 5a passed\n"; releaseFreeList(READ); + releaseEMIndex(READ); releaseEMEntryTable(READ); return 0; } @@ -5693,6 +6305,9 @@ void ExtentMap::finishChanges() if (flLocked) releaseFreeList(WRITE); + if (emIndexLocked) + releaseEMIndex(WRITE); + if (emLocked) releaseEMEntryTable(WRITE); } @@ -5707,6 +6322,11 @@ const bool* ExtentMap::getEMLockStatus() return &emLocked; } +const bool* ExtentMap::getEMIndexLockStatus() +{ + return &emIndexLocked; +} + //------------------------------------------------------------------------------ // Reload Config cache if config file time stamp has changed //------------------------------------------------------------------------------ @@ -5854,19 +6474,36 @@ unsigned ExtentMap::getDbRootCount() // Get list of DBRoots that map to the specified PM. DBRoot list is cached // internally in fPmDbRootMap after getting from Columnstore.xml via OAM. //------------------------------------------------------------------------------ -void ExtentMap::getPmDbRoots(int pm, vector& dbRootList) +void ExtentMap::getPmDbRoots(int pm, vector& dbRootVec) { oam::OamCache* oamcache = oam::OamCache::makeOamCache(); oam::OamCache::PMDbrootsMap_t pmDbroots = oamcache->getPMToDbrootsMap(); - dbRootList.clear(); - dbRootList = (*pmDbroots)[pm]; + dbRootVec.clear(); + dbRootVec = (*pmDbroots)[pm]; +} + +DBRootVec ExtentMap::getAllDbRoots() +{ + DBRootVec dbRootResultVec; + oam::OamCache* oamcache = oam::OamCache::makeOamCache(); + // NB The routine uses int for dbroot id that contradicts with the type used here, namely uint16_t + oam::OamCache::PMDbrootsMap_t pmDbroots = oamcache->getPMToDbrootsMap(); + auto& pmDbrootsRef = *pmDbroots; + + for (auto& pmDBRootPair : pmDbrootsRef) + { + for (auto dbRootId : pmDBRootPair.second) + dbRootResultVec.push_back(dbRootId); + } + return dbRootResultVec; } vector ExtentMap::getFreeListEntries() { vector v; grabEMEntryTable(READ); + grabEMIndex(READ); grabFreeList(READ); int allocdSize = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); @@ -5875,6 +6512,7 @@ vector ExtentMap::getFreeListEntries() v.push_back(fFreeList[i]); releaseFreeList(READ); + releaseEMIndex(READ); releaseEMEntryTable(READ); return v; } @@ -5882,6 +6520,7 @@ vector ExtentMap::getFreeListEntries() void ExtentMap::dumpTo(ostream& os) { grabEMEntryTable(READ); + grabEMIndex(READ); unsigned emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); for (unsigned i = 0; i < emEntries; i++) @@ -5897,54 +6536,25 @@ void ExtentMap::dumpTo(ostream& os) } } + releaseEMIndex(READ); releaseEMEntryTable(READ); } -/*int ExtentMap::physicalPartitionNum(const set& oids, - const set& partitionNums, - vector& partitionInfos) +size_t ExtentMap::EMIndexShmemSize() { -#ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITENOW("physicalPartitionNum"); - ostringstream oss; - set::const_iterator partIt; - oss << "partitionNums: " - for (partIt=partitionNums.begin(); it!=partitionNums.end(); ++it) - oss << (*it) << " "; - oss << endl; - TRACER_WRITEDIRECT(oss.str()); - } -#endif - - set::const_iterator it; - grabEMEntryTable(READ); - - int emEntries = fEMShminfo->allocdSize/sizeof(struct EMEntry); - PartitionInfo partInfo; - vector extents; - set foundPartitions; - for (int i = 0; i < emEntries; i++) - { - if ((fExtentMap[i].range.size != 0 ) && - partitionNums.find(logicalPartitionNum(fExtentMap[i])) != partitionNums.end()) - { - it = oids.find( fExtentMap[i].fileID ); - if (it != oids.end()) - { - partInfo.oid = fExtentMap[i].fileID; - partInfo.lp.dbroot = fExtentMap[i].dbRoot; - partInfo.lp.pp = fExtentMap[i].partitionNum; - partInfo.lp.seg = fExtentMap[i].segmentNum; - partitionInfos.push_back(partInfo); - } - } - } - releaseEMEntryTable(READ); - return 0; + grabEMIndex(READ); + size_t EMIndexShmemSize = fPExtMapIndexImpl_->getShmemSize(); + releaseEMIndex(READ); + return EMIndexShmemSize; +} + +size_t ExtentMap::EMIndexShmemFree() +{ + grabEMIndex(READ); + size_t EMIndexShmemFree = fPExtMapIndexImpl_->getShmemFree(); + releaseEMIndex(READ); + return EMIndexShmemFree; } -*/ template int ExtentMap::getMaxMin(const LBID_t lbidRange, int128_t& max, int128_t& min, int32_t& seqNum); @@ -5952,5 +6562,4 @@ template int ExtentMap::getMaxMin(const LBID_t lbidRange, int128_t& ma template int ExtentMap::getMaxMin(const LBID_t lbidRange, int64_t& max, int64_t& min, int32_t& seqNum); -} // namespace BRM -// vim:ts=4 sw=4: +} // namespace BRM \ No newline at end of file diff --git a/versioning/BRM/extentmap.h b/versioning/BRM/extentmap.h index de8cf4df0..8452d822e 100644 --- a/versioning/BRM/extentmap.h +++ b/versioning/BRM/extentmap.h @@ -1,4 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2016-2022 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -29,15 +30,19 @@ #include #include #include -#ifdef _MSC_VER #include -#else #include -#endif +#include + //#define NDEBUG #include -#include +#include //boost::hash +#include +#include +#include #include +#include +#include #include "shmkeys.h" #include "brmtypes.h" @@ -63,6 +68,8 @@ #define EXPORT #endif +namespace bi = boost::interprocess; + namespace oam { typedef std::vector DBRootConfigList; @@ -75,6 +82,15 @@ class IDBDataFile; namespace BRM { +using PartitionNumberT = uint32_t; +using DBRootT = uint16_t; +using SegmentT = uint16_t; +using LastExtentIndexT = int; +using EmptyEMEntry = int; +using HighestOffset = uint32_t; +using LastIndEmptyIndEmptyInd = std::pair; +using DBRootVec = std::vector; + // assumed column width when calculating dictionary store extent size #define DICT_COL_WIDTH 8 @@ -98,8 +114,6 @@ const char CP_INVALID = 0; const char CP_UPDATING = 1; const char CP_VALID = 2; -// The _v4 structs are defined below for upgrading extent map -// from v4 to v5; see ExtentMap::loadVersion4or5 for details. struct EMCasualPartition_struct_v4 { RangePartitionData_t hi_val; // This needs to be reinterpreted as unsigned for uint64_t column types. @@ -113,16 +127,15 @@ struct EMPartition_struct_v4 { EMCasualPartition_struct_v4 cprange; }; - struct EMEntry_v4 { InlineLBIDRange range; int fileID; uint32_t blockOffset; HWM_t HWM; - uint32_t partitionNum; // starts at 0 - uint16_t segmentNum; // starts at 0 - uint16_t dbRoot; // starts at 1 to match Columnstore.xml + PartitionNumberT partitionNum; // starts at 0 + uint16_t segmentNum; // starts at 0 + DBRootT dbRoot; // starts at 1 to match Columnstore.xml uint16_t colWid; int16_t status; // extent avail for query or not, or out of service EMPartition_struct_v4 partition; @@ -151,7 +164,7 @@ struct EMCasualPartition_struct EXPORT EMCasualPartition_struct(const EMCasualPartition_struct& em); EXPORT EMCasualPartition_struct& operator=(const EMCasualPartition_struct& em); }; -typedef EMCasualPartition_struct EMCasualPartition_t; +using EMCasualPartition_t = EMCasualPartition_struct; struct EMPartition_struct { @@ -165,9 +178,9 @@ struct EMEntry int fileID; uint32_t blockOffset; HWM_t HWM; - uint32_t partitionNum; // starts at 0 - uint16_t segmentNum; // starts at 0 - uint16_t dbRoot; // starts at 1 to match Columnstore.xml + PartitionNumberT partitionNum; // starts at 0 + uint16_t segmentNum; // starts at 0 + DBRootT dbRoot; // starts at 1 to match Columnstore.xml uint16_t colWid; int16_t status; // extent avail for query or not, or out of service EMPartition_t partition; @@ -319,6 +332,146 @@ class FreeListImpl static FreeListImpl* fInstance; }; +using ShmSegmentManagerT = bi::managed_shared_memory::segment_manager; +using ShmVoidAllocator = bi::allocator; + +using ExtentMapIdxT = size_t; +using ExtentMapIdxTAlloc = bi::allocator; +using PartitionNumberTAlloc = bi::allocator; +using ExtentMapIndicesT = bi::vector; + +using PartitionIndexContainerKeyT = PartitionNumberT; +using PartitionIndexContainerValT = std::pair; +using PartitionIndexContainerValTAlloc = bi::allocator; +// Can't use std::unordered_map presumably b/c the map's pointer type doesn't use offset_type as boost::u_map +// does +using PartitionIndexContainerT = + boost::unordered_map, std::equal_to, + PartitionIndexContainerValTAlloc>; + +using OIDIndexContainerKeyT = OID_t; +using OIDIndexContainerValT = std::pair; +using OIDIndexContainerValTAlloc = bi::allocator; +using OIDIndexContainerT = + boost::unordered_map, + std::equal_to, OIDIndexContainerValTAlloc>; + +using DBRootIndexTAlloc = bi::allocator; +using DBRootIndexContainerT = bi::vector; +using ExtentMapIndex = DBRootIndexContainerT; +using ExtentMapIndexFindResult = bi::vector; +using InsertUpdateShmemKeyPair = std::pair; + +class ExtentMapIndexImpl +{ + public: + ~ExtentMapIndexImpl(){}; + + static ExtentMapIndexImpl* makeExtentMapIndexImpl(unsigned key, off_t size, bool readOnly = false); + static void refreshShm() + { + if (fInstance_) + { + delete fInstance_; + fInstance_ = nullptr; + } + } + + // The multipliers and constants here are pure theoretical + // tested using customer's data. + static size_t estimateEMIndexSize(uint32_t numberOfExtents) + { + // These are just educated guess values to calculate initial + // managed shmem size. + constexpr const size_t tablesNumber_ = 100ULL; + constexpr const size_t columnsNumber_ = 200ULL; + constexpr const size_t dbRootsNumber_ = 3ULL; + constexpr const size_t filesInPartition_ = 4ULL; + constexpr const size_t extentsInPartition_ = filesInPartition_ * 2; + return numberOfExtents * emIdentUnitSize_ + + numberOfExtents / extentsInPartition_ * partitionContainerUnitSize_ + + dbRootsNumber_ * tablesNumber_ * columnsNumber_; + } + + bool growIfNeeded(const size_t memoryNeeded); + + inline void grow(off_t size) + { + int rc = fBRMManagedShmMemImpl_.grow(size); + idbassert(rc == 0); + } + // After this call one needs to refresh any refs or ptrs sourced + // from this shmem. + inline void makeReadOnly() + { + fBRMManagedShmMemImpl_.setReadOnly(); + } + + inline void swapout(BRMManagedShmImpl& rhs) + { + fBRMManagedShmMemImpl_.swap(rhs); + } + + inline unsigned key() const + { + return fBRMManagedShmMemImpl_.key(); + } + + unsigned getShmemSize() + { + return fBRMManagedShmMemImpl_.getManagedSegment()->get_size(); + } + + size_t getShmemFree() + { + return fBRMManagedShmMemImpl_.getManagedSegment()->get_free_memory(); + } + + unsigned getShmemImplSize() + { + return fBRMManagedShmMemImpl_.size(); + } + + void createExtentMapIndexIfNeeded(); + ExtentMapIndex* get(); + InsertUpdateShmemKeyPair insert(const EMEntry& emEntry, const size_t emIdx); + InsertUpdateShmemKeyPair insert2ndLayerWrapper(OIDIndexContainerT& oids, const EMEntry& emEntry, + const size_t emIdx, const bool aShmemHasGrown); + InsertUpdateShmemKeyPair insert2ndLayer(OIDIndexContainerT& oids, const EMEntry& emEntry, + const size_t emIdx, const bool aShmemHasGrown); + InsertUpdateShmemKeyPair insert3dLayerWrapper(PartitionIndexContainerT& partitions, const EMEntry& emEntry, + const size_t emIdx, const bool aShmemHasGrown); + InsertUpdateShmemKeyPair insert3dLayer(PartitionIndexContainerT& partitions, const EMEntry& emEntry, + const size_t emIdx, const bool aShmemHasGrown); + ExtentMapIndexFindResult find(const DBRootT dbroot, const OID_t oid, + const PartitionNumberT partitionNumber); + ExtentMapIndexFindResult find(const DBRootT dbroot, const OID_t oid); + ExtentMapIndexFindResult search2ndLayer(OIDIndexContainerT& oids, const OID_t oid, + const PartitionNumberT partitionNumber); + ExtentMapIndexFindResult search2ndLayer(OIDIndexContainerT& oids, const OID_t oid); + ExtentMapIndexFindResult search3dLayer(PartitionIndexContainerT& partitions, + const PartitionNumberT partitionNumber); + void deleteDbRoot(const DBRootT dbroot); + void deleteOID(const DBRootT dbroot, const OID_t oid); + void deleteEMEntry(const EMEntry& emEntry, const ExtentMapIdxT emIdent); + + private: + BRMManagedShmImpl fBRMManagedShmMemImpl_; + ExtentMapIndexImpl(unsigned key, off_t size, bool readOnly = false); + ExtentMapIndexImpl(const ExtentMapIndexImpl& rhs); + ExtentMapIndexImpl& operator=(const ExtentMapIndexImpl& rhs); + + static std::mutex fInstanceMutex_; + static ExtentMapIndexImpl* fInstance_; + static const constexpr uint32_t dbRootContainerUnitSize_ = 64ULL; + static const constexpr uint32_t oidContainerUnitSize_ = 352ULL; // 2 * map overhead + static const constexpr uint32_t partitionContainerUnitSize_ = 368ULL; // single map overhead + static const constexpr uint32_t emIdentUnitSize_ = sizeof(uint64_t); + static const constexpr uint32_t extraUnits_ = 2; + static const constexpr size_t freeSpaceThreshold_ = 256 * 1024; +}; + /** @brief This class encapsulates the extent map functionality of the system * * This class encapsulates the extent map functionality of the system. It @@ -345,7 +498,7 @@ class ExtentMap : public Undoable */ EXPORT void load(const std::string& filename, bool fixFL = false); - /** @brief Loads the ExtentMap entries from a binayr blob. + /** @brief Loads the ExtentMap entries from a binary blob. * * Loads the ExtentMap entries from a file. This will * clear out any existing entries. The intention is that before @@ -886,6 +1039,9 @@ class ExtentMap : public Undoable EXPORT void dumpTo(std::ostream& os); EXPORT const bool* getEMLockStatus(); EXPORT const bool* getEMFLLockStatus(); + EXPORT const bool* getEMIndexLockStatus(); + size_t EMIndexShmemSize(); + size_t EMIndexShmemFree(); #ifdef BRM_DEBUG EXPORT void printEM() const; @@ -895,11 +1051,11 @@ class ExtentMap : public Undoable #endif private: - static const size_t EM_INCREMENT_ROWS = 100; - static const size_t EM_INITIAL_SIZE = EM_INCREMENT_ROWS * 10 * sizeof(EMEntry); - static const size_t EM_INCREMENT = EM_INCREMENT_ROWS * sizeof(EMEntry); - static const size_t EM_FREELIST_INITIAL_SIZE = 50 * sizeof(InlineLBIDRange); - static const size_t EM_FREELIST_INCREMENT = 50 * sizeof(InlineLBIDRange); + static const constexpr size_t EM_INCREMENT_ROWS = 100; + static const constexpr size_t EM_INITIAL_SIZE = EM_INCREMENT_ROWS * 10 * sizeof(EMEntry); + static const constexpr size_t EM_INCREMENT = EM_INCREMENT_ROWS * sizeof(EMEntry); + static const constexpr size_t EM_FREELIST_INITIAL_SIZE = 50 * sizeof(InlineLBIDRange); + static const constexpr size_t EM_FREELIST_INCREMENT = 50 * sizeof(InlineLBIDRange); ExtentMap(const ExtentMap& em); ExtentMap& operator=(const ExtentMap& em); @@ -910,6 +1066,7 @@ class ExtentMap : public Undoable key_t fCurrentFLShmkey; MSTEntry* fEMShminfo; MSTEntry* fFLShminfo; + MSTEntry* fEMIndexShminfo; const MasterSegmentTable fMST; bool r_only; typedef std::tr1::unordered_map PmDbRootMap_t; @@ -917,8 +1074,9 @@ class ExtentMap : public Undoable time_t fCacheTime; // timestamp associated with config cache int numUndoRecords; - bool flLocked, emLocked; - static boost::mutex mutex; // @bug5355 - made mutex static + bool flLocked, emLocked, emIndexLocked; + static boost::mutex mutex; // @bug5355 - made mutex static + static boost::mutex emIndexMutex; boost::mutex fConfigCacheMutex; // protect access to Config Cache enum OPS @@ -930,6 +1088,12 @@ class ExtentMap : public Undoable OPS EMLock, FLLock; + LastIndEmptyIndEmptyInd _createExtentCommonSearch(const OID_t OID, const DBRootT dbRoot, + const PartitionNumberT partitionNum, + const SegmentT segmentNum); + + void logAndSetEMIndexReadOnly(const std::string& funcName); + LBID_t _createColumnExtent_DBroot(uint32_t size, int OID, uint32_t colWidth, uint16_t dbRoot, execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t& partitionNum, uint16_t& segmentNum, uint32_t& startBlockOffset); @@ -941,24 +1105,32 @@ class ExtentMap : public Undoable uint16_t segmentNum); template bool isValidCPRange(const T& max, const T& min, execplan::CalpontSystemCatalog::ColDataType type) const; - void deleteExtent(int emIndex); + void deleteExtent(const int emIndex, const bool clearEMIndex = true); LBID_t getLBIDsFromFreeList(uint32_t size); void reserveLBIDRange(LBID_t start, uint8_t size); // used by load() to allocate pre-existing LBIDs - key_t chooseEMShmkey(); // see the code for how keys are segmented - key_t chooseFLShmkey(); // see the code for how keys are segmented + key_t chooseEMShmkey(); + key_t chooseFLShmkey(); + key_t chooseEMIndexShmkey(); + key_t getInitialEMIndexShmkey() const; + // see the code for how keys are segmented + key_t chooseShmkey(const MSTEntry* masterTableEntry, const uint32_t keyRangeBase) const; void grabEMEntryTable(OPS op); void grabFreeList(OPS op); + void grabEMIndex(OPS op); void releaseEMEntryTable(OPS op); void releaseFreeList(OPS op); + void releaseEMIndex(OPS op); void growEMShmseg(size_t nrows = 0); void growFLShmseg(); + void growEMIndexShmseg(const size_t suggestedSize = 0); void finishChanges(); EXPORT unsigned getFilesPerColumnPartition(); unsigned getExtentsPerSegmentFile(); unsigned getDbRootCount(); void getPmDbRoots(int pm, std::vector& dbRootList); + DBRootVec getAllDbRoots(); void checkReloadConfig(); ShmKeys fShmKeys; @@ -979,6 +1151,7 @@ class ExtentMap : public Undoable ExtentMapImpl* fPExtMapImpl; FreeListImpl* fPFreeListImpl; + ExtentMapIndexImpl* fPExtMapIndexImpl_; }; inline std::ostream& operator<<(std::ostream& os, ExtentMap& rhs) diff --git a/versioning/BRM/load_brm.cpp b/versioning/BRM/load_brm.cpp index 4378fb766..b72bd3c2a 100644 --- a/versioning/BRM/load_brm.cpp +++ b/versioning/BRM/load_brm.cpp @@ -127,4 +127,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/versioning/BRM/lock_grabber.cpp b/versioning/BRM/lock_grabber.cpp index 6932b7708..7c6884bb8 100644 --- a/versioning/BRM/lock_grabber.cpp +++ b/versioning/BRM/lock_grabber.cpp @@ -1,4 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2016-2022 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -21,6 +22,7 @@ * third, lock or unlock it */ +#include #include #include #include @@ -32,10 +34,15 @@ char* name; void usage() { - cout << "Usage " << name << " which_lock_to_use which_side_to_use lock_or_unlock\n" - << " which_lock_to_use: 1=VSS 2=ExtentMap 3=FreeList 4=VBBM 5=CopyLocks\n" - << " which_side_to_use: r|w (read or write)\n" - << " lock_or_unlock: l|u (lock or unlock)\n"; + std::cout << "Usage " << name << " which_lock_to_use which_side_to_use lock_or_unlock" << std::endl; + size_t lockId = 0; + for (auto& lockName : RWLockNames) + { + std::cout << " " << lockId++ << "=" << lockName << " "; + } + std::cout << std::endl + << " which_side_to_use: r|w (read or write)" << std::endl + << " lock_or_unlock: l|u (lock or unlock)" << std::endl; exit(1); } @@ -54,10 +61,21 @@ int main(int argc, char** argv) if (strlen(argv[1]) != 1 || strlen(argv[2]) != 1 || strlen(argv[3]) != 1) usage(); - which_lock = atoi(argv[1]); - - if (which_lock < 1 || which_lock > 5) + try + { + which_lock = std::stoi(argv[1]); + } + catch (std::exception const& e) + { + std::cerr << "Cannot convert the lock id: " << e.what() << std::endl; usage(); + } + + if (which_lock >= RWLockNames.size()) + usage(); + + size_t minLockId = (which_lock > 0) ? which_lock : 1; + size_t maxLockId = (which_lock > 0) ? which_lock : RWLockNames.size() - 1; if (argv[2][0] == 'r') which_side = 0; @@ -73,17 +91,28 @@ int main(int argc, char** argv) else usage(); - rwlock = new RWLock(0x10000 * which_lock); + for (size_t i = minLockId; i <= maxLockId; ++i) + { + rwlock = new RWLock(0x10000 * which_lock); - if (which_side == 0) - if (lock_unlock == 0) - rwlock->read_lock(); + if (which_side == 0) + { + if (lock_unlock == 0) + rwlock->read_lock(); + else + rwlock->read_unlock(); + } + else if (lock_unlock == 0) + { + rwlock->write_lock(); + } else - rwlock->read_unlock(); - else if (lock_unlock == 0) - rwlock->write_lock(); - else - rwlock->write_unlock(); + { + rwlock->write_unlock(); + } + + delete rwlock; + } return 0; } diff --git a/versioning/BRM/lock_state.cpp b/versioning/BRM/lock_state.cpp index 812b59c08..4b5cef63e 100644 --- a/versioning/BRM/lock_state.cpp +++ b/versioning/BRM/lock_state.cpp @@ -1,4 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2016-2022 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -22,6 +23,7 @@ */ #include +#include #include #include @@ -32,14 +34,18 @@ char* name; void usage() { - cout << "Usage " << name << " which_lock_to_use which_side_to_use lock_or_unlock\n" - << " which_lock_to_use: 1=VSS 2=ExtentMap 3=FreeList 4=VBBM 5=CopyLocks\n"; + std::cout << "Usage " << name << " which_lock_to_use:" << std::endl; + size_t lockId = 0; + for (auto& lockName : RWLockNames) + { + std::cout << " " << lockId++ << "=" << lockName << std::endl; + } exit(1); } int main(int argc, char** argv) { - uint32_t which_lock; // 1-5 + uint32_t which_lock; // 0-6 RWLock* rwlock; LockState state; @@ -51,18 +57,35 @@ int main(int argc, char** argv) if (strlen(argv[1]) != 1) usage(); - which_lock = atoi(argv[1]); + try + { + which_lock = std::stoi(argv[1]); + } + catch (std::exception const& e) + { + std::cerr << "Cannot convert the lock id: " << e.what() << std::endl; + usage(); + } - if (which_lock < 1 || which_lock > 5) + if (which_lock >= RWLockNames.size()) usage(); - rwlock = new RWLock(0x10000 * which_lock); - state = rwlock->getLockState(); - cout << "readers = " << state.reading << endl - << "writers = " << state.writing << endl - << "readers waiting = " << state.readerswaiting << endl - << "writers waiting = " << state.writerswaiting << endl - << "mutex locked = " << (int)state.mutexLocked << endl; + size_t minLockId = (which_lock > 0) ? which_lock : 1; + size_t maxLockId = (which_lock > 0) ? which_lock : RWLockNames.size() - 1; + + for (size_t i = minLockId; i <= maxLockId; ++i) + { + rwlock = new RWLock(0x10000 * i); + state = rwlock->getLockState(); + + cout << RWLockNames[i] << " RWLock" << std::endl + << " readers = " << state.reading << std::endl + << " writers = " << state.writing << std::endl + << " readers waiting = " << state.readerswaiting << std::endl + << " writers waiting = " << state.writerswaiting << std::endl + << " mutex locked = " << (int)state.mutexLocked << std::endl; + delete rwlock; + } return 0; } diff --git a/versioning/BRM/mastersegmenttable.cpp b/versioning/BRM/mastersegmenttable.cpp index c88dc13ba..31b2da6f7 100644 --- a/versioning/BRM/mastersegmenttable.cpp +++ b/versioning/BRM/mastersegmenttable.cpp @@ -138,6 +138,7 @@ MasterSegmentTable::MasterSegmentTable() RWLockKeys[2] = fShmKeys.KEYRANGE_VBBM_BASE; RWLockKeys[3] = fShmKeys.KEYRANGE_VSS_BASE; RWLockKeys[4] = fShmKeys.KEYRANGE_CL_BASE; + RWLockKeys[5] = fShmKeys.KEYRANGE_EXTENTMAP_INDEX_BASE; try { diff --git a/versioning/BRM/mastersegmenttable.h b/versioning/BRM/mastersegmenttable.h index 25580490f..be7b96639 100644 --- a/versioning/BRM/mastersegmenttable.h +++ b/versioning/BRM/mastersegmenttable.h @@ -109,8 +109,10 @@ class MasterSegmentTable static const int VSSSegment = 3; /// specifies the copy lock segment static const int CLSegment = 4; + /// specifies the EM Index segment + static const int EMIndex = 5; /// the number of tables currently defined - static const int nTables = 5; + static const int nTables = 6; /** @brief This function gets the specified table. * diff --git a/versioning/BRM/shmkeys.cpp b/versioning/BRM/shmkeys.cpp index f7642ff9f..943d81d1b 100644 --- a/versioning/BRM/shmkeys.cpp +++ b/versioning/BRM/shmkeys.cpp @@ -50,6 +50,7 @@ ShmKeys::ShmKeys() KEYRANGE_EMFREELIST_BASE = 0x30000 | (BRM_UID << 20); KEYRANGE_VBBM_BASE = 0x40000 | (BRM_UID << 20); KEYRANGE_CL_BASE = 0x50000 | (BRM_UID << 20); + KEYRANGE_EXTENTMAP_INDEX_BASE = 0x60000 | (BRM_UID << 20); MST_SYSVKEY = 0xff000000 | BRM_UID; PROCESSSTATUS_SYSVKEY = 0xfd000000 | BRM_UID; SYSTEMSTATUS_SYSVKEY = 0xfc000000 | BRM_UID; @@ -62,7 +63,7 @@ ShmKeys::ShmKeys() string ShmKeys::keyToName(unsigned key) { ostringstream oss; - oss << "InfiniDB-shm-"; + oss << "MCS-shm-"; oss << setw(8) << setfill('0') << hex << key; return oss.str(); } diff --git a/versioning/BRM/shmkeys.h b/versioning/BRM/shmkeys.h index 6b4ef9999..ce7f00d1a 100644 --- a/versioning/BRM/shmkeys.h +++ b/versioning/BRM/shmkeys.h @@ -55,6 +55,7 @@ struct ShmKeys uint32_t KEYRANGE_EMFREELIST_BASE; uint32_t KEYRANGE_VBBM_BASE; uint32_t KEYRANGE_VSS_BASE; + uint32_t KEYRANGE_EXTENTMAP_INDEX_BASE; /****** Fixed location assignments *******/ uint32_t MST_SYSVKEY; diff --git a/versioning/BRM/slavecomm.cpp b/versioning/BRM/slavecomm.cpp index 7b665a1a6..1f490dd8c 100644 --- a/versioning/BRM/slavecomm.cpp +++ b/versioning/BRM/slavecomm.cpp @@ -2432,4 +2432,3 @@ void SlaveComm::do_dmlReleaseLBIDRanges(ByteStream& msg) } // namespace BRM -// vim:ts=4 sw=4: diff --git a/versioning/BRM/slavedbrmnode.cpp b/versioning/BRM/slavedbrmnode.cpp index ae7bb1d11..2331169a9 100644 --- a/versioning/BRM/slavedbrmnode.cpp +++ b/versioning/BRM/slavedbrmnode.cpp @@ -1492,6 +1492,11 @@ const bool* SlaveDBRMNode::getEMLockStatus() return em.getEMLockStatus(); } +const bool* SlaveDBRMNode::getEMIndexLockStatus() +{ + return em.getEMIndexLockStatus(); +} + const bool* SlaveDBRMNode::getVBBMLockStatus() { return &locked[0]; diff --git a/versioning/BRM/slavedbrmnode.h b/versioning/BRM/slavedbrmnode.h index f614c8ce5..e53eb4936 100644 --- a/versioning/BRM/slavedbrmnode.h +++ b/versioning/BRM/slavedbrmnode.h @@ -461,6 +461,7 @@ class SlaveDBRMNode EXPORT const bool* getEMFLLockStatus(); EXPORT const bool* getEMLockStatus(); + EXPORT const bool* getEMIndexLockStatus(); EXPORT const bool* getVBBMLockStatus(); EXPORT const bool* getVSSLockStatus(); diff --git a/versioning/BRM/slavenode.cpp b/versioning/BRM/slavenode.cpp index 8651dc823..667feaa08 100644 --- a/versioning/BRM/slavenode.cpp +++ b/versioning/BRM/slavenode.cpp @@ -147,6 +147,8 @@ int ServiceWorkerNode::Child() monitorThreads.create_thread(RWLockMonitor(&die, slave.getEMLockStatus(), keys.KEYRANGE_EXTENTMAP_BASE)); monitorThreads.create_thread(RWLockMonitor(&die, slave.getVBBMLockStatus(), keys.KEYRANGE_VBBM_BASE)); monitorThreads.create_thread(RWLockMonitor(&die, slave.getVSSLockStatus(), keys.KEYRANGE_VSS_BASE)); + monitorThreads.create_thread( + RWLockMonitor(&die, slave.getEMIndexLockStatus(), keys.KEYRANGE_EXTENTMAP_INDEX_BASE)); try { diff --git a/versioning/BRM/vbbm.cpp b/versioning/BRM/vbbm.cpp index 253bd0512..2ed2454d1 100644 --- a/versioning/BRM/vbbm.cpp +++ b/versioning/BRM/vbbm.cpp @@ -40,7 +40,6 @@ #include #include -namespace bi = boost::interprocess; #include #include @@ -1076,8 +1075,8 @@ void VBBM::save(string filename) } var = VBBM_MAGIC_V2; - int bytesWritten = 0; - int bytesToWrite = 12; + [[maybe_unused]] int bytesWritten = 0; + [[maybe_unused]] int bytesToWrite = 12; bytesWritten += out->write((char*)&var, 4); bytesWritten += out->write((char*)&vbbm->vbCurrentSize, 4); bytesWritten += out->write((char*)&vbbm->nFiles, 4); diff --git a/writeengine/bulk/cpimport.cpp b/writeengine/bulk/cpimport.cpp index bd7a62ea5..54aeaa2c5 100644 --- a/writeengine/bulk/cpimport.cpp +++ b/writeengine/bulk/cpimport.cpp @@ -172,12 +172,11 @@ void printUsage() << " -T Timezone used for TIMESTAMP datatype" << endl << " Possible values: \"SYSTEM\" (default)" << endl << " : Offset in the form +/-HH:MM" << endl - << endl - << " -y S3 Authentication Key (for S3 imports)" << endl - << " -K S3 Authentication Secret (for S3 imports)" << endl - << " -t S3 Bucket (for S3 imports)" << endl - << " -H S3 Hostname (for S3 imports, Amazon's S3 default)" << endl - << " -g S3 Regions (for S3 imports)" << endl +// << " -y S3 Authentication Key (for S3 imports)" << endl +// << " -K S3 Authentication Secret (for S3 imports)" << endl +// << " -t S3 Bucket (for S3 imports)" << endl +// << " -H S3 Hostname (for S3 imports, Amazon's S3 default)" << endl +// << " -g S3 Regions (for S3 imports)" << endl << " -U username of new data files owner. Default is mysql" << endl; cout << " Example1:" << endl @@ -310,7 +309,7 @@ void parseCmdLineArgs(int argc, char** argv, BulkLoad& curJob, std::string& sJob BulkModeType bulkMode = BULK_MODE_LOCAL; std::string jobUUID; - while ((option = getopt(argc, argv, "b:c:d:e:f:hij:kl:m:n:p:r:s:u:w:B:C:DE:I:P:R:ST:X:NL:y:K:t:H:g:U:")) != + while ((option = getopt(argc, argv, "b:c:d:e:f:hij:kl:m:n:p:r:s:u:w:B:C:DE:I:P:R:ST:X:NL:U:")) != EOF) { switch (option) @@ -648,12 +647,12 @@ void parseCmdLineArgs(int argc, char** argv, BulkLoad& curJob, std::string& sJob std::string timeZone = optarg; long offset; - if (timeZone != "SYSTEM" && dataconvert::timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset)) + if (dataconvert::timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset)) { startupError(std::string("Value for option -T is invalid"), true); } - curJob.setTimeZone(timeZone); + curJob.setTimeZone(offset); break; } @@ -676,7 +675,7 @@ void parseCmdLineArgs(int argc, char** argv, BulkLoad& curJob, std::string& sJob BulkLoad::disableConsoleOutput(true); break; } - +/* case 'y': { curJob.setS3Key(optarg); @@ -706,7 +705,7 @@ void parseCmdLineArgs(int argc, char** argv, BulkLoad& curJob, std::string& sJob curJob.setS3Region(optarg); break; } - +*/ case 'U': { curJob.setUsername(optarg); @@ -917,16 +916,24 @@ void getTableOID(const std::string& xmlGenSchema, const std::string& xmlGenTable void constructTempXmlFile(const std::string& tempJobDir, const std::string& sJobIdStr, const std::string& xmlGenSchema, const std::string& xmlGenTable, const std::string& alternateImportDir, const std::string& S3Bucket, - boost::filesystem::path& sFileName) + const std::string& tableOIDStr, boost::filesystem::path& sFileName) { // Construct the job description file name std::string xmlErrMsg; int rc = 0; - std::string tableOIDStr; - getTableOID(xmlGenSchema, xmlGenTable, tableOIDStr); + std::string localTableOIDStr; + if (tableOIDStr.empty()) + { + getTableOID(xmlGenSchema, xmlGenTable, localTableOIDStr); + } + else + { + localTableOIDStr = tableOIDStr; + } + rc = XMLJob::genJobXMLFileName(std::string(), tempJobDir, sJobIdStr, true, // using temp job xml file - xmlGenSchema, xmlGenTable, sFileName, xmlErrMsg, tableOIDStr); + xmlGenSchema, xmlGenTable, sFileName, xmlErrMsg, localTableOIDStr); if (rc != NO_ERROR) { @@ -946,7 +953,7 @@ void constructTempXmlFile(const std::string& tempJobDir, const std::string& sJob { genProc.startXMLFile(); execplan::CalpontSystemCatalog::TableName tbl(xmlGenSchema, xmlGenTable); - genProc.makeTableData(tbl); + genProc.makeTableData(tbl, localTableOIDStr); if (!genProc.makeColumnData(tbl)) { @@ -1223,9 +1230,9 @@ int main(int argc, char** argv) if (!xmlGenSchema.empty()) // create temporary job file name { // If JobID is not provided, then default to the table OID + std::string tableOIDStr{""}; if (sJobIdStr.empty()) { - std::string tableOIDStr; getTableOID(xmlGenSchema, xmlGenTable, tableOIDStr); if (!(BulkLoad::disableConsoleOutput())) @@ -1240,7 +1247,7 @@ int main(int argc, char** argv) bUseTempJobFile = true; constructTempXmlFile(curJob.getTempJobDir(), sJobIdStr, xmlGenSchema, xmlGenTable, - curJob.getAlternateImportDir(), curJob.getS3Bucket(), sFileName); + curJob.getAlternateImportDir(), curJob.getS3Bucket(), tableOIDStr, sFileName); } else // create user's persistent job file name { diff --git a/writeengine/bulk/we_bulkload.cpp b/writeengine/bulk/we_bulkload.cpp index 566b99759..7e51018ef 100644 --- a/writeengine/bulk/we_bulkload.cpp +++ b/writeengine/bulk/we_bulkload.cpp @@ -159,7 +159,7 @@ BulkLoad::BulkLoad() , fbContinue(false) , fDisableTimeOut(false) , fUUID(boost::uuids::nil_generator()()) - , fTimeZone("SYSTEM") + , fTimeZone(dataconvert::systemTimeZoneOffset()) , fUsername("mysql") // MCOL-4328 default file owner { fTableInfo.clear(); @@ -739,7 +739,7 @@ int BulkLoad::preProcess(Job& job, int tableNo, TableInfo* tableInfo) // Setup import to start loading into starting HWM DB file RETURN_ON_ERROR(info->setupInitialColumnExtent(dbRoot, partition, segment, job.jobTableList[tableNo].tblName, lbid, oldHwm, hwm, - bSkippedToNewExtent, false)); + bSkippedToNewExtent, bSkippedToNewExtent || oldHwm < 1)); } tableInfo->addColumn(info); diff --git a/writeengine/bulk/we_bulkload.h b/writeengine/bulk/we_bulkload.h index cd800dd9e..ccc362ca4 100644 --- a/writeengine/bulk/we_bulkload.h +++ b/writeengine/bulk/we_bulkload.h @@ -107,7 +107,7 @@ class BulkLoad : public FileOp void addToCmdLineImportFileList(const std::string& importFile); const std::string& getAlternateImportDir() const; const std::string& getErrorDir() const; - const std::string& getTimeZone() const; + long getTimeZone() const; const std::string& getJobDir() const; const std::string& getSchema() const; const std::string& getTempJobDir() const; @@ -145,7 +145,7 @@ class BulkLoad : public FileOp void setTruncationAsError(bool bTruncationAsError); void setJobUUID(const std::string& jobUUID); void setErrorDir(const std::string& errorDir); - void setTimeZone(const std::string& timeZone); + void setTimeZone(long timeZone); void setS3Key(const std::string& key); void setS3Secret(const std::string& secret); void setS3Bucket(const std::string& bucket); @@ -229,7 +229,9 @@ class BulkLoad : public FileOp bool fDisableTimeOut; // disable timeout when waiting for table lock boost::uuids::uuid fUUID; // job UUID static bool fNoConsoleOutput; // disable output to console - std::string fTimeZone; // Timezone to use for TIMESTAMP data type + long fTimeZone; // Timezone offset (in seconds) relative to UTC, + // to use for TIMESTAMP data type. For example, + // for EST which is UTC-5:00, offset will be -18000s. std::string fS3Key; // S3 Key std::string fS3Secret; // S3 Secret std::string fS3Host; // S3 Host @@ -318,7 +320,7 @@ inline const std::string& BulkLoad::getErrorDir() const return fErrorDir; } -inline const std::string& BulkLoad::getTimeZone() const +inline long BulkLoad::getTimeZone() const { return fTimeZone; } @@ -486,7 +488,7 @@ inline void BulkLoad::setErrorDir(const std::string& errorDir) fErrorDir = errorDir; } -inline void BulkLoad::setTimeZone(const std::string& timeZone) +inline void BulkLoad::setTimeZone(long timeZone) { fTimeZone = timeZone; } diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index 9f26a161f..106689eaf 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -139,7 +139,7 @@ BulkLoadBuffer::BulkLoadBuffer(unsigned numberOfCols, unsigned bufferSize, Log* , fTableName(tableName) , fbTruncationAsError(false) , fImportDataMode(IMPORT_DATA_TEXT) - , fTimeZone("SYSTEM") + , fTimeZone(dataconvert::systemTimeZoneOffset()) , fFixedBinaryRecLen(0) { fData = new char[bufferSize]; @@ -1717,7 +1717,7 @@ int BulkLoadBuffer::parseCol(ColumnInfo& columnInfo) lastInputRowInExtent += columnInfo.rowsPerExtent(); - if (isUnsigned(columnInfo.column.dataType) || isCharType(columnInfo.column.dataType)) + if (isUnsigned(columnInfo.column.dataType)) { if (columnInfo.column.width <= 8) { diff --git a/writeengine/bulk/we_bulkloadbuffer.h b/writeengine/bulk/we_bulkloadbuffer.h index ea0625e39..2abc40e63 100644 --- a/writeengine/bulk/we_bulkloadbuffer.h +++ b/writeengine/bulk/we_bulkloadbuffer.h @@ -54,7 +54,7 @@ class BLBufferStats }; BLBufferStats(ColDataType colDataType) : satCount(0) { - if (isUnsigned(colDataType) || isCharType(colDataType)) + if (isUnsigned(colDataType)) { minBufferVal = static_cast(MAX_UBIGINT); maxBufferVal = static_cast(MIN_UBIGINT); @@ -152,7 +152,9 @@ class BulkLoadBuffer // for db cols (omits default cols) bool fbTruncationAsError; // Treat string truncation as error ImportDataMode fImportDataMode; // Import data in text or binary mode - std::string fTimeZone; // Timezone used by TIMESTAMP datatype + long fTimeZone; // Timezone offset (in seconds) relative to UTC, + // to use for TIMESTAMP data type. For example, + // for EST which is UTC-5:00, offset will be -18000s. unsigned int fFixedBinaryRecLen; // Fixed rec len used in binary mode //-------------------------------------------------------------------------- @@ -388,7 +390,7 @@ class BulkLoadBuffer /** @brief set timezone. */ - void setTimeZone(const std::string& timeZone) + void setTimeZone(long timeZone) { fTimeZone = timeZone; } diff --git a/writeengine/bulk/we_colextinf.cpp b/writeengine/bulk/we_colextinf.cpp index 1551cd822..74271c2f4 100644 --- a/writeengine/bulk/we_colextinf.cpp +++ b/writeengine/bulk/we_colextinf.cpp @@ -104,7 +104,7 @@ void ColExtInf::addOrUpdateEntryTemplate(RID lastInputRow, T minVal, T maxVal, C } else // Update the range { - if (isUnsigned(colDataType) || isCharType(colDataType)) + if (isUnsigned(colDataType)) { if (width <= 8) { diff --git a/writeengine/bulk/we_tableinfo.cpp b/writeengine/bulk/we_tableinfo.cpp index 29dc7b202..cdda59c17 100644 --- a/writeengine/bulk/we_tableinfo.cpp +++ b/writeengine/bulk/we_tableinfo.cpp @@ -146,7 +146,7 @@ TableInfo::TableInfo(Log* logger, const BRM::TxnID txnID, const string& processN , fKeepRbMetaFile(bKeepRbMetaFile) , fbTruncationAsError(false) , fImportDataMode(IMPORT_DATA_TEXT) - , fTimeZone("SYSTEM") + , fTimeZone(dataconvert::systemTimeZoneOffset()) , fTableLocked(false) , fReadFromStdin(false) , fReadFromS3(false) diff --git a/writeengine/bulk/we_tableinfo.h b/writeengine/bulk/we_tableinfo.h index 522760fe7..6fbca9fc7 100644 --- a/writeengine/bulk/we_tableinfo.h +++ b/writeengine/bulk/we_tableinfo.h @@ -78,7 +78,7 @@ class TableInfo : public WeUIDGID int fCurrentReadBuffer; // Id of current buffer being popu- // lated by the read thread RID fTotalReadRows; // Total number of rows read - volatile unsigned fTotalErrRows; // Total error rows among all input + unsigned fTotalErrRows; // Total error rows among all input // for this table. Is volatile to // insure parser & reader threads // see the latest value. @@ -127,7 +127,9 @@ class TableInfo : public WeUIDGID // data file bool fbTruncationAsError; // Treat string truncation as error ImportDataMode fImportDataMode; // Import data in text or binary mode - std::string fTimeZone; // Timezone used by TIMESTAMP data type + long fTimeZone; // Timezone offset (in seconds) relative to UTC, + // to use for TIMESTAMP data type. For example, + // for EST which is UTC-5:00, offset will be -18000s. volatile bool fTableLocked; // Do we have db table lock @@ -254,7 +256,7 @@ class TableInfo : public WeUIDGID /** @brief Get timezone. */ - const std::string& getTimeZone() const; + long getTimeZone() const; /** @brief Get number of buffers */ @@ -315,7 +317,7 @@ class TableInfo : public WeUIDGID /** @brief Set timezone. */ - void setTimeZone(const std::string& timeZone); + void setTimeZone(long timeZone); /** @brief Enable distributed mode, saving BRM updates in rptFileName */ @@ -481,7 +483,7 @@ inline ImportDataMode TableInfo::getImportDataMode() const return fImportDataMode; } -inline const std::string& TableInfo::getTimeZone() const +inline long TableInfo::getTimeZone() const { return fTimeZone; } @@ -582,7 +584,7 @@ inline void TableInfo::setImportDataMode(ImportDataMode importMode) fImportDataMode = importMode; } -inline void TableInfo::setTimeZone(const std::string& timeZone) +inline void TableInfo::setTimeZone(long timeZone) { fTimeZone = timeZone; } diff --git a/writeengine/client/we_clients.cpp b/writeengine/client/we_clients.cpp index c696dbb6d..b6d66e2e9 100644 --- a/writeengine/client/we_clients.cpp +++ b/writeengine/client/we_clients.cpp @@ -545,4 +545,3 @@ void WEClients::addDataToOutput(SBS sbs, uint32_t connIndex) } // namespace WriteEngine -// vim:ts=4 sw=4: diff --git a/writeengine/client/we_clients.h b/writeengine/client/we_clients.h index 82c40a0e8..6b1a59d7f 100644 --- a/writeengine/client/we_clients.h +++ b/writeengine/client/we_clients.h @@ -180,4 +180,3 @@ class WEClients #undef EXPORT -// vim:ts=4 sw=4: diff --git a/writeengine/redistribute/we_redistribute.cpp b/writeengine/redistribute/we_redistribute.cpp index d155715b5..9f9101935 100644 --- a/writeengine/redistribute/we_redistribute.cpp +++ b/writeengine/redistribute/we_redistribute.cpp @@ -52,4 +52,3 @@ void Redistribute::handleRedistributeMessage(ByteStream& bs, IOSocket& ios) } // namespace redistribute -// vim:ts=4 sw=4: diff --git a/writeengine/redistribute/we_redistribute.h b/writeengine/redistribute/we_redistribute.h index 4af403484..121efb11e 100644 --- a/writeengine/redistribute/we_redistribute.h +++ b/writeengine/redistribute/we_redistribute.h @@ -44,4 +44,3 @@ class Redistribute } // namespace redistribute -// vim:ts=4 sw=4: diff --git a/writeengine/redistribute/we_redistributecontrol.cpp b/writeengine/redistribute/we_redistributecontrol.cpp index e84bc9478..76a4892e5 100644 --- a/writeengine/redistribute/we_redistributecontrol.cpp +++ b/writeengine/redistribute/we_redistributecontrol.cpp @@ -698,4 +698,3 @@ void RedistributeControl::logMessage(const string& msg) } // namespace redistribute -// vim:ts=4 sw=4: diff --git a/writeengine/redistribute/we_redistributecontrol.h b/writeengine/redistribute/we_redistributecontrol.h index 3dcf350e4..13359344e 100644 --- a/writeengine/redistribute/we_redistributecontrol.h +++ b/writeengine/redistribute/we_redistributecontrol.h @@ -126,4 +126,3 @@ class RedistributeControl } // namespace redistribute -// vim:ts=4 sw=4: diff --git a/writeengine/redistribute/we_redistributecontrolthread.cpp b/writeengine/redistribute/we_redistributecontrolthread.cpp index ce808eb20..774f40118 100644 --- a/writeengine/redistribute/we_redistributecontrolthread.cpp +++ b/writeengine/redistribute/we_redistributecontrolthread.cpp @@ -838,4 +838,3 @@ void RedistributeControlThread::doStopAction() } // namespace redistribute -// vim:ts=4 sw=4: diff --git a/writeengine/redistribute/we_redistributecontrolthread.h b/writeengine/redistribute/we_redistributecontrolthread.h index ea32f3f5c..9b4c88ac8 100644 --- a/writeengine/redistribute/we_redistributecontrolthread.h +++ b/writeengine/redistribute/we_redistributecontrolthread.h @@ -125,4 +125,3 @@ class RedistributeControlThread } // namespace redistribute -// vim:ts=4 sw=4: diff --git a/writeengine/redistribute/we_redistributedef.h b/writeengine/redistribute/we_redistributedef.h index 7ddcf4e5e..4b1d7a895 100644 --- a/writeengine/redistribute/we_redistributedef.h +++ b/writeengine/redistribute/we_redistributedef.h @@ -203,4 +203,3 @@ struct RedistributeInfo } // namespace redistribute -// vim:ts=4 sw=4: diff --git a/writeengine/redistribute/we_redistributeworkerthread.cpp b/writeengine/redistribute/we_redistributeworkerthread.cpp index 6f4509839..822caff24 100644 --- a/writeengine/redistribute/we_redistributeworkerthread.cpp +++ b/writeengine/redistribute/we_redistributeworkerthread.cpp @@ -1520,4 +1520,3 @@ void RedistributeWorkerThread::logMessage(const string& msg, int line) } // namespace redistribute -// vim:ts=4 sw=4: diff --git a/writeengine/redistribute/we_redistributeworkerthread.h b/writeengine/redistribute/we_redistributeworkerthread.h index c989cd286..9dc5fa417 100644 --- a/writeengine/redistribute/we_redistributeworkerthread.h +++ b/writeengine/redistribute/we_redistributeworkerthread.h @@ -141,4 +141,3 @@ class RedistributeWorkerThread } // namespace redistribute -// vim:ts=4 sw=4: diff --git a/writeengine/server/we_dataloader.cpp b/writeengine/server/we_dataloader.cpp index 18a616799..1e9984c8d 100644 --- a/writeengine/server/we_dataloader.cpp +++ b/writeengine/server/we_dataloader.cpp @@ -891,7 +891,7 @@ void WEDataLoader::onReceiveData(ByteStream& Ibs) if (aQsz < MAX_QSIZE) sendDataRequest(); - if (aQsz > 1.5 * MAX_QSIZE) // > 2*250 + if (aQsz > 1.5 * static_cast(MAX_QSIZE)) // > 2*250 { cout << "WARNING : Data Queuing up : QSize = " << aQsz << endl; diff --git a/writeengine/server/we_ddlcommandproc.cpp b/writeengine/server/we_ddlcommandproc.cpp index 69373fff6..5ad015aa7 100644 --- a/writeengine/server/we_ddlcommandproc.cpp +++ b/writeengine/server/we_ddlcommandproc.cpp @@ -241,6 +241,7 @@ uint8_t WE_DDLCommandProc::writeSystable(ByteStream& bs, std::string& err) if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.fCharsetNumber = column.colType.charsetNumber; dctnryStruct.columnOid = column.oid; } else @@ -486,7 +487,7 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err (dataType != CalpontSystemCatalog::TEXT)) { ostringstream os; - os << "char, varchar and varbinary length may not exceed 8000"; + os << "char, varchar and varbinary length may not exceed 8000 bytes"; throw std::runtime_error(os.str()); } } @@ -656,9 +657,10 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err dctnryStruct.fCompressionType = 2; } - if (colStruct.tokenFlag) + if (colStruct.tokenFlag) // TODO: XXX: this is copied aplenty. NEED TO REFACTOR. { dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.fCharsetNumber = column.colType.charsetNumber; dctnryStruct.columnOid = column.oid; } else @@ -870,7 +872,7 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string& err) (dataType != CalpontSystemCatalog::TEXT)) { ostringstream os; - os << "char, varchar and varbinary length may not exceed 8000"; + os << "char, varchar and varbinary length may not exceed 8000 bytes"; throw std::runtime_error(os.str()); } } @@ -1046,6 +1048,7 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string& err) if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.fCharsetNumber = column.colType.charsetNumber; dctnryStruct.columnOid = column.oid; } else @@ -2442,6 +2445,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnTablename(ByteStream& bs, std::string& if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.fCharsetNumber = column.colType.charsetNumber; dctnryStruct.columnOid = colStruct.dataOid; } else @@ -2846,6 +2850,7 @@ uint8_t WE_DDLCommandProc::updateSystableTablename(ByteStream& bs, std::string& if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.fCharsetNumber = column.colType.charsetNumber; dctnryStruct.columnOid = colStruct.dataOid; } else @@ -3087,6 +3092,7 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string& if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.fCharsetNumber = column.colType.charsetNumber; dctnryStruct.columnOid = colStruct.dataOid; } else @@ -3273,6 +3279,7 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string& if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.fCharsetNumber = column.colType.charsetNumber; dctnryStruct.columnOid = colStruct.dataOid; } else @@ -3552,7 +3559,7 @@ uint8_t WE_DDLCommandProc::fillNewColumn(ByteStream& bs, std::string& err) int dataWidth, scale, precision, compressionType, refColWidth, refCompressionType; string defaultValStr; ColTuple defaultVal; - string timeZone; + long timeZone; bs >> tmp32; txnID = tmp32; @@ -3581,7 +3588,9 @@ uint8_t WE_DDLCommandProc::fillNewColumn(ByteStream& bs, std::string& err) refColWidth = tmp32; bs >> tmp8; refCompressionType = tmp8; - bs >> timeZone; + messageqcpp::ByteStream::octbyte timeZoneTemp; + bs >> timeZoneTemp; + timeZone = timeZoneTemp; // Find the fill in value bool isNULL = false; @@ -4256,6 +4265,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnSetDefault(messageqcpp::ByteStream& bs if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.fCharsetNumber = column.colType.charsetNumber; dctnryStruct.columnOid = colStruct.dataOid; } else @@ -4543,6 +4553,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream& if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = column1.colType.ddn.dictOID; + dctnryStruct.fCharsetNumber = column1.colType.charsetNumber; dctnryStruct.columnOid = colStruct.dataOid; } else @@ -4754,6 +4765,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream& if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = column5.colType.ddn.dictOID; + dctnryStruct.fCharsetNumber = column5.colType.charsetNumber; dctnryStruct.columnOid = colStruct.dataOid; } else @@ -4950,4 +4962,3 @@ void WE_DDLCommandProc::purgeFDCache() } } // namespace WriteEngine -// vim:ts=4 sw=4: diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index d183e3b0f..e44844c1c 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -181,7 +181,9 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: colStruct.colDataType = colType.colDataType; - if (colStruct.tokenFlag) + dctnryStruct.fCharsetNumber = colType.charsetNumber; + + if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = colType.ddn.dictOID; dctnryStruct.columnOid = colStruct.dataOid; @@ -1037,6 +1039,8 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std:: colStruct.colDataType = colType.colDataType; + dctnryStruct.fCharsetNumber = colType.charsetNumber; + if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = colType.ddn.dictOID; @@ -1619,6 +1623,8 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, colStruct.colDataType = colType.colDataType; + dctnryStruct.fCharsetNumber = colType.charsetNumber; + if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = colType.ddn.dictOID; @@ -2681,7 +2687,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin CalpontSystemCatalog::OID oid = 0; CalpontSystemCatalog::ROPair tableRO; - std::string timeZone = cpackages[txnId].get_TimeZone(); + long timeZone = cpackages[txnId].get_TimeZone(); try { @@ -2830,6 +2836,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin dctnryStruct.dctnryOid = colType.ddn.dictOID; dctnryStruct.columnOid = colStruct.dataOid; dctnryStruct.fCompressionType = colType.compressionType; + dctnryStruct.fCharsetNumber = colType.charsetNumber; dctnryStruct.colWidth = colType.colWidth; if (NO_ERROR != (error = fWEWrapper.openDctnry(txnId, dctnryStruct, false))) // @bug 5572 HDFS tmp file @@ -4445,6 +4452,8 @@ uint8_t WE_DMLCommandProc::processFixRows(messageqcpp::ByteStream& bs, std::stri dctnryStruct.fCompressionType = colStruct.fCompressionType; dctnryStruct.dctnryOid = 0; + dctnryStruct.fCharsetNumber = colType.charsetNumber; + if (colType.colWidth > 8) // token { colStruct.colWidth = 8; diff --git a/writeengine/server/we_server.cpp b/writeengine/server/we_server.cpp index b7c9fc194..1e0497319 100644 --- a/writeengine/server/we_server.cpp +++ b/writeengine/server/we_server.cpp @@ -149,11 +149,10 @@ int ServiceWriteEngine::setupResources() return -3; } - if (rlim.rlim_cur != 65536) + if (rlim.rlim_cur < 65536) { return -4; } - #endif return 0; } diff --git a/writeengine/shared/we_brm.cpp b/writeengine/shared/we_brm.cpp index a049806b1..fb5d88fac 100644 --- a/writeengine/shared/we_brm.cpp +++ b/writeengine/shared/we_brm.cpp @@ -1741,4 +1741,3 @@ int BRMWrapper::getExtentCPMaxMin(const BRM::LBID_t lbid, BRM::CPMaxMin& cpMaxMi } } // namespace WriteEngine -// vim:ts=4 sw=4: diff --git a/writeengine/shared/we_brm.h b/writeengine/shared/we_brm.h index cb5916a88..27a7e2a93 100644 --- a/writeengine/shared/we_brm.h +++ b/writeengine/shared/we_brm.h @@ -24,6 +24,7 @@ #pragma once #include +#include #include #include #include @@ -49,17 +50,19 @@ namespace WriteEngine // forward reference class DbFileOp; -/** @brief Extended CPInfo - with type handler for all type-related information */ +/** @brief Extended CPInfo - with all type-related information and associated range data */ struct ExtCPInfo { execplan::CalpontSystemCatalog::ColDataType fColType; int fColWidth; BRM::CPInfo fCPInfo; + std::shared_ptr> fStringsPrefixes; ExtCPInfo(execplan::CalpontSystemCatalog::ColDataType colType, int colWidth) : fColType(colType), fColWidth(colWidth) { fCPInfo.isBinaryColumn = (unsigned int)colWidth > datatypes::MAXLEGACYWIDTH; } + void toInvalid() { auto mm = datatypes::MinMaxInfo::invalidRange(fColType); @@ -68,7 +71,22 @@ struct ExtCPInfo fCPInfo.bigMax = mm.int128Max; fCPInfo.bigMin = mm.int128Min; } - + void addStringPrefix(int64_t strPrefix) + { + if (!fStringsPrefixes) + { + fStringsPrefixes.reset(new std::vector()); + } + fStringsPrefixes->push_back(strPrefix); + } + bool hasStringsPrefixes() const + { + return fStringsPrefixes.get() != nullptr; + } + int64_t* stringsPrefixes() const + { + return hasStringsPrefixes() ? fStringsPrefixes->data() : nullptr; + } bool isInvalid() { datatypes::MinMaxInfo mm; diff --git a/writeengine/shared/we_chunkmanager.cpp b/writeengine/shared/we_chunkmanager.cpp index d2ab6d324..1254176ef 100644 --- a/writeengine/shared/we_chunkmanager.cpp +++ b/writeengine/shared/we_chunkmanager.cpp @@ -2640,4 +2640,3 @@ int ChunkManager::checkFixLastDictChunk(const FID& fid, uint16_t root, uint32_t } } // namespace WriteEngine -// vim:ts=4 sw=4: diff --git a/writeengine/shared/we_type.h b/writeengine/shared/we_type.h index e54f03b59..7bd11f892 100644 --- a/writeengine/shared/we_type.h +++ b/writeengine/shared/we_type.h @@ -344,6 +344,7 @@ struct DctnryStruct /** @brief Dctnry Interface Struct*/ uint16_t fColSegment; /** @brief Segment for column file */ uint16_t fColDbRoot; /** @brief DBRoot for column file */ int fCompressionType; /** @brief Compression tpye for column file */ + int fCharsetNumber; /** @brief Charset number to account for collation when computing string prefixes */ DctnryStruct() : dctnryOid(0) , columnOid(0) @@ -353,6 +354,7 @@ struct DctnryStruct /** @brief Dctnry Interface Struct*/ , fColSegment(0) , fColDbRoot(0) , fCompressionType(idbdatafile::IDBPolicy::useHdfs() ? 2 : 0) + , fCharsetNumber(8) { } }; diff --git a/writeengine/splitter/we_brmupdater.cpp b/writeengine/splitter/we_brmupdater.cpp index f6ea954e8..a7be2c0df 100644 --- a/writeengine/splitter/we_brmupdater.cpp +++ b/writeengine/splitter/we_brmupdater.cpp @@ -214,6 +214,7 @@ int WEBrmUpdater::updateHighWaterMarkInBRM() int WEBrmUpdater::updateCPAndHWMInBRM() { int rc = 0; + size_t i; // BUG 4232. some imports may not contain CP but HWM if ((fCPInfo.size() > 0) || (fHWMInfo.size() > 0)) @@ -227,6 +228,13 @@ int WEBrmUpdater::updateCPAndHWMInBRM() const std::vector & mergeCPDataArgs, VER_t transID = 0) DBRM_THROW; */ + for (i = 0; i < fCPInfo.size(); i++) + { + if (fCPInfo[i].newExtent) + { + fCPInfo[i].seqNum = 0; // to be in sync with DBRM. + } + } rc = fpBrm->bulkSetHWMAndCP(fHWMInfo, fCPInfoData, fCPInfo, 0); // rc = fpBrm->mergeExtentsMaxMin(fCPInfo); diff --git a/writeengine/splitter/we_cmdargs.cpp b/writeengine/splitter/we_cmdargs.cpp index 27efb1d37..9ad76a451 100644 --- a/writeengine/splitter/we_cmdargs.cpp +++ b/writeengine/splitter/we_cmdargs.cpp @@ -561,11 +561,11 @@ void WECmdArgs::usage() << "\t-T\tTimezone used for TIMESTAMP datatype.\n" << "\t\tPossible values: \"SYSTEM\" (default)\n" << "\t\t : Offset in the form +/-HH:MM\n" - << "\t-y\tS3 Authentication Key (for S3 imports)\n" - << "\t-K\tS3 Authentication Secret (for S3 imports)\n" - << "\t-t\tS3 Bucket (for S3 imports)\n" - << "\t-H\tS3 Hostname (for S3 imports, Amazon's S3 default)\n" - << "\t-g\tS3 Region (for S3 imports)\n" +// << "\t-y\tS3 Authentication Key (for S3 imports)\n" +// << "\t-K\tS3 Authentication Secret (for S3 imports)\n" +// << "\t-t\tS3 Bucket (for S3 imports)\n" +// << "\t-H\tS3 Hostname (for S3 imports, Amazon's S3 default)\n" +// << "\t-g\tS3 Region (for S3 imports)\n" << "\t-L\tDirectory for the output .err and .bad files.\n" << "\t\tDefault is " << string(MCSLOGDIR); @@ -598,7 +598,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv) if (argc > 0) fPrgmName = string(MCSBINDIR) + "/" + "cpimport.bin"; // argv[0] is splitter but we need cpimport - while ((aCh = getopt(argc, argv, "d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:ST:Ny:K:t:H:g:U:L:")) != EOF) + while ((aCh = getopt(argc, argv, "d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:ST:NU:L:")) != EOF) { switch (aCh) { @@ -906,7 +906,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv) fConsoleOutput = false; break; } - +/* case 'y': //-y S3 Key { fS3Key = optarg; @@ -936,7 +936,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv) fS3Region = optarg; break; } - +*/ case 'U': //-U username of the files owner { fUsername = optarg; diff --git a/writeengine/splitter/we_filereadthread.cpp b/writeengine/splitter/we_filereadthread.cpp index 4faab3fa6..4b64ad3fb 100644 --- a/writeengine/splitter/we_filereadthread.cpp +++ b/writeengine/splitter/we_filereadthread.cpp @@ -481,6 +481,15 @@ void WEFileReadThread::openInFile() use ms3 lib to d/l data into mem use boost::iostreams to wrap the mem in a stream interface point infile's stream buffer to it. + + MCOL-4576: The options to setup S3 with cpimport have been removed and this + code is unreachable. However we may need to resurrect it at some point in some form. + Performance issues with extremely large data files as well as the fact files larger + than system memory will cause an OOM error. Multipart downloads/uploads need to be + implemented or more likely a different streaming solution developed with external API tools + + MCOL-4576 work around is to use 3rd party CLI tools and pipe data file from S3 bucket + into cpimport stdin. 3rd party tooling for large object downloads will be more efficient. */ if (fSdh.getDebugLvl()) @@ -580,7 +589,7 @@ int WEFileReadThread::getNextRow(istream& ifs, char* pBuf, int MaxLen) const char ESC = fEsc; bool aTrailEsc = false; char* pEnd = pBuf; - char aCh = ifs.get(); + int aCh = ifs.get(); while (ifs.good()) { diff --git a/writeengine/splitter/we_splitterapp.cpp b/writeengine/splitter/we_splitterapp.cpp index 366c64ad7..22a29100a 100644 --- a/writeengine/splitter/we_splitterapp.cpp +++ b/writeengine/splitter/we_splitterapp.cpp @@ -572,4 +572,3 @@ int main(int argc, char** argv) return SPLTR_EXIT_STATUS; } -// vim:ts=4 sw=4: diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index 25bed1ebe..bbe3dc42a 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -1910,4 +1910,3 @@ int ColumnOp::writeRowsValues(Column& curCol, uint64_t totalRow, const RIDList& } } // namespace WriteEngine -// vim:ts=4 sw=4: diff --git a/writeengine/wrapper/we_tablemetadata.cpp b/writeengine/wrapper/we_tablemetadata.cpp index 64bfff06d..421ecbcb8 100644 --- a/writeengine/wrapper/we_tablemetadata.cpp +++ b/writeengine/wrapper/we_tablemetadata.cpp @@ -109,4 +109,3 @@ ColsExtsInfoMap& TableMetaData::getColsExtsInfoMap() return fColsExtsInfoMap; } } // namespace WriteEngine -// vim:ts=4 sw=4: diff --git a/writeengine/wrapper/we_tablemetadata.h b/writeengine/wrapper/we_tablemetadata.h index 1202d6afd..4a753c960 100644 --- a/writeengine/wrapper/we_tablemetadata.h +++ b/writeengine/wrapper/we_tablemetadata.h @@ -98,4 +98,3 @@ class TableMetaData #undef EXPORT -// vim:ts=4 sw=4: diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 815204120..9e87e8be5 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -21,6 +21,10 @@ /** @writeengine.cpp * A wrapper class for the write engine to write information to files */ + +// XXX: a definition to switch off computations for token columns. +//#define XXX_WRITEENGINE_TOKENS_RANGES_XXX + #include #include #include @@ -59,6 +63,7 @@ using namespace execplan; #include "MonitorProcMem.h" using namespace idbdatafile; #include "dataconvert.h" +#include "string_prefixes.h" #ifdef _MSC_VER #define isnan _isnan @@ -362,6 +367,9 @@ void WriteEngineWrapper::updateMaxMinRange(const size_t totalNewRow, const size_ case WR_UINT: case WR_ULONGLONG: case WR_CHAR: +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + case WR_TOKEN: +#endif { isUnsigned = true; break; @@ -385,6 +393,13 @@ void WriteEngineWrapper::updateMaxMinRange(const size_t totalNewRow, const size_ maxMin->fromToChars(); } } +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + if (colType == WR_TOKEN) + { + oldValArrayVoid = nullptr; // no old values for tokens, sadly. + valArrayVoid = (void*)maxMin->stringsPrefixes(); + } +#endif size_t i; for (i = 0; i < totalOldRow; i++) { @@ -435,6 +450,9 @@ void WriteEngineWrapper::updateMaxMinRange(const size_t totalNewRow, const size_ fetchNewOldValues(value, oldValue, valArrayVoid, oldValArrayVoid, i, totalNewRow); break; } +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + case WR_TOKEN: +#endif case WR_ULONGLONG: { fetchNewOldValues(uvalue, oldUValue, valArrayVoid, oldValArrayVoid, i, @@ -449,12 +467,11 @@ void WriteEngineWrapper::updateMaxMinRange(const size_t totalNewRow, const size_ } case WR_CHAR: { - fetchNewOldValues(uvalue, oldUValue, valArrayVoid, oldValArrayVoid, i, - totalNewRow); + fetchNewOldValues(value, oldValue, valArrayVoid, oldValArrayVoid, i, totalNewRow); // for characters (strings, actually), we fetched then in LSB order, on x86, at the very least. // this means most significant byte of the string, which is first, is now in LSB of uvalue/oldValue. // we must perform a conversion. - uvalue = uint64ToStr(uvalue); + value = uint64ToStr(uvalue); oldValue = uint64ToStr(oldValue); break; } @@ -576,6 +593,7 @@ void WriteEngineWrapper::convertValue(const execplan::CalpontSystemCatalog::ColT curStr = curStr.substr(0, MAX_COLUMN_BOUNDARY); memcpy(value, curStr.c_str(), curStr.length()); + break; case WriteEngine::WR_FLOAT: @@ -1179,10 +1197,17 @@ static void log_this(const char *message, #endif /** @brief Determine whether we may update a column's ranges (by type) and return nullptr if we can't */ -static ExtCPInfo* getCPInfoToUpdateForUpdatableType(const ColStruct& colStruct, ExtCPInfo* currentCPInfo) +static ExtCPInfo* getCPInfoToUpdateForUpdatableType(const ColStruct& colStruct, ExtCPInfo* currentCPInfo, + OpType optype) { if (colStruct.tokenFlag) { +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + if (currentCPInfo && currentCPInfo->hasStringsPrefixes() && optype == INSERT) + { + return currentCPInfo; + } +#endif return nullptr; } switch (colStruct.colType) @@ -1689,10 +1714,16 @@ int WriteEngineWrapper::insertColumnRecs( for (uint32_t rows = 0; rows < (totalRow - rowsLeft); rows++) { +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + int64_t strPrefix; +#endif if (dctStr_iter->length() == 0) { Token nullToken; col_iter->data = nullToken; +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + strPrefix = (int64_t)joblist::UBIGINTNULL; // the string prefixes are signed long ints. +#endif } else { @@ -1702,6 +1733,10 @@ int WriteEngineWrapper::insertColumnRecs( DctnryTuple dctTuple; dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str(); dctTuple.sigSize = dctStr_iter->length(); +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + strPrefix = encodeStringPrefix_check_null(dctTuple.sigValue, dctTuple.sigSize, + dctnryStructList[i].fCharsetNumber); +#endif dctTuple.isNull = false; rc = tokenize(txnid, dctTuple, dctnryStructList[i].fCompressionType); @@ -1717,6 +1752,9 @@ int WriteEngineWrapper::insertColumnRecs( col_iter->data = dctTuple.token; } +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + maxMins[i].fSplitMaxMinInfo[0].addStringPrefix(strPrefix); +#endif dctStr_iter++; col_iter++; } @@ -1744,10 +1782,16 @@ int WriteEngineWrapper::insertColumnRecs( for (uint32_t rows = 0; rows < rowsLeft; rows++) { +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + int64_t strPrefix; +#endif if (dctStr_iter->length() == 0) { Token nullToken; col_iter->data = nullToken; +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + strPrefix = joblist::UBIGINTNULL; // string prefixes are signed long ints. +#endif } else { @@ -1757,6 +1801,10 @@ int WriteEngineWrapper::insertColumnRecs( DctnryTuple dctTuple; dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str(); dctTuple.sigSize = dctStr_iter->length(); +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + strPrefix = encodeStringPrefix_check_null(dctTuple.sigValue, dctTuple.sigSize, + dctnryStructList[i].fCharsetNumber); +#endif dctTuple.isNull = false; rc = tokenize(txnid, dctTuple, newDctnryStructList[i].fCompressionType); @@ -1772,6 +1820,9 @@ int WriteEngineWrapper::insertColumnRecs( col_iter->data = dctTuple.token; } +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + maxMins[i].fSplitMaxMinInfo[1].addStringPrefix(strPrefix); +#endif dctStr_iter++; col_iter++; } @@ -1938,7 +1989,7 @@ int WriteEngineWrapper::insertColumnRecs( if (isFirstBatchPm && (totalRow == rowsLeft)) { - // in this particular case we already marked extents as invalid up there. + // in this particular case we already marked extents as invalid above. } else { @@ -1950,7 +2001,7 @@ int WriteEngineWrapper::insertColumnRecs( if (firstHalfCount) { ExtCPInfo* cpInfoP = - getCPInfoToUpdateForUpdatableType(colStructList[i], &maxMins[i].fSplitMaxMinInfo[0]); + getCPInfoToUpdateForUpdatableType(colStructList[i], &maxMins[i].fSplitMaxMinInfo[0], m_opType); RID thisRid = rowsLeft ? lastRid : lastRidNew; successFlag = colOp->calculateRowId(thisRid, BYTE_PER_BLOCK / width, width, curFbo, curBio); @@ -1966,7 +2017,7 @@ int WriteEngineWrapper::insertColumnRecs( if (rowsLeft) { ExtCPInfo* cpInfoP = - getCPInfoToUpdateForUpdatableType(colStructList[i], &maxMins[i].fSplitMaxMinInfo[1]); + getCPInfoToUpdateForUpdatableType(colStructList[i], &maxMins[i].fSplitMaxMinInfo[1], m_opType); if (cpInfoP) { RETURN_ON_ERROR(GetLBIDRange(newExtentsStartingLbids[i], colStructList[i], *cpInfoP)); @@ -4446,11 +4497,6 @@ int WriteEngineWrapper::updateColumnRec(const TxnID& txnid, const vector currentExtentRangesPtrs(colStructList.size(), NULL); // pointers for each extent. + if (m_opType != DELETE) + m_opType = UPDATE; + for (unsigned j = 0; j < colStructList.size(); j++) { colOp = m_colOp[op(colStructList[j].fCompressionType)]; ExtCPInfo* cpInfoP = &(currentExtentRanges[j]); - cpInfoP = getCPInfoToUpdateForUpdatableType(colStructList[j], cpInfoP); + cpInfoP = getCPInfoToUpdateForUpdatableType(colStructList[j], cpInfoP, m_opType); currentExtentRangesPtrs[j] = cpInfoP; - if (colStructList[j].tokenFlag) - continue; + // XXX: highly dubious. + // if (!colStructList[j].tokenFlag) + // continue; width = colOp->getCorrectRowWidth(colStructList[j].colDataType, colStructList[j].colWidth); successFlag = colOp->calculateRowId(aRid, BYTE_PER_BLOCK / width, width, curFbo, curBio); @@ -4550,9 +4600,6 @@ int WriteEngineWrapper::updateColumnRec(const TxnID& txnid, const vectorsetExtentsMaxMin(infosToDrop); setInvalidCPInfosSpecialMarks(infosToUpdate); rc = BRMWrapper::getInstance()->setExtentsMaxMin(infosToUpdate); @@ -4611,12 +4659,9 @@ int WriteEngineWrapper::updateColumnRecs(const TxnID& txnid, const CSCTypesList& colOp = m_colOp[op(colExtentsStruct[j].fCompressionType)]; ExtCPInfo* cpInfoP = &(infosToUpdate[j]); - cpInfoP = getCPInfoToUpdateForUpdatableType(colExtentsStruct[j], cpInfoP); + cpInfoP = getCPInfoToUpdateForUpdatableType(colExtentsStruct[j], cpInfoP, m_opType); pointersToInfos.push_back(cpInfoP); - if (colExtentsStruct[j].tokenFlag) - continue; - width = colOp->getCorrectRowWidth(colExtentsStruct[j].colDataType, colExtentsStruct[j].colWidth); successFlag = colOp->calculateRowId(aRid, BYTE_PER_BLOCK / width, width, curFbo, curBio); @@ -4964,7 +5009,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, const CSCTypesList& c allocateValArray(valArray, totalRow1, colStructList[i].colType, colStructList[i].colWidth); ExtCPInfo* cpInfo = getCPInfoToUpdateForUpdatableType( - colStructList[i], maxMins ? ((*maxMins)[i]).fSplitMaxMinInfoPtrs[0] : NULL); + colStructList[i], maxMins ? ((*maxMins)[i]).fSplitMaxMinInfoPtrs[0] : NULL, m_opType); if (m_opType != INSERT && cpInfo != NULL) // we allocate space for old values only when we need them. { @@ -5109,7 +5154,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, const CSCTypesList& c } ExtCPInfo* cpInfo = getCPInfoToUpdateForUpdatableType( - newColStructList[i], maxMins ? ((*maxMins)[i]).fSplitMaxMinInfoPtrs[1] : NULL); + newColStructList[i], maxMins ? ((*maxMins)[i]).fSplitMaxMinInfoPtrs[1] : NULL, m_opType); allocateValArray(valArray, totalRow2, newColStructList[i].colType, newColStructList[i].colWidth); if (m_opType != INSERT && cpInfo != NULL) // we allocate space for old values only when we need them. @@ -5190,7 +5235,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, const CSCTypesList& c ColumnOp* colOp = m_colOp[op(colStructList[i].fCompressionType)]; ExtCPInfo* cpInfo = getCPInfoToUpdateForUpdatableType( - colStructList[i], maxMins ? ((*maxMins)[i]).fSplitMaxMinInfoPtrs[0] : NULL); + colStructList[i], maxMins ? ((*maxMins)[i]).fSplitMaxMinInfoPtrs[0] : NULL, m_opType); // set params colOp->initColumn(curCol); @@ -6628,4 +6673,3 @@ int WriteEngineWrapper::RemoveTxnFromLBIDMap(const TxnID txnid) } } // end of namespace -// vim:ts=4 sw=4: diff --git a/writeengine/xml/we_xmlgenproc.cpp b/writeengine/xml/we_xmlgenproc.cpp index 3fca53e50..afd6d2b6c 100644 --- a/writeengine/xml/we_xmlgenproc.cpp +++ b/writeengine/xml/we_xmlgenproc.cpp @@ -168,7 +168,21 @@ void XMLGenProc::startXMLFile() // makeTableData // Create XML tag for a table. //------------------------------------------------------------------------------ +// This method is used by colxml only and it can be relatively slower doing tableRID() +// first call. All subsequent calls will re-use data from CalpontSystemCatalog cache. void XMLGenProc::makeTableData(const CalpontSystemCatalog::TableName& table) +{ + boost::shared_ptr cat = + CalpontSystemCatalog::makeCalpontSystemCatalog(BULK_SYSCAT_SESSION_ID); + cat->identity(CalpontSystemCatalog::EC); + std::ostringstream oss; + // tableRID method might take a lot with a significant EM. + oss << cat->tableRID(table).objnum; + + makeTableData(table, oss.str()); +} + +void XMLGenProc::makeTableData(const CalpontSystemCatalog::TableName& table, const std::string& tableOIDStr) { static unsigned kount; @@ -180,11 +194,8 @@ void XMLGenProc::makeTableData(const CalpontSystemCatalog::TableName& table) { try { - boost::shared_ptr cat = - CalpontSystemCatalog::makeCalpontSystemCatalog(BULK_SYSCAT_SESSION_ID); - cat->identity(CalpontSystemCatalog::EC); - xmlTextWriterWriteFormatAttribute(fWriter, BAD_CAST xmlTagTable[TAG_TBL_OID], "%d", - cat->tableRID(table).objnum); + xmlTextWriterWriteFormatAttribute(fWriter, BAD_CAST xmlTagTable[TAG_TBL_OID], "%s", + tableOIDStr.c_str()); } catch (std::exception& ex) { diff --git a/writeengine/xml/we_xmlgenproc.h b/writeengine/xml/we_xmlgenproc.h index 2b880329e..2da7fc925 100644 --- a/writeengine/xml/we_xmlgenproc.h +++ b/writeengine/xml/we_xmlgenproc.h @@ -75,6 +75,9 @@ class XMLGenProc * * @param table Name of table for which the table tag is to be generated. */ + EXPORT void makeTableData(const execplan::CalpontSystemCatalog::TableName& table, + const std::string& tableOIDStr); + EXPORT void makeTableData(const execplan::CalpontSystemCatalog::TableName& table); /** @brief Creates column tags for the specified table. diff --git a/writeengine/xml/we_xmljob.cpp b/writeengine/xml/we_xmljob.cpp index 0cd44b458..2a95203df 100644 --- a/writeengine/xml/we_xmljob.cpp +++ b/writeengine/xml/we_xmljob.cpp @@ -71,7 +71,11 @@ const long long columnstore_precision[19] = {0, //------------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------------ -XMLJob::XMLJob() : fDebugLevel(DEBUG_0), fDeleteTempFile(false), fValidateColList(true), fTimeZone("SYSTEM") +XMLJob::XMLJob() + : fDebugLevel(DEBUG_0) + , fDeleteTempFile(false) + , fValidateColList(true) + , fTimeZone(dataconvert::systemTimeZoneOffset()) { } @@ -377,11 +381,11 @@ void XMLJob::setJobData(xmlNode* pNode, const xmlTag tag, bool bExpectContent, X if (tagType == TYPE_INT) bSuccess = getNodeContent(pNode, &intVal, TYPE_INT); else // longlong - if (tagType == TYPE_LONGLONG) - bSuccess = getNodeContent(pNode, &llVal, TYPE_LONGLONG); - else // char + if (tagType == TYPE_LONGLONG) + bSuccess = getNodeContent(pNode, &llVal, TYPE_LONGLONG); + else // char if (tagType == TYPE_CHAR) - bSuccess = getNodeContentStr(pNode, bufString); + bSuccess = getNodeContentStr(pNode, bufString); if (!bSuccess) return; @@ -1190,7 +1194,8 @@ void XMLJob::validateAllColumnsHaveTags(const execplan::CalpontSystemCatalog::RI /* static */ int XMLJob::genJobXMLFileName(const string& sXMLJobDir, const string& jobDir, const string& jobId, bool bTempFile, const string& schemaName, const string& tableName, - boost::filesystem::path& xmlFilePath, string& errMsg, std::string& tableOIDStr) + boost::filesystem::path& xmlFilePath, string& errMsg, + const std::string& tableOIDStr) { // get full file directory path for XML job description file if (sXMLJobDir.empty()) diff --git a/writeengine/xml/we_xmljob.h b/writeengine/xml/we_xmljob.h index 61aae0f94..0910bea0b 100644 --- a/writeengine/xml/we_xmljob.h +++ b/writeengine/xml/we_xmljob.h @@ -74,7 +74,7 @@ class XMLJob : public XMLOp EXPORT static int genJobXMLFileName(const std::string& sXMLJobDir, const std::string& jobDir, const std::string& jobId, bool bTempFile, const std::string& schemaName, const std::string& tableName, boost::filesystem::path& xmlDirPath, - std::string& errMsg, std::string& tableOIDStr); + std::string& errMsg, const std::string& tableOIDStr); /** * @brief Get job structure @@ -116,7 +116,7 @@ class XMLJob : public XMLOp /** * @brief Set timezone */ - void setTimeZone(const std::string& timeZone) + void setTimeZone(long timeZone) { fTimeZone = timeZone; } @@ -144,7 +144,9 @@ class XMLJob : public XMLOp JobColList fDefaultColumns; // temporary list of default cols // for table node being processed bool fValidateColList; // Validate all cols have XML tag - std::string fTimeZone; // Timezone used for TIMESTAMP datatype + long fTimeZone; // Timezone offset (in seconds) relative to UTC, + // to use for TIMESTAMP data type. For example, + // for EST which is UTC-5:00, offset will be -18000s. }; } // namespace WriteEngine