From 5dfcb6175b59479419e33e6271916bb1ce36538a Mon Sep 17 00:00:00 2001 From: Wez Furlong Date: Fri, 3 May 2019 15:52:39 -0700 Subject: [PATCH] fbcode_builder: getdeps: add sqlite3 builder Summary: Previously, we were using autoconf to build sqlite, but that isn't available on Windows. Instead, here's a builder that generates a little cmake configuration for building and installing sqlite. Using cmake for this means that we can test the same builder on all platforms that need to pull in sqlite. Reviewed By: simpkins Differential Revision: D15179387 fbshipit-source-id: fccf969182565982bd5be55545a2d2625aa99124 --- build/fbcode_builder/getdeps/builder.py | 142 +++++++++++++++++------ build/fbcode_builder/getdeps/manifest.py | 4 + build/fbcode_builder/manifests/sqlite3 | 12 +- 3 files changed, 117 insertions(+), 41 deletions(-) diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index 8cb5d7cbe..f835063ca 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -17,6 +17,7 @@ import subprocess import sys from .envfuncs import Env, add_path_entry, path_search +from .fetcher import copy_if_different from .runcmd import run_cmd @@ -99,6 +100,43 @@ class BuilderBase(object): system needs to regenerate its rules. """ pass + def _compute_env(self, install_dirs): + # CMAKE_PREFIX_PATH is only respected when passed through the + # environment, so we construct an appropriate path to pass down + env = self.env.copy() + + lib_path = None + if self.build_opts.is_darwin(): + lib_path = "DYLD_LIBRARY_PATH" + elif self.build_opts.is_linux(): + lib_path = "LD_LIBRARY_PATH" + else: + lib_path = None + + for d in install_dirs: + add_path_entry(env, "CMAKE_PREFIX_PATH", d) + + pkgconfig = os.path.join(d, "lib/pkgconfig") + if os.path.exists(pkgconfig): + add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig) + + # Allow resolving shared objects built earlier (eg: zstd + # doesn't include the full path to the dylib in its linkage + # so we need to give it an assist) + if lib_path: + for lib in ["lib", "lib64"]: + libdir = os.path.join(d, lib) + if os.path.exists(libdir): + add_path_entry(env, lib_path, libdir) + + # Allow resolving binaries (eg: cmake, ninja) and dlls + # built by earlier steps + bindir = os.path.join(d, "bin") + if os.path.exists(bindir): + add_path_entry(env, "PATH", bindir, append=False) + + return env + class MakeBuilder(BuilderBase): def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, args): @@ -178,43 +216,6 @@ class CMakeBuilder(BuilderBase): return True return False - def _compute_env(self, install_dirs): - # CMAKE_PREFIX_PATH is only respected when passed through the - # environment, so we construct an appropriate path to pass down - env = self.env.copy() - - lib_path = None - if self.build_opts.is_darwin(): - lib_path = "DYLD_LIBRARY_PATH" - elif self.build_opts.is_linux(): - lib_path = "LD_LIBRARY_PATH" - else: - lib_path = None - - for d in install_dirs: - add_path_entry(env, "CMAKE_PREFIX_PATH", d) - - pkgconfig = os.path.join(d, "lib/pkgconfig") - if os.path.exists(pkgconfig): - add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig) - - # Allow resolving shared objects built earlier (eg: zstd - # doesn't include the full path to the dylib in its linkage - # so we need to give it an assist) - if lib_path: - for lib in ["lib", "lib64"]: - libdir = os.path.join(d, lib) - if os.path.exists(libdir): - add_path_entry(env, lib_path, libdir) - - # Allow resolving binaries (eg: cmake, ninja) and dlls - # built by earlier steps - bindir = os.path.join(d, "bin") - if os.path.exists(bindir): - add_path_entry(env, "PATH", bindir, append=False) - - return env - def _build(self, install_dirs, reconfigure): reconfigure = reconfigure or self._needs_reconfigure() @@ -482,3 +483,70 @@ class NopBuilder(BuilderBase): else: if not os.path.exists(self.inst_dir): shutil.copytree(self.src_dir, self.inst_dir) + + +class SqliteBuilder(BuilderBase): + def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir): + super(SqliteBuilder, self).__init__( + build_opts, ctx, manifest, src_dir, build_dir, inst_dir + ) + + def _build(self, install_dirs, reconfigure): + for f in ["sqlite3.c", "sqlite3.h", "sqlite3ext.h"]: + src = os.path.join(self.src_dir, f) + dest = os.path.join(self.build_dir, f) + copy_if_different(src, dest) + + cmake_lists = """ +cmake_minimum_required(VERSION 3.1.3 FATAL_ERROR) +project(sqlite3 C) +add_library(sqlite3 STATIC sqlite3.c) +# These options are taken from the defaults in Makefile.msc in +# the sqlite distribution +target_compile_definitions(sqlite3 PRIVATE + -DSQLITE_ENABLE_COLUMN_METADATA=1 + -DSQLITE_ENABLE_FTS3=1 + -DSQLITE_ENABLE_RTREE=1 + -DSQLITE_ENABLE_GEOPOLY=1 + -DSQLITE_ENABLE_JSON1=1 + -DSQLITE_ENABLE_STMTVTAB=1 + -DSQLITE_ENABLE_DBPAGE_VTAB=1 + -DSQLITE_ENABLE_DBSTAT_VTAB=1 + -DSQLITE_INTROSPECTION_PRAGMAS=1 + -DSQLITE_ENABLE_DESERIALIZE=1 +) +install(TARGETS sqlite3) +install(FILES sqlite3.h sqlite3ext.h DESTINATION include) + """ + + with open(os.path.join(self.build_dir, "CMakeLists.txt"), "w") as f: + f.write(cmake_lists) + + defines = { + "CMAKE_INSTALL_PREFIX": self.inst_dir, + "BUILD_SHARED_LIBS": "OFF", + "CMAKE_BUILD_TYPE": "RelWithDebInfo", + } + define_args = ["-D%s=%s" % (k, v) for (k, v) in defines.items()] + define_args += ["-G", "Ninja"] + + env = self._compute_env(install_dirs) + + # Resolve the cmake that we installed + cmake = path_search(env, "cmake") + + self._run_cmd([cmake, self.build_dir] + define_args, env=env) + self._run_cmd( + [ + cmake, + "--build", + self.build_dir, + "--target", + "install", + "--config", + "Release", + "-j", + str(self.build_opts.num_jobs), + ], + env=env, + ) diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index 8247c0f3e..470fb65fc 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -18,6 +18,7 @@ from .builder import ( NinjaBootstrap, NopBuilder, OpenSSLBuilder, + SqliteBuilder, ) from .expr import parse_expr from .fetcher import ( @@ -376,6 +377,9 @@ class ManifestParser(object): build_options, ctx, self, src_dir, build_dir, inst_dir, defines ) + if builder == "sqlite": + return SqliteBuilder(build_options, ctx, self, src_dir, build_dir, inst_dir) + if builder == "ninja_bootstrap": return NinjaBootstrap( build_options, ctx, self, build_dir, src_dir, inst_dir diff --git a/build/fbcode_builder/manifests/sqlite3 b/build/fbcode_builder/manifests/sqlite3 index b03b17a5d..ef7e1a2f8 100644 --- a/build/fbcode_builder/manifests/sqlite3 +++ b/build/fbcode_builder/manifests/sqlite3 @@ -2,9 +2,13 @@ name = sqlite3 [download] -url = https://sqlite.org/2019/sqlite-autoconf-3270200.tar.gz -sha256 = 50c39e85ea28b5ecfdb3f9e860afe9ba606381e21836b2849efca6a0bfe6ef6e +url = https://sqlite.org/2019/sqlite-amalgamation-3280000.zip +sha256 = d02fc4e95cfef672b45052e221617a050b7f2e20103661cda88387349a9b1327 + +[dependencies] +cmake +ninja [build] -builder = autoconf -subdir = sqlite-autoconf-3270200 +builder = sqlite +subdir = sqlite-amalgamation-3280000