From 9155c1d72544916d6be1de689c848f6929d21ee8 Mon Sep 17 00:00:00 2001
From: Alexey Spiridonov <lesha@fb.com>
Date: Fri, 18 Aug 2017 12:12:49 -0700
Subject: [PATCH] Use ShipIt dependency hashes for a consistent build

Summary:
Previously, when building e.g. Proxygen, we would check out the master branch of dependencies like Folly. This could result in version shear, and the odds of it would go up as you iterated on your branch, while the dependencies moved forward. This degrades our developer experience.

The solution is to actually build from the same point in Facebook's monorepo.

As implemented, this is still imperfect, because the in-tree dependency hashes under `build/deps/github_hashes` are not currently updated atomically. However, on balance, this predictability should still be better -- and if I recall sdwilsh's explanation correctly, the atomicity of dependency commits could be fixed if it really impacts many people.

This will also improve cache coherence when you're using ccache for iteration, see D5648461.

(Also includes some extra "ideas" comments, and a whitespace fix.)

Reviewed By: sdwilsh

Differential Revision: D5648388

fbshipit-source-id: 1baa68ff3ef2e204a6cf5f5297e52514a3b62d21
---
 build/fbcode_builder/fbcode_builder.py | 44 ++++++++++++++++++++++++--
 1 file changed, 41 insertions(+), 3 deletions(-)

diff --git a/build/fbcode_builder/fbcode_builder.py b/build/fbcode_builder/fbcode_builder.py
index 928970eb8..373fa4926 100644
--- a/build/fbcode_builder/fbcode_builder.py
+++ b/build/fbcode_builder/fbcode_builder.py
@@ -33,6 +33,11 @@ Non-goals:
 
 Ideas for the future -- these may not be very good :)
 
+ - Especially on Ubuntu 14.04 the current initial setup is inefficient:
+   we add PPAs after having installed a bunch of packages -- this prompts
+   reinstalls of large amounts of code.  We also `apt-get update` a few
+   times.
+
  - A "shell script" builder. Like DockerFBCodeBuilder, but outputs a
    shell script that runs outside of a container. Or maybe even
    synchronously executes the shell commands, `make`-style.
@@ -40,13 +45,39 @@ Ideas for the future -- these may not be very good :)
  - A "Makefile" generator. That might make iterating on builds even quicker
    than what you can currently get with Docker build caching.
 
+ - Generate a rebuild script that can be run e.g. inside the built Docker
+   container by tagging certain steps with list-inheriting Python objects:
+     * do change directories
+     * do NOT `git clone` -- if we want to update code this should be a
+       separate script that e.g. runs rebase on top of specific targets
+       across all the repos.
+     * do NOT install software (most / all setup can be skipped)
+     * do NOT `autoreconf` or `configure`
+     * do `make` and `cmake`
+
+ - If we get non-Debian OSes, part of ccache setup should be factored out.
 '''
 
 import os
+import re
 
 from shell_quoting import path_join, shell_join, ShellQuoted
 
 
+def _read_project_github_hashes():
+    base_dir = 'deps/github_hashes/'  # trailing slash used in regex below
+    for dirname, _, files in os.walk(base_dir):
+        for filename in files:
+            path = os.path.join(dirname, filename)
+            with open(path) as f:
+                m_proj = re.match('^' + base_dir + '(.*)-rev\.txt$', path)
+                if m_proj is None:
+                    raise RuntimeError('Not a hash file? {0}'.format(path))
+                m_hash = re.match('^Subproject commit ([0-9a-f]+)\n$', f.read())
+                if m_hash is None:
+                    raise RuntimeError('No hash in {0}'.format(path))
+                yield m_proj.group(1), m_hash.group(1)
+
 class FBCodeBuilder(object):
 
     def __init__(self, **kwargs):
@@ -54,6 +85,7 @@ class FBCodeBuilder(object):
         # This raises upon detecting options that are specified but unused,
         # because otherwise it is very easy to make a typo in option names.
         self.options_used = set()
+        self._github_hashes = dict(_read_project_github_hashes())
 
     def __repr__(self):
         return '{0}({1})'.format(
@@ -214,7 +246,12 @@ class FBCodeBuilder(object):
     def github_project_workdir(self, project, path):
         # Only check out a non-default branch if requested. This especially
         # makes sense when building from a local repo.
-        git_hash = self.option('{0}:git_hash'.format(project), '')
+        git_hash = self.option(
+            '{0}:git_hash'.format(project),
+            # Any repo that has a hash in deps/github_hashes defaults to
+            # that, with the goal of making builds maximally consistent.
+            self._github_hashes.get(project, '')
+        )
         maybe_change_branch = [
             self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)),
         ] if git_hash else []
@@ -285,12 +322,13 @@ class FBCodeBuilder(object):
             self.run(ShellQuoted(
                 'CXXFLAGS="$CXXFLAGS -isystem "{p}"/include" '
                 'CFLAGS="$CFLAGS -isystem "{p}"/include" '
-                'cmake {args} ..').format(
+                'cmake {args} ..'
+            ).format(
                 p=self.option('prefix'),
                 args=shell_join(' ', (
                     ShellQuoted('-D{k}={v}').format(k=k, v=v)
                         for k, v in cmake_defines.items()
-                ))
+                )),
             )),
         ]