Refactor repository layout and convert build system to Mage.

This commit implements a massive refactor of the repository, and
moves the build system over to use Mage (magefile.org) which should
allow seamless building across multiple platforms.

vendor/github.com/magefile/mage/parse/import_go1.9.go

@@ -0,0 +1,13 @@
+// +build go1.9
+
+package parse
+
+import (
+	"go/importer"
+	"go/token"
+	"go/types"
+)
+
+func getImporter(*token.FileSet) types.Importer {
+	return importer.For("source", nil)
+}

vendor/github.com/magefile/mage/parse/import_not_go1.9.go

@@ -0,0 +1,15 @@
+// +build !go1.9
+
+package parse
+
+import (
+	"go/build"
+	"go/token"
+	"go/types"
+
+	"github.com/magefile/mage/parse/srcimporter"
+)
+
+func getImporter(fset *token.FileSet) types.Importer {
+	return srcimporter.New(&build.Default, fset, make(map[string]*types.Package))
+}

vendor/github.com/magefile/mage/parse/parse.go

@@ -0,0 +1,341 @@
+package parse
+
+import (
+	"fmt"
+	"go/ast"
+	"go/build"
+	"go/doc"
+	"go/parser"
+	"go/token"
+	"go/types"
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+
+	mgTypes "github.com/magefile/mage/types"
+)
+
+type PkgInfo struct {
+	Funcs            []Function
+	DefaultIsError   bool
+	DefaultIsContext bool
+	DefaultName      string
+	DefaultFunc      Function
+	Aliases          map[string]string
+}
+
+// Function represented a job function from a mage file
+type Function struct {
+	Name      string
+	IsError   bool
+	IsContext bool
+	Synopsis  string
+	Comment   string
+}
+
+// TemplateString returns code for the template switch to run the target.
+// It wraps each target call to match the func(context.Context) error that
+// runTarget requires.
+func (f Function) TemplateString() string {
+	if f.IsContext && f.IsError {
+		out := `wrapFn := func(ctx context.Context) error {
+				return %s(ctx)
+			}
+			err := runTarget(wrapFn)`
+		return fmt.Sprintf(out, f.Name)
+	}
+	if f.IsContext && !f.IsError {
+		out := `wrapFn := func(ctx context.Context) error {
+				%s(ctx)
+				return nil
+			}
+			err := runTarget(wrapFn)`
+		return fmt.Sprintf(out, f.Name)
+	}
+	if !f.IsContext && f.IsError {
+		out := `wrapFn := func(ctx context.Context) error {
+				return %s()
+			}
+			err := runTarget(wrapFn)`
+		return fmt.Sprintf(out, f.Name)
+	}
+	if !f.IsContext && !f.IsError {
+		out := `wrapFn := func(ctx context.Context) error {
+				%s()
+				return nil
+			}
+			err := runTarget(wrapFn)`
+		return fmt.Sprintf(out, f.Name)
+	}
+	return `fmt.Printf("Error formatting job code\n")
+	os.Exit(1)`
+}
+
+// Package parses a package
+func Package(path string, files []string) (*PkgInfo, error) {
+	fset := token.NewFileSet()
+
+	pkg, err := getPackage(path, files, fset)
+	if err != nil {
+		return nil, err
+	}
+
+	info, err := makeInfo(path, fset, pkg.Files)
+	if err != nil {
+		return nil, err
+	}
+
+	pi := &PkgInfo{}
+
+	p := doc.New(pkg, "./", 0)
+	for _, f := range p.Funcs {
+		if f.Recv != "" {
+			// skip methods
+			continue
+		}
+		if !ast.IsExported(f.Name) {
+			// skip non-exported functions
+			continue
+		}
+		if typ := voidOrError(f.Decl.Type, info); typ != mgTypes.InvalidType {
+			pi.Funcs = append(pi.Funcs, Function{
+				Name:      f.Name,
+				Comment:   f.Doc,
+				Synopsis:  sanitizeSynopsis(f),
+				IsError:   typ == mgTypes.ErrorType || typ == mgTypes.ContextErrorType,
+				IsContext: typ == mgTypes.ContextVoidType || typ == mgTypes.ContextErrorType,
+			})
+		}
+	}
+
+	setDefault(p, pi, info)
+	setAliases(p, pi, info)
+
+	return pi, nil
+}
+
+// sanitizeSynopsis sanitizes function Doc to create a summary.
+func sanitizeSynopsis(f *doc.Func) string {
+	synopsis := doc.Synopsis(f.Doc)
+
+	// If the synopsis begins with the function name, remove it. This is done to
+	// not repeat the text.
+	// From:
+	// clean	Clean removes the temporarily generated files
+	// To:
+	// clean 	removes the temporarily generated files
+	if syns := strings.Split(synopsis, " "); strings.EqualFold(f.Name, syns[0]) {
+		return strings.Join(syns[1:], " ")
+	}
+
+	return synopsis
+}
+
+func setDefault(p *doc.Package, pi *PkgInfo, info types.Info) {
+	for _, v := range p.Vars {
+		for x, name := range v.Names {
+			if name != "Default" {
+				continue
+			}
+			spec := v.Decl.Specs[x].(*ast.ValueSpec)
+			if len(spec.Values) != 1 {
+				log.Println("warning: default declaration has multiple values")
+			}
+			id, ok := spec.Values[0].(*ast.Ident)
+			if !ok {
+				log.Println("warning: default declaration is not a function name")
+			}
+			for _, f := range pi.Funcs {
+				if f.Name == id.Name {
+					pi.DefaultName = f.Name
+					pi.DefaultIsError = f.IsError
+					pi.DefaultIsContext = f.IsContext
+					pi.DefaultFunc = f
+					return
+				}
+			}
+			log.Println("warning: default declaration does not reference a mage target")
+		}
+	}
+}
+
+func setAliases(p *doc.Package, pi *PkgInfo, info types.Info) {
+	for _, v := range p.Vars {
+		for x, name := range v.Names {
+			if name != "Aliases" {
+				continue
+			}
+			spec, ok := v.Decl.Specs[x].(*ast.ValueSpec)
+			if !ok {
+				log.Println("warning: aliases declaration is not a value")
+				return
+			}
+			if len(spec.Values) != 1 {
+				log.Println("warning: aliases declaration has multiple values")
+			}
+			comp, ok := spec.Values[0].(*ast.CompositeLit)
+			if !ok {
+				log.Println("warning: aliases declaration is not a map")
+				return
+			}
+			pi.Aliases = make(map[string]string)
+			for _, elem := range comp.Elts {
+				kv, ok := elem.(*ast.KeyValueExpr)
+				if !ok {
+					log.Println("warning: alias declaration is not a map element")
+					return
+				}
+				k, ok := kv.Key.(*ast.BasicLit)
+				if !ok || k.Kind != token.STRING {
+					log.Println("warning: alias is not a string")
+					return
+				}
+				v, ok := kv.Value.(*ast.Ident)
+				if !ok {
+					log.Println("warning: alias target is not a function")
+					return
+				}
+				alias := strings.Trim(k.Value, "\"")
+				valid := false
+				for _, f := range pi.Funcs {
+					valid = valid || f.Name == v.Name
+				}
+				if !valid {
+					log.Printf("warning: alias declaration (%s) does not reference a mage target", alias)
+				}
+				pi.Aliases[alias] = v.Name
+			}
+			return
+		}
+	}
+}
+
+// getPackage returns the non-test package at the given path.
+func getPackage(path string, files []string, fset *token.FileSet) (*ast.Package, error) {
+	fm := make(map[string]bool, len(files))
+	for _, f := range files {
+		fm[f] = true
+	}
+
+	filter := func(f os.FileInfo) bool {
+		return fm[f.Name()]
+	}
+
+	pkgs, err := parser.ParseDir(fset, path, filter, parser.ParseComments)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse directory: %v", err)
+	}
+
+	for name, pkg := range pkgs {
+		if !strings.HasSuffix(name, "_test") {
+			return pkg, nil
+		}
+	}
+	return nil, fmt.Errorf("no non-test packages found in %s", path)
+}
+
+func makeInfo(dir string, fset *token.FileSet, files map[string]*ast.File) (types.Info, error) {
+	goroot := os.Getenv("GOROOT")
+	if goroot == "" {
+		c := exec.Command("go", "env", "GOROOT")
+		b, err := c.Output()
+		if err != nil {
+			return types.Info{}, fmt.Errorf("failed to get GOROOT from 'go env': %v", err)
+		}
+		goroot = strings.TrimSpace(string(b))
+		if goroot == "" {
+			return types.Info{}, fmt.Errorf("could not determine GOROOT")
+		}
+	}
+
+	build.Default.GOROOT = goroot
+
+	cfg := types.Config{
+		Importer: getImporter(fset),
+	}
+
+	info := types.Info{
+		Types: make(map[ast.Expr]types.TypeAndValue),
+		Defs:  make(map[*ast.Ident]types.Object),
+		Uses:  make(map[*ast.Ident]types.Object),
+	}
+
+	fs := make([]*ast.File, 0, len(files))
+	for _, v := range files {
+		fs = append(fs, v)
+	}
+
+	_, err := cfg.Check(dir, fset, fs, &info)
+	if err != nil {
+		return info, fmt.Errorf("failed to check types in directory: %v", err)
+	}
+	return info, nil
+}
+
+// errorOrVoid filters the list of functions to only those that return only an
+// error or have no return value, and have no parameters.
+func errorOrVoid(fns []*ast.FuncDecl, info types.Info) []*ast.FuncDecl {
+	fds := []*ast.FuncDecl{}
+
+	for _, fn := range fns {
+		if voidOrError(fn.Type, info) != mgTypes.InvalidType {
+			fds = append(fds, fn)
+		}
+	}
+	return fds
+}
+
+func hasContextParam(ft *ast.FuncType, info types.Info) bool {
+	if ft.Params.NumFields() == 1 {
+		ret := ft.Params.List[0]
+		t := info.TypeOf(ret.Type)
+		if t != nil && t.String() == "context.Context" {
+			return true
+		}
+	}
+	return false
+}
+
+func hasVoidReturn(ft *ast.FuncType, info types.Info) bool {
+	res := ft.Results
+	if res.NumFields() == 0 {
+		return true
+	}
+	return false
+}
+
+func hasErrorReturn(ft *ast.FuncType, info types.Info) bool {
+	res := ft.Results
+	if res.NumFields() == 1 {
+		ret := res.List[0]
+		if len(ret.Names) > 1 {
+			return false
+		}
+		t := info.TypeOf(ret.Type)
+		if t != nil && t.String() == "error" {
+			return true
+		}
+	}
+	return false
+}
+
+func voidOrError(ft *ast.FuncType, info types.Info) mgTypes.FuncType {
+	if hasContextParam(ft, info) {
+		if hasVoidReturn(ft, info) {
+			return mgTypes.ContextVoidType
+		}
+		if hasErrorReturn(ft, info) {
+			return mgTypes.ContextErrorType
+		}
+	}
+	if ft.Params.NumFields() == 0 {
+		if hasVoidReturn(ft, info) {
+			return mgTypes.VoidType
+		}
+		if hasErrorReturn(ft, info) {
+			return mgTypes.ErrorType
+		}
+	}
+	return mgTypes.InvalidType
+}

vendor/github.com/magefile/mage/parse/srcimporter/sizes.go

@@ -0,0 +1,40 @@
+// +build !go1.9
+
+package srcimporter
+
+import "go/types"
+
+// common architecture word sizes and alignments
+var gcArchSizes = map[string]*types.StdSizes{
+	"386":      {4, 4},
+	"arm":      {4, 4},
+	"arm64":    {8, 8},
+	"amd64":    {8, 8},
+	"amd64p32": {4, 8},
+	"mips":     {4, 4},
+	"mipsle":   {4, 4},
+	"mips64":   {8, 8},
+	"mips64le": {8, 8},
+	"ppc64":    {8, 8},
+	"ppc64le":  {8, 8},
+	"s390x":    {8, 8},
+	// When adding more architectures here,
+	// update the doc string of SizesFor below.
+}
+
+// SizesFor returns the Sizes used by a compiler for an architecture.
+// The result is nil if a compiler/architecture pair is not known.
+//
+// Supported architectures for compiler "gc":
+// "386", "arm", "arm64", "amd64", "amd64p32", "mips", "mipsle",
+// "mips64", "mips64le", "ppc64", "ppc64le", "s390x".
+func SizesFor(compiler, arch string) types.Sizes {
+	if compiler != "gc" {
+		return nil
+	}
+	s, ok := gcArchSizes[arch]
+	if !ok {
+		return nil
+	}
+	return s
+}

vendor/github.com/magefile/mage/parse/srcimporter/srcimporter.go

@@ -0,0 +1,213 @@
+// +build !go1.9
+
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package srcimporter implements importing directly
+// from source files rather than installed packages.
+package srcimporter
+
+import (
+	"fmt"
+	"go/ast"
+	"go/build"
+	"go/parser"
+	"go/token"
+	"go/types"
+	"path/filepath"
+	"sync"
+)
+
+// An Importer provides the context for importing packages from source code.
+type Importer struct {
+	ctxt     *build.Context
+	fset     *token.FileSet
+	sizes    types.Sizes
+	packages map[string]*types.Package
+}
+
+// NewImporter returns a new Importer for the given context, file set, and map
+// of packages. The context is used to resolve import paths to package paths,
+// and identifying the files belonging to the package. If the context provides
+// non-nil file system functions, they are used instead of the regular package
+// os functions. The file set is used to track position information of package
+// files; and imported packages are added to the packages map.
+func New(ctxt *build.Context, fset *token.FileSet, packages map[string]*types.Package) *Importer {
+	return &Importer{
+		ctxt:     ctxt,
+		fset:     fset,
+		sizes:    SizesFor(ctxt.Compiler, ctxt.GOARCH), // uses go/types default if GOARCH not found
+		packages: packages,
+	}
+}
+
+// Importing is a sentinel taking the place in Importer.packages
+// for a package that is in the process of being imported.
+var importing types.Package
+
+// Import(path) is a shortcut for ImportFrom(path, "", 0).
+func (p *Importer) Import(path string) (*types.Package, error) {
+	return p.ImportFrom(path, "", 0)
+}
+
+// ImportFrom imports the package with the given import path resolved from the given srcDir,
+// adds the new package to the set of packages maintained by the importer, and returns the
+// package. Package path resolution and file system operations are controlled by the context
+// maintained with the importer. The import mode must be zero but is otherwise ignored.
+// Packages that are not comprised entirely of pure Go files may fail to import because the
+// type checker may not be able to determine all exported entities (e.g. due to cgo dependencies).
+func (p *Importer) ImportFrom(path, srcDir string, mode types.ImportMode) (*types.Package, error) {
+	if mode != 0 {
+		panic("non-zero import mode")
+	}
+
+	// determine package path (do vendor resolution)
+	var bp *build.Package
+	var err error
+	switch {
+	default:
+		if abs, err := p.absPath(srcDir); err == nil { // see issue #14282
+			srcDir = abs
+		}
+		bp, err = p.ctxt.Import(path, srcDir, build.FindOnly)
+
+	case build.IsLocalImport(path):
+		// "./x" -> "srcDir/x"
+		bp, err = p.ctxt.ImportDir(filepath.Join(srcDir, path), build.FindOnly)
+
+	case p.isAbsPath(path):
+		return nil, fmt.Errorf("invalid absolute import path %q", path)
+	}
+	if err != nil {
+		return nil, err // err may be *build.NoGoError - return as is
+	}
+
+	// package unsafe is known to the type checker
+	if bp.ImportPath == "unsafe" {
+		return types.Unsafe, nil
+	}
+
+	// no need to re-import if the package was imported completely before
+	pkg := p.packages[bp.ImportPath]
+	if pkg != nil {
+		if pkg == &importing {
+			return nil, fmt.Errorf("import cycle through package %q", bp.ImportPath)
+		}
+		if !pkg.Complete() {
+			// Package exists but is not complete - we cannot handle this
+			// at the moment since the source importer replaces the package
+			// wholesale rather than augmenting it (see #19337 for details).
+			// Return incomplete package with error (see #16088).
+			return pkg, fmt.Errorf("reimported partially imported package %q", bp.ImportPath)
+		}
+		return pkg, nil
+	}
+
+	p.packages[bp.ImportPath] = &importing
+	defer func() {
+		// clean up in case of error
+		// TODO(gri) Eventually we may want to leave a (possibly empty)
+		// package in the map in all cases (and use that package to
+		// identify cycles). See also issue 16088.
+		if p.packages[bp.ImportPath] == &importing {
+			p.packages[bp.ImportPath] = nil
+		}
+	}()
+
+	// collect package files
+	bp, err = p.ctxt.ImportDir(bp.Dir, 0)
+	if err != nil {
+		return nil, err // err may be *build.NoGoError - return as is
+	}
+	var filenames []string
+	filenames = append(filenames, bp.GoFiles...)
+	filenames = append(filenames, bp.CgoFiles...)
+
+	files, err := p.parseFiles(bp.Dir, filenames)
+	if err != nil {
+		return nil, err
+	}
+
+	// type-check package files
+	conf := types.Config{
+		IgnoreFuncBodies: true,
+		FakeImportC:      true,
+		Importer:         p,
+		Sizes:            p.sizes,
+	}
+	pkg, err = conf.Check(bp.ImportPath, p.fset, files, nil)
+	if err != nil {
+		// Type-checking stops after the first error (types.Config.Error is not set),
+		// so the returned package is very likely incomplete. Don't return it since
+		// we don't know its condition: It's very likely unsafe to use and it's also
+		// not added to p.packages which may cause further problems (issue #20837).
+		return nil, fmt.Errorf("type-checking package %q failed (%v)", bp.ImportPath, err)
+	}
+
+	p.packages[bp.ImportPath] = pkg
+	return pkg, nil
+}
+
+func (p *Importer) parseFiles(dir string, filenames []string) ([]*ast.File, error) {
+	open := p.ctxt.OpenFile // possibly nil
+
+	files := make([]*ast.File, len(filenames))
+	errors := make([]error, len(filenames))
+
+	var wg sync.WaitGroup
+	wg.Add(len(filenames))
+	for i, filename := range filenames {
+		go func(i int, filepath string) {
+			defer wg.Done()
+			if open != nil {
+				src, err := open(filepath)
+				if err != nil {
+					errors[i] = fmt.Errorf("opening package file %s failed (%v)", filepath, err)
+					return
+				}
+				files[i], errors[i] = parser.ParseFile(p.fset, filepath, src, 0)
+				src.Close() // ignore Close error - parsing may have succeeded which is all we need
+			} else {
+				// Special-case when ctxt doesn't provide a custom OpenFile and use the
+				// parser's file reading mechanism directly. This appears to be quite a
+				// bit faster than opening the file and providing an io.ReaderCloser in
+				// both cases.
+				// TODO(gri) investigate performance difference (issue #19281)
+				files[i], errors[i] = parser.ParseFile(p.fset, filepath, nil, 0)
+			}
+		}(i, p.joinPath(dir, filename))
+	}
+	wg.Wait()
+
+	// if there are errors, return the first one for deterministic results
+	for _, err := range errors {
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return files, nil
+}
+
+// context-controlled file system operations
+
+func (p *Importer) absPath(path string) (string, error) {
+	// TODO(gri) This should be using p.ctxt.AbsPath which doesn't
+	// exist but probably should. See also issue #14282.
+	return filepath.Abs(path)
+}
+
+func (p *Importer) isAbsPath(path string) bool {
+	if f := p.ctxt.IsAbsPath; f != nil {
+		return f(path)
+	}
+	return filepath.IsAbs(path)
+}
+
+func (p *Importer) joinPath(elem ...string) string {
+	if f := p.ctxt.JoinPath; f != nil {
+		return f(elem...)
+	}
+	return filepath.Join(elem...)
+}