1
0
mirror of https://github.com/docker/docs.git synced 2026-10-23 18:09:02 +03:00
Files
docs/hack/api-docs/main.go
David Karlsson 2b12ac0ef8 build: vendor API reference data to speed up site builds (#26182)
API generation introduced in #26048 added roughly five minutes to site
builds: median successful main-branch build-workflow duration rose from
4m 05s to 8m 48s across September 3–25. The pre-build command took 4m
41s–4m 53s in each of four jobs on the merge commit.

Vendor the generated presentation model in `data/api-reference.json`,
mark it as generated, and let Docker, Netlify, and local Hugo builds
consume it directly. Keep the existing rendered-output checks.

Move generator tests, strict spec validation, and a byte-for-byte
regeneration check into `docker buildx bake validate-api-reference`,
inheriting the shared Docker base stage. The target uses the pinned
Vacuum release binary with Dockerfile ADD checksum verification and
archive extraction instead of compiling Vacuum from source. The existing
build workflow runs this target alongside the other validation matrix
entries. Only this matrix entry is advisory; its failures do not block
site builds or deployment.

Validated locally with the Docker target, rejection of deliberately
stale data, repeatable generation across directories, a production Hugo
build and verification of 453 API HTML/Markdown page pairs, workflow
syntax checks, and scoped documentation lint. The prebuilt-binary CI
check passed in 1m 40s, down from 5m 15s with source compilation. Remote
layer-cache settings were subsequently removed; local Go cache mounts
remain.

Generated by Codex
2026-09-25 18:45:09 +02:00

758 lines
21 KiB
Go

// api-docs prepares a source-preserving reference model for Docker API documentation.
package main
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"net/url"
"os"
"path/filepath"
"sort"
"strings"
"github.com/pb33f/libopenapi"
"github.com/pb33f/libopenapi/datamodel"
js "github.com/santhosh-tekuri/jsonschema/v6"
yaml "go.yaml.in/yaml/v4"
)
type Object = map[string]any
const dialect = "https://spec.openapis.org/oas/3.1/dialect/base"
var methods = []string{"get", "put", "post", "delete", "options", "head", "patch", "trace", "query"}
func obj(v any) Object {
m, _ := v.(map[string]any)
if m == nil {
return Object{}
}
return m
}
func arr(v any) []any { a, _ := v.([]any); return a }
func str(v any) string { s, _ := v.(string); return s }
func keys(m Object) []string {
a := make([]string, 0, len(m))
for k := range m {
a = append(a, k)
}
sort.Strings(a)
return a
}
func hash(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) }
func encoded(v any) []byte {
b, e := json.MarshalIndent(v, "", " ")
if e != nil {
panic(e)
}
return append(b, '\n')
}
func esc(s string) string { return strings.ReplaceAll(strings.ReplaceAll(s, "~", "~0"), "/", "~1") }
func pointer(root any, p string) (any, error) {
if p == "" {
return root, nil
}
if !strings.HasPrefix(p, "/") {
return nil, fmt.Errorf("unsupported pointer %s", p)
}
for _, k := range strings.Split(p[1:], "/") {
k = strings.ReplaceAll(strings.ReplaceAll(k, "~1", "/"), "~0", "~")
switch v := root.(type) {
case map[string]any:
var ok bool
root, ok = v[k]
if !ok {
return nil, fmt.Errorf("missing pointer %s", p)
}
case []any:
var n int
if _, e := fmt.Sscanf(k, "%d", &n); e != nil || n < 0 || n >= len(v) {
return nil, fmt.Errorf("invalid array pointer %s", p)
}
root = v[n]
default:
return nil, fmt.Errorf("non-container pointer %s", p)
}
}
return root, nil
}
func readJSON(p string) (any, error) {
b, e := os.ReadFile(p)
if e != nil {
return nil, e
}
return js.UnmarshalJSON(bytes.NewReader(b))
}
func writeJSON(p string, v any) error {
if e := os.MkdirAll(filepath.Dir(p), 0755); e != nil {
return e
}
return os.WriteFile(p, encoded(v), 0644)
}
type Diagnostic struct {
Rule string `json:"rule"`
Pointer string `json:"pointer"`
Message string `json:"message"`
}
type Source struct {
Experimental bool `json:"experimental,omitempty"`
ID string `json:"id"`
Product string `json:"product"`
Title string `json:"title"`
Source string `json:"source"`
Owner string `json:"owner"`
Manual string `json:"manual"`
Connection string `json:"connection"`
Guides []string `json:"guides"`
}
type Registry struct{ resources map[string]any }
func (r *Registry) Load(uri string) (any, error) {
if v, ok := r.resources[uri]; ok {
return v, nil
}
return nil, fmt.Errorf("unlocked resource: %s", uri)
}
type Document struct {
Root Object
URI string
Registry *Registry
Diagnostics []Diagnostic
Schemas map[string]any
Locations map[string]int
Compiler *js.Compiler
Compiled map[string]*js.Schema
Digest string
Source Source
SchemaCount int
ExampleCount int
}
func (d *Document) issue(rule, p, msg string) {
d.Diagnostics = append(d.Diagnostics, Diagnostic{Rule: rule, Pointer: p, Message: strings.ReplaceAll(msg, d.URI, "source:"+d.Source.ID)})
}
func walk(v any, p string, fn func(Object, string)) {
switch n := v.(type) {
case map[string]any:
fn(n, p)
for _, k := range keys(n) {
walk(n[k], p+"/"+esc(k), fn)
}
case []any:
for i, x := range n {
walk(x, fmt.Sprintf("%s/%d", p, i), fn)
}
}
}
func strictSource(file string) (Object, libopenapi.Document, error) {
b, e := os.ReadFile(file)
if e != nil {
return nil, nil, e
}
cfg := datamodel.NewDocumentConfiguration()
cfg.AllowRemoteReferences = false
cfg.AllowFileReferences = false
cfg.SkipExternalRefResolution = true
doc, e := libopenapi.NewDocumentWithConfiguration(b, cfg)
if e != nil {
return nil, nil, e
}
v, e := strictFragment(file)
return obj(v), doc, e
}
// YAML timestamp scalars are strings in the JSON data model. Decode source nodes
// explicitly: libopenapi's JSON convenience view normalizes their spelling.
func strictFragment(file string) (any, error) {
b, e := os.ReadFile(file)
if e != nil {
return nil, e
}
var node yaml.Node
if e = yaml.Unmarshal(b, &node); e != nil {
return nil, e
}
v, e := sourceValue(&node, map[*yaml.Node]bool{})
if e != nil {
return nil, e
}
raw, e := json.Marshal(v)
if e != nil {
return nil, e
}
return js.UnmarshalJSON(bytes.NewReader(raw))
}
func sourceValue(n *yaml.Node, active map[*yaml.Node]bool) (any, error) {
if active[n] {
return nil, errors.New("cyclic YAML alias")
}
active[n] = true
defer delete(active, n)
switch n.Kind {
case yaml.DocumentNode:
if len(n.Content) != 1 {
return nil, errors.New("expected one YAML document")
}
return sourceValue(n.Content[0], active)
case yaml.AliasNode:
return sourceValue(n.Alias, active)
case yaml.MappingNode:
m := Object{}
for i := 0; i < len(n.Content); i += 2 {
k := n.Content[i]
if k.Kind != yaml.ScalarNode || k.Value == "<<" {
return nil, errors.New("complex/merge YAML keys are outside the source profile")
}
if _, ok := m[k.Value]; ok {
return nil, fmt.Errorf("duplicate key %s at line %d", k.Value, k.Line)
}
v, e := sourceValue(n.Content[i+1], active)
if e != nil {
return nil, e
}
m[k.Value] = v
}
return m, nil
case yaml.SequenceNode:
a := []any{}
for _, c := range n.Content {
v, e := sourceValue(c, active)
if e != nil {
return nil, e
}
a = append(a, v)
}
return a, nil
case yaml.ScalarNode:
if n.Tag == "!!str" || n.Tag == "!!timestamp" {
return n.Value, nil
}
var v any
e := n.Decode(&v)
return v, e
}
return nil, errors.New("unsupported YAML node")
}
func loadDocument(file, metaDir string) (*Document, error) {
abs, _ := filepath.Abs(file)
root, parsed, e := strictSource(abs)
if e != nil {
return nil, e
}
uri := (&url.URL{Scheme: "file", Path: abs}).String()
b, _ := os.ReadFile(file)
d := &Document{Root: root, URI: uri, Diagnostics: []Diagnostic{}, Registry: &Registry{resources: map[string]any{uri: root}}, Schemas: map[string]any{}, Locations: map[string]int{}, Compiled: map[string]*js.Schema{}, Digest: hash(b)}
// libopenapi's typed model supplies operation views/locations; schema serialization is never used.
model, modelErr := parsed.BuildV3Model()
if modelErr == nil && model != nil && model.Model.Paths != nil {
for p, item := range model.Model.Paths.PathItems.FromOldest() {
for m, op := range item.GetOperations().FromOldest() {
if op.GoLow().RootNode != nil {
d.Locations["/paths/"+esc(p)+"/"+m] = op.GoLow().RootNode.Line
}
}
}
}
// All file resources are acquired before compilation, under a bounded source directory.
var acquire func(any, string) error
acquire = func(v any, base string) error {
var failure error
walk(v, "", func(n Object, _ string) {
for _, k := range []string{"$ref", "$dynamicRef"} {
ref, ok := n[k].(string)
if !ok {
continue
}
u, er := url.Parse(ref)
if er != nil {
failure = er
return
}
bu, _ := url.Parse(base)
u = bu.ResolveReference(u)
u.Fragment = ""
id := u.String()
if _, ok := d.Registry.resources[id]; ok {
continue
}
if u.Scheme != "file" {
failure = fmt.Errorf("unlocked reference %s", id)
return
}
rel, er := filepath.Rel(filepath.Dir(abs), u.Path)
if er != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
failure = fmt.Errorf("reference escapes package: %s", ref)
return
}
r, er := strictFragment(u.Path)
if er != nil {
failure = er
return
}
d.Registry.resources[id] = r
if er = acquire(r, id); er != nil {
failure = er
}
}
})
return failure
}
if e = acquire(root, uri); e != nil {
return nil, e
}
lockRaw, e := os.ReadFile(filepath.Join(metaDir, "lock.json"))
if e != nil {
return nil, e
}
var lock []struct{ File, URI, SHA256 string }
if e = json.Unmarshal(lockRaw, &lock); e != nil {
return nil, e
}
for _, l := range lock {
b, e := os.ReadFile(filepath.Join(metaDir, l.File))
if e != nil {
return nil, e
}
if hash(b) != l.SHA256 {
return nil, fmt.Errorf("dialect digest mismatch: %s", l.File)
}
v, e := js.UnmarshalJSON(bytes.NewReader(b))
if e != nil {
return nil, e
}
d.Registry.resources[l.URI] = v
if id := str(obj(v)["$id"]); id != "" {
d.Registry.resources[id] = v
}
}
d.Compiler = js.NewCompiler()
d.Compiler.DefaultDraft(js.Draft2020)
d.Compiler.UseLoader(d.Registry)
for id, v := range d.Registry.resources {
if e = d.Compiler.AddResource(id, v); e != nil {
return nil, e
}
}
return d, nil
}
func (d *Document) resolve(v any) any {
seen := map[string]bool{}
for {
r := str(obj(v)["$ref"])
if r == "" {
return v
}
if seen[r] {
return v
}
seen[r] = true
u, e := url.Parse(r)
if e != nil {
return v
}
base, _ := url.Parse(d.URI)
u = base.ResolveReference(u)
frag := u.Fragment
u.Fragment = ""
target, ok := d.Registry.resources[u.String()]
if !ok {
return v
}
resolved, e := pointer(target, frag)
if e != nil {
return v
}
merged := Object{}
for k, x := range obj(resolved) {
merged[k] = x
}
for _, k := range []string{"summary", "description"} {
if x, ok := obj(v)[k]; ok {
merged[k] = x
}
}
if len(merged) == 0 {
return resolved
}
v = merged
}
}
func (d *Document) schemaRoots(v any, p string) {
switch n := v.(type) {
case map[string]any:
for _, k := range keys(n) {
x := n[k]
q := p + "/" + esc(k)
if k == "schemas" && p == "/components" {
for name, s := range obj(x) {
d.Schemas[q+"/"+esc(name)] = s
}
} else if k == "schema" || k == "itemSchema" {
d.Schemas[q] = x
} else if !strings.HasPrefix(k, "x-") && k != "example" && k != "examples" && k != "value" {
d.schemaRoots(x, q)
}
}
case []any:
for i, x := range n {
d.schemaRoots(x, fmt.Sprintf("%s/%d", p, i))
}
}
}
func (d *Document) validate(metaDir string) {
if d.Root["openapi"] != "3.2.0" {
d.issue("S1", "/openapi", "Expected OpenAPI 3.2.0")
}
if d.Root["jsonSchemaDialect"] != dialect {
d.issue("S1", "/jsonSchemaDialect", "Expected the selected OAS dialect")
}
documentSchema, e := d.Compiler.Compile("https://spec.openapis.org/oas/3.2/schema/2025-09-17")
if e != nil {
d.issue("structure", "", e.Error())
} else if e = documentSchema.Validate(d.Root); e != nil {
d.issue("structure", "", e.Error())
}
// OAS Reference Objects also need explicit validation: a document schema cannot prove their target exists.
walk(d.Root, "", func(n Object, p string) {
if r := str(n["$ref"]); r != "" {
u, er := url.Parse(r)
if er != nil {
d.issue("reference", p, er.Error())
return
}
base, _ := url.Parse(d.URI)
u = base.ResolveReference(u)
frag := u.Fragment
u.Fragment = ""
v, ok := d.Registry.resources[u.String()]
if !ok {
d.issue("reference", p, "Unregistered resource "+r)
} else if strings.HasPrefix(frag, "/") {
if _, er = pointer(v, frag); er != nil {
d.issue("reference", p, er.Error())
}
}
}
})
if len(obj(d.Root["webhooks"])) > 0 {
d.issue("capability", "/webhooks", "Webhook navigation is not supported by the reference renderer")
}
walk(d.Root, "", func(n Object, p string) {
if len(obj(n["callbacks"])) > 0 {
d.issue("capability", p+"/callbacks", "Callback navigation is not supported by the reference renderer")
}
})
d.schemaRoots(d.Root, "")
schemaMeta, e := d.Compiler.Compile(dialect)
if e != nil {
d.issue("dialect", "", e.Error())
return
}
for _, p := range keys(d.Schemas) {
s := d.Schemas[p]
d.SchemaCount++
if e = schemaMeta.Validate(s); e != nil {
d.issue("schema", p, e.Error())
continue
}
compiled, e := d.Compiler.Compile(d.URI + "#" + p)
if e != nil {
d.issue("schema", p, e.Error())
continue
}
d.Compiled[p] = compiled
d.schemaExamples(s, p)
}
// Media, parameter and header examples are separate from examples nested inside schemas.
walk(d.Root, "", func(n Object, p string) {
schemaKey := "schema"
if _, ok := n["itemSchema"]; ok {
schemaKey = "itemSchema"
}
c := d.Compiled[p+"/"+schemaKey]
if c == nil {
return
}
if v, ok := n["example"]; ok {
d.example(c, v, p+"/example")
}
for _, name := range keys(obj(n["examples"])) {
ex := obj(d.resolve(obj(n["examples"])[name]))
if v, ok := ex["value"]; ok {
d.example(c, v, p+"/examples/"+esc(name)+"/value")
}
if _, ok := ex["externalValue"]; ok {
d.issue("example-external", p+"/examples/"+esc(name), "External example requires a locked media fixture")
}
}
})
ids := map[string]bool{}
tags := map[string]string{}
for _, raw := range arr(d.Root["tags"]) {
t := obj(raw)
tags[str(t["name"])] = str(t["kind"])
}
for _, op := range d.operations() {
id := str(op["id"])
p := str(op["pointer"])
if id == "" || ids[id] {
d.issue("S4", p, "Operation ID must be present and unique")
}
for i, requirement := range arr(op["security"]) {
for _, name := range keys(obj(requirement)) {
if _, ok := obj(obj(d.Root["components"])["securitySchemes"])[name]; !ok {
d.issue("security", fmt.Sprintf("%s/security/%d/%s", p, i, esc(name)), "Undefined security scheme: "+name)
}
}
}
ids[id] = true
opTags := arr(op["tags"])
if len(opTags) == 0 || tags[str(opTags[0])] != "nav" {
d.issue("S5", p+"/tags", "The first operation tag must identify a declared navigation group")
}
for _, tag := range opTags {
if _, ok := tags[str(tag)]; !ok {
d.issue("S5", p+"/tags", "Operation tag is undeclared: "+str(tag))
}
}
if len(arr(op["servers"])) == 0 && d.Source.Connection != "unix" {
d.issue("S6", p, "Operation needs effective servers or a local connection profile")
}
if strings.TrimSpace(str(op["description"])) == "" {
d.issue("S4", p+"/description", "Operation description required")
}
for _, pr := range arr(op["parameters"]) {
param := obj(pr)
if param["in"] == "header" {
switch strings.ToLower(str(param["name"])) {
case "authorization", "accept", "content-type":
d.issue("S8", str(param["pointer"]), "OpenAPI ignores this header parameter; use security or media types")
}
}
if str(param["description"]) == "" {
d.issue("S8", str(param["pointer"])+"/description", "Parameter description requires editorial review")
}
}
}
}
func (d *Document) schemaExamples(s any, p string) {
n := obj(s)
if len(n) == 0 {
return
}
if _, ok := n["$schema"]; ok && n["$schema"] != dialect {
d.issue("S1", p+"/$schema", "Schema dialect override is outside the profile")
}
c, e := d.Compiler.Compile(d.URI + "#" + p)
if e == nil {
if v, ok := n["example"]; ok {
d.example(c, v, p+"/example")
}
for i, v := range arr(n["examples"]) {
d.example(c, v, fmt.Sprintf("%s/examples/%d", p, i))
}
}
for k, x := range n {
switch k {
case "properties", "patternProperties", "$defs", "dependentSchemas":
for _, name := range keys(obj(x)) {
d.schemaExamples(obj(x)[name], p+"/"+k+"/"+esc(name))
}
case "allOf", "anyOf", "oneOf", "prefixItems":
for i, v := range arr(x) {
d.schemaExamples(v, fmt.Sprintf("%s/%s/%d", p, k, i))
}
case "items", "additionalProperties", "unevaluatedProperties", "contains", "not", "if", "then", "else", "contentSchema", "propertyNames":
d.schemaExamples(x, p+"/"+k)
}
}
}
func (d *Document) example(c *js.Schema, v any, p string) {
d.ExampleCount++
if e := c.Validate(v); e != nil {
d.issue("example", p, e.Error())
}
}
func (d *Document) effective(root, item, op Object, k string) any {
if v, ok := op[k]; ok {
return v
}
if k == "servers" {
if v, ok := item[k]; ok {
return v
}
}
return root[k]
}
func (d *Document) parameters(item, op Object, p string) []any {
a := []any{}
positions := map[string]int{}
for i, container := range []Object{item, op} {
for j, raw := range arr(container["parameters"]) {
v := obj(d.resolve(raw))
c := Object{}
for k, x := range v {
c[k] = x
}
prefix := p
if i == 0 {
prefix = p[:strings.LastIndex(p, "/")]
}
c["pointer"] = fmt.Sprintf("%s/parameters/%d", prefix, j)
key := str(v["in"]) + ":" + str(v["name"])
if n, ok := positions[key]; ok {
a[n] = c
} else {
positions[key] = len(a)
a = append(a, c)
}
}
}
return a
}
func (d *Document) exampleList(v Object) []any {
out := []any{}
if x, ok := v["example"]; ok {
out = append(out, Object{"name": "Example", "value": x})
}
for _, name := range keys(obj(v["examples"])) {
x := obj(d.resolve(obj(v["examples"])[name]))
if value, ok := x["value"]; ok {
out = append(out, Object{"name": name, "value": value})
}
}
return out
}
func (d *Document) variants(op Object, p string) []any {
out := []any{}
add := func(direction, status string, raw any, ptr string) {
container := obj(d.resolve(raw))
content := obj(container["content"])
if len(content) == 0 {
out = append(out, Object{"direction": direction, "status": status, "description": container["description"], "headers": container["headers"], "media": "", "pointer": ptr, "examples": []any{}})
}
for _, media := range keys(content) {
v := obj(content[media])
a := Object{"direction": direction, "status": status, "description": container["description"], "headers": container["headers"], "media": media, "pointer": ptr + "/content/" + esc(media), "examples": d.mediaExamples(v), "required": container["required"]}
for _, k := range []string{"schema", "itemSchema", "encoding", "itemEncoding", "prefixEncoding"} {
if s, ok := v[k]; ok {
a[k] = s
}
}
out = append(out, a)
}
}
if v, ok := op["requestBody"]; ok {
add("Request", "", v, p+"/requestBody")
}
for _, code := range keys(obj(op["responses"])) {
add("Response", code, obj(op["responses"])[code], p+"/responses/"+esc(code))
}
return out
}
func (d *Document) operations() []Object {
out := []Object{}
for _, path := range keys(obj(d.Root["paths"])) {
item := obj(d.resolve(obj(d.Root["paths"])[path]))
ms := append([]string{}, methods...)
for m := range obj(item["additionalOperations"]) {
ms = append(ms, m)
}
for _, method := range ms {
raw, ok := item[method]
extra := !ok
if !ok {
raw, ok = obj(item["additionalOperations"])[method]
}
if !ok {
continue
}
op := obj(raw)
p := "/paths/" + esc(path) + "/" + method
if extra {
p = "/paths/" + esc(path) + "/additionalOperations/" + esc(method)
}
v := Object{"id": op["operationId"], "method": strings.ToUpper(method), "path": path, "pointer": p, "line": d.Locations[p], "summary": op["summary"], "description": op["description"], "tags": op["tags"], "deprecated": op["deprecated"], "servers": d.effective(d.Root, item, op, "servers"), "security": d.effective(d.Root, item, op, "security"), "parameters": d.parameters(item, op, p), "variants": d.variants(op, p), "raw": op}
out = append(out, v)
}
}
return out
}
func main() {
if e := run(); e != nil {
fmt.Fprintln(os.Stderr, e)
os.Exit(1)
}
}
func run() error {
if len(os.Args) < 3 {
return errors.New("usage: api-docs check|generate|inspect|sources ROOT")
}
command := os.Args[1]
if command != "check" && command != "generate" && command != "inspect" && command != "sources" {
return fmt.Errorf("unknown command: %s", command)
}
if len(os.Args) > 3 {
return errors.New("unexpected arguments")
}
root, _ := filepath.Abs(os.Args[2])
dir := filepath.Join(root, "hack/api-docs")
meta := filepath.Join(dir, "validation", "dialects")
manifestRaw, e := os.ReadFile(filepath.Join(dir, "sources.json"))
if e != nil {
return e
}
var manifest struct {
APIs []Source `json:"apis"`
LegacyAPIs []Object `json:"legacyAPIs"`
}
if e = json.Unmarshal(manifestRaw, &manifest); e != nil {
return e
}
if command == "sources" {
for _, src := range manifest.APIs {
fmt.Printf("%s\t%s\n", src.ID, src.Source)
}
return nil
}
reports := []any{}
models := []any{}
blocking := 0
for _, src := range manifest.APIs {
d, e := loadDocument(filepath.Join(root, src.Source), meta)
if e != nil {
return fmt.Errorf("%s: %w", src.ID, e)
}
d.Source = src
d.validate(meta)
blocking += len(d.Diagnostics)
reports = append(reports, Object{"api": src.ID, "sha256": d.Digest, "schemas": d.SchemaCount, "examples": d.ExampleCount, "operations": len(d.operations()), "diagnostics": d.Diagnostics})
models = append(models, d.model())
fmt.Printf("%s: %d operations, %d schema roots, %d examples, %d diagnostics\n", src.ID, len(d.operations()), d.SchemaCount, d.ExampleCount, len(d.Diagnostics))
}
out := filepath.Join(root, "tmp/api-reference")
if e = writeJSON(filepath.Join(out, "validation.json"), reports); e != nil {
return e
}
if command == "inspect" {
return nil
}
if blocking > 0 {
return fmt.Errorf("%d blocking diagnostics; see tmp/api-reference/validation.json", blocking)
}
if command == "generate" {
if e = writeJSON(filepath.Join(root, "data/api-reference.json"), Object{"modelVersion": 1, "apis": models, "legacyAPIs": manifest.LegacyAPIs}); e != nil {
return e
}
}
return nil
}