docs: add dev instructions (#150)

fix #149 Reviewed-on: https://codeberg.org/crowci/crow/pulls/150 Co-authored-by: pat-s <patrick.schratz@gmail.com> Co-committed-by: pat-s <patrick.schratz@gmail.com>
2025-04-18 04:44:01 +03:00 · 2025-04-14 10:39:36 +00:00 · 2025-04-14 10:39:36 +00:00 · a42e00072d
commit a42e00072d
parent 48db7f3bba
6 changed files with 178 additions and 45 deletions
--- a/.crow/docs.yaml
+++ b/.crow/docs.yaml
@ -13,8 +13,6 @@ clone:

 steps:
  - name: 'Build docs'
-    when:
-      - event: push
    image: alpine:3.21
    environment:
      IS_CI: true
@ -30,7 +28,7 @@ steps:
      - git config --global user.name "crowci-bot"
      - git remote set-url origin https://crowci-bot:$${CODEBERG_BOT_TOKEN}@codeberg.org/crowci/crow.git
      # build & deploy
-      - just docs-venv-fish
+      - just docs-venv
      - just docs-mike "deploy dev --push"
      - mv site/domains site/.domains

--- a/18
+++ b/18
@ -64,14 +64,14 @@ test: test-agent test-server test-server-datastore test-cli test-lib
 generate: generate-openapi
 	CGO_ENABLED=0 go generate ./...

-generate-openapi: install-tools
+generate-openapi: install-dev-deps
 		go run github.com/swaggo/swag/cmd/swag fmt
 		CGO_ENABLED=0 go generate cmd/server/openapi.go

-mockery: install-tools
+mockery: install-dev-deps
 		mockery

-install-tools:
+install-dev-deps:
 		# @hash golangci-lint > /dev/null 2>&1 || go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
 		@hash gofumpt > /dev/null 2>&1 || go install mvdan.cc/gofumpt@latest
 		@hash addlicense > /dev/null 2>&1 || go install github.com/google/addlicense@latest
@ -110,7 +110,7 @@ test-lib:

 ## docs

-docs-venv-fish:
+docs-venv:
    test -d venv || python3 -m venv venv
    fish -c 'if set -q GH_TOKEN_MKDOCS_MATERIAL; source venv/bin/activate.fish; and pip3 install -r docs/requirements-insiders.txt; else; source venv/bin/activate.fish; and pip3 install -r docs/requirements.txt; end'

@ -118,7 +118,7 @@ docs-mike ARG:
 		fish -c 'source venv/bin/activate.fish && mike {{ ARG }}'

 docs-serve:
-		fish -c 'source venv/bin/activate.fish && mkdocs --version && mkdocs serve'
+		fish -c 'source venv/bin/activate.fish && mkdocs --version && mkdocs serve -a localhost:5555'

 docs-build:
 		fish -c 'source venv/bin/activate.fish && mkdocs --version && CI={{CI}} mkdocs build'
@ -133,11 +133,11 @@ cherry-pick COMMIT:

 ## build

-lint: install-tools
+lint-backend: install-dev-deps
 		golangci-lint run

 # FIXME: feature of 1.38.0 [working-directory('web')]
-lint-ui:
+lint-frontend:
 		cd web/ && pnpm lint --quiet

 # FIXME: feature of 1.38.0 [working-directory('web')]
@ -257,12 +257,12 @@ image-agent-alpine TAG:
 		echo $codeberg_bot_token | docker login codeberg.org -u crowci-bot --password-stdin
 		docker buildx build --platform $PLATFORMS -t codeberg.org/crowci/crow-agent:{{TAG}}-alpine -f docker/Dockerfile.agent.alpine.multiarch --push .

-# env PLATFORMS='linux/amd64,linux/arm64' just image-cli
+# env PLATFORMS='linux/amd64,linux/arm64' just image-cli dev
 image-cli TAG:
 		echo $codeberg_bot_token | docker login codeberg.org -u crowci-bot --password-stdin
 		docker buildx build --platform $PLATFORMS -t codeberg.org/crowci/crow-cli:{{TAG}} -f docker/Dockerfile.cli.multiarch.rootless --push .

-# env PLATFORMS='linux/amd64,linux/arm64' just image-cli-alpine
+# env PLATFORMS='linux/amd64,linux/arm64' just image-cli-alpine dev
 image-cli-alpine TAG:
 		echo $codeberg_bot_token | docker login codeberg.org -u crowci-bot --password-stdin
 		docker buildx build --platform $PLATFORMS -t codeberg.org/crowci/crow-cli:{{TAG}}-alpine -f docker/Dockerfile.cli.alpine.multiarch.rootless --push .
--- a/devcontainer.json
+++ b/devcontainer.json
@ -0,0 +1,18 @@
+{
+  "image": "mcr.microsoft.com/devcontainers/go",
+  "features": {
+    "ghcr.io/meaningful-ooo/devcontainer-features/fish:2": {
+      "version": "4.0.1"
+    },
+    "ghcr.io/devcontainers-extra/features/pnpm:2": {
+      "version": "latest"
+    },
+    "ghcr.io/devcontainers/features/go:1": {
+      "version": "1.24.2"
+    },
+    "ghcr.io/guiyomh/features/just:0": {},
+    "ghcr.io/devcontainers/features/python:1": { "version": "3.13.3" },
+    "ghcr.io/gvatsal60/dev-container-features/pre-commit:1": {},
+    "ghcr.io/devcontainers-community/npm-features/prettier:1": {}
+  }
+}
--- a/docker/Dockerfile.make
+++ b/docker/Dockerfile.make
@ -1,26 +0,0 @@
-# docker build --rm -f docker/Dockerfile.make -t woodpecker/make:local .
-FROM docker.io/golang:1.24-alpine3.21 AS golang_image
-FROM docker.io/node:23-alpine
-
-RUN apk add --no-cache --update make gcc binutils-gold musl-dev protoc && \
-  corepack enable
-
-# Build packages.
-COPY --from=golang_image /usr/local/go /usr/local/go
-COPY Makefile /
-ENV PATH=$PATH:/usr/local/go/bin
-ENV COREPACK_ENABLE_DOWNLOAD_PROMPT=0
-ENV COREPACK_ENABLE_AUTO_PIN=0
-
-# Cache tools
-RUN GOBIN=/usr/local/go/bin make install-tools && \
-    rm -rf /Makefile
-
-ENV GOPATH=/tmp/go
-ENV HOME=/tmp/home
-ENV PATH=$PATH:/usr/local/go/bin:/tmp/go/bin
-
-WORKDIR /build
-RUN chmod -R 777 /root
-
-CMD [ "/bin/sh" ]
--- a/docs/development/index.md
+++ b/docs/development/index.md
@ -1,9 +1,155 @@
-<!-- markdownlint-disable MD041 -->
+---
+hide:
+  - navigation
+---

-WIP - Please see the [Woodpecker documentation](https://woodpecker-ci.org/docs/intro) in the meantime.
+<!-- markdownlint-disable MD041 -->

 # Development

-## Images
+## Required tools
+
+The following tools are required for development:
+
+| Technology/Tool                                                 | Component/Purpose  | Comment                                    |
+| --------------------------------------------------------------- | ------------------ | ------------------------------------------ |
+| [Go](https://go.dev/dl/)                                        | Backend            |                                            |
+| [pnpm](https://pnpm.io/installation)                            | Frontend           |                                            |
+| [Just](https://github.com/casey/just)                           | Frontend & Backend | modern `make` alternative                  |
+| [mkdocs-material](https://github.com/squidfunk/mkdocs-material) | Docs               |                                            |
+| [Python](https://www.python.org/downloads/)                     | Docs               | `mkdocs` uses Python                       |
+| [fish](https://fishshell.com/)                                  | Docs               | Personal preference, not strictly required |
+| [prettier](https://prettier.io/)                                | Lint               |                                            |
+| [golangci-lint](https://golangci-lint.run/)                     | Lint               |                                            |
+| [pre-commit](https://pre-commit.com/)                           | Lint               | Personal preference, not strictly required |
+| [Docker](https://docs.docker.com/get-docker/)                   | Images             | `podman` is not (yet) supported            |
+
+Once `just` is installed, you can install adjacent golang dependencies using the following command:
+
+```sh
+just install-dev-deps
+```
+
+/// note
+`just install-dev-deps` does not install all dev tools listed above but only go-related ones.
+You are responsible for installing the complete stack of tools and dependencies on your system.
+///
+
+/// tip
+If you are having issues installing all dependencies or have others reasons for not installing them, you can also make use of the `.devcontainers.json` and use this one with services that spin up virtual dev environments, e.g. [devpod.sh](https://devpod.sh/)
+///
+
+The following just rules are helpful for local development:
+
+```sh
+just test-server
+just test-agent
+just test-cli
+just test-server-datastore # test datastore / database related code like migrations
+
+just lint
+just lint-backend
+just lint-frontend
+```
+
+## Building images
+
+/// note
+Building images is not required for local development.
+You only need to build them if you want to test branch builds on existing Crow instances.
+For local development testing, the easiest way is to use the provided `launch.json` config and test the changes in VSCode.
+///
+
+Images for dev purposes can be built using
+
+```sh
+# server
+env PLATFORMS='linux|amd64;linux|arm64' just image-server-alpine <tag>
+
+# agent
+env PLATFORMS='linux/amd64,linux/arm64' just image-agent-alpine <tag>
+
+# cli
+env PLATFORMS='linux/amd64,linux/arm64' just image-cli-alpine <tag>
+```
+
+where `<tag>` is the docker tag to be used for the image.
+You do not need to built multiarch for testing, just choose the architecture you want to test the images on.
+
+## VSCode debug/test configuration
+
+`.vscode/launch.json` holds a complete configuration for starting all Crow components locally.
+Use the "Crow CI" configuration and then access `localhost:8000` in your browser.
+
+To make this all work, some env vars need to be provided in a local `.env` file.
+
+```sh
+export CROW_OPEN=true
+export CROW_ADMIN=<your forge username>
+
+export CROW_HOST=http://localhost:8000
+
+# forge config (forgejo) (see /docs/administration/forge/overview for other forges)
+export CROW_FORGEJO=true
+export CROW_FORGEJO_URL=https://codeberg.org
+export CROW_FORGEJO_CLIENT=<id>
+export CROW_FORGEJO_SECRET=<secret>
+
+# agent
+export CROW_SERVER=localhost:9000
+export CROW_AGENT_SECRET=<agent secret>
+export CROW_MAX_WORKFLOWS=1
+
+# enable if you want to develop the UI
+export CROW_DEV_WWW_PROXY=http://localhost:8010
+
+# if you want to test webhooks with a forge this address needs to be set and accessible from public server
+export CROW_EXPERT_WEBHOOK_HOST=
+
+# disable health-checks while debugging (normally not needed while developing)
+export CROW_HEALTHCHECK=false
+
+# (optional) when running the vite dev server in a container
+# export VITE_DEV_SERVER_HOST=0.0.0.0
+
+export CROW_LOG_LEVEL=debug
+```
+
+Notes:
+
+- Workflows will be run on your local machine using the `docker` backend.
+- The forge-specific settings (`CROW_FORGEJO_URL`, `CROW_FORGEJO_CLIENT`, `CROW_FORGEJO_SECRET`) must be set if you want to test against existing repositories of a forge. You can use an existing application and append `http://localhost:8000/authorize` to the redirect URI list. See [FIXME] for instructions on how to create an oauth2 app for the respective forge.
+- Crow will use a SQLite DB in this scenario, stored at `./crow.sqlite` in the workspace root (and gitignore'd by default). Any backend action you do (e.g. adding secret or enabling a repo) will be persisted in this DB.
+- `CROW_EXPERT_WEBHOOK_HOST` sets the URL which incoming webhooks should target, i.e. the local development server. By default this one is running on localhost and accessible from the public. To make this work, you need to expose the local Crow server to the outside world.
+  To make this work, you can either
+
+      - run the local development server on `0.0.0.0` and expose the port directly to the     outside world (not recommended without https if you're working with an oauth2 app)
+
+      - run a reverse proxy locally which listens for incoming requests, does SSL termination and forwards the requests to the local development server. This is recommended if you're working with an oauth2 app.
+
+Usually, the easiest way is to not use `CROW_EXPERT_WEBHOOK_HOST` and work with `event: manual` instead.
+This event trigger is not dependend on remote webhooks but is triggered from within the Crow server itself.
+
+/// tip
+To test if everything works, you can make a simple change to Crow, e.g. adjust a text in the UI (easiest in `en.json`).
+When you hit the "refresh" button in the VSCode debug tools toolbar, you should see the UI being refreshed with the change on the fly.
+///

 ## Documentation
+
+When working on the documentation, the following commands can be used:
+
+```sh
+# inits and populates the virtual environment
+just docs-venv
+just docs-serve
+```
+
+Crow uses the "insiders" build of `mkdocs-material`, which is only available through a private repository which must be accessed via a token.
+The Justfile rules automatically accounts for this and will download the free version of `mkdocs-material` if the token is not available.
+Most features for development do not rely on the "insiders" build.
+
+/// info
+When you open a PR in the repository, the `surge-preview` plugin will automatically build and deploy the documentation state of the PR to a temporary URL.
+This can be an alternative way to work on the documentation if you are facing troubles getting the local development server to work.
+///
--- a/web/vite.config.ts
+++ b/web/vite.config.ts
@ -14,10 +14,7 @@ function woodpeckerInfoPlugin(): Plugin {
  return {
    name: 'woodpecker-info',
    configureServer() {
-      const info =
-        '1) Please add `WOODPECKER_DEV_WWW_PROXY=http://localhost:8010` to your `.env` file.\n' +
-        'After starting the woodpecker server as well you should now be able to access the UI at http://localhost:8000/\n\n' +
-        '2) If you want to run the vite dev server (`pnpm start`) within a container please set `VITE_DEV_SERVER_HOST=0.0.0.0`.';
+      const info = 'Access the Crow UI at http://localhost:8000/\n\n';
      console.log(info);
    },
  };