1
0
mirror of https://github.com/owncloud/ocis.git synced 2026-10-23 23:29:02 +03:00

feat: drop drone (#12243)

This commit is contained in:
Michal Klos
2026-05-13 12:14:45 +02:00
committed by GitHub
parent d2843a8fbb
commit b2adb15d6d
60 changed files with 546 additions and 5369 deletions

View File

@@ -1,3 +0,0 @@
# The test runner source for UI tests
WEB_COMMITID=39d6215da6daa184cd2365966915a659bdbd2c8b
WEB_BRANCH=stable-12.3

File diff suppressed because it is too large Load Diff

1
.github/CODEOWNERS vendored
View File

@@ -1,4 +1,3 @@
services/settings/ @lukashirt
services/web/ @lukashirt
.drone.star @kobergj @phil-davis @individual-it
assets/End-User-License-Agreement-for-ownCloud-Infinite-Scale.pdf @kobergj @hodyroff

View File

@@ -8,7 +8,6 @@
* [ ] DEV/QA: Check new strings and align with clients
* [ ] DEV/DOCS: Create list of pending docs tasks
* [ ] DEV: Create branch `release-x.x.x-rc.x` -> CODEFREEZE
* [ ] DEV: specify if it is a production release in [.drone.star](https://github.com/owncloud/ocis/blob/b4cf38fa1ba180c58519026dfe762b7c45695466/.drone.star#L8) (if needed)
* [ ] DEV: bump ocis version in necessary files
* [ ] DEV: `changelog/CHANGELOG.tmpl`
* [ ] DEV: `ocis-pkg/version/version.go`

View File

@@ -219,7 +219,7 @@ jobs:
fail-fast: false
matrix:
suite:
- cliCommands,apiServiceAvailability # grouped like drone; needs ClamAV + email
- cliCommands,apiServiceAvailability # grouped: both need ClamAV + email services
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
@@ -375,7 +375,7 @@ jobs:
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
with:
path: ~/.cache/ms-playwright
key: playwright-chromium-${{ hashFiles('.drone.env') }}
key: playwright-chromium-${{ hashFiles('ci.env') }}
# --- Tika (search suite only) ---
- name: Start Tika
@@ -409,9 +409,10 @@ jobs:
- name: Start Keycloak
if: matrix.keycloak == true
run: |
# Patch realm: replace Drone Docker hostname with localhost IP
# Patch realm: ocis-ci-realm.dist.json uses ocis-server:9200 as hostname.
# GitHub Actions runs on the host network so patch it to 127.0.0.1.
sed 's|https://ocis-server:9200|https://127.0.0.1:9200|g' \
tests/config/drone/ocis-ci-realm.dist.json > /tmp/ocis-realm.json
tests/config/ci/ocis-ci-realm.dist.json > /tmp/ocis-realm.json
docker run -d --name keycloak --network host \
-e OCIS_DOMAIN=https://127.0.0.1:9200 \
-e KC_HOSTNAME=localhost \

View File

@@ -24,9 +24,8 @@ jobs:
working-directory: tools/ci-reporter
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Review the pipelines in last 14 days before the dorne logs expire
run: |
GOWORK=off go run ./cmd/list-failed-commits \
GOWORK=off go run ./cmd/list-flaky-tests \
--github-token "$GITHUB_TOKEN" \
--since 14d \
--max-commits 9999999
--workflow acceptance-tests.yml \
--since 4w

2
.gitignore vendored
View File

@@ -45,8 +45,6 @@ vendor-php
# QA activity reports
tests/qa-activity-report/reports/
# drone CI is in .drone.star, do not let someone accidentally commit a local .drone.yml
.drone.yml
**/l10n/template.pot

View File

@@ -20,7 +20,6 @@ L10N_MODULES := \
services/userlog \
services/settings
# if you add a module here please also add it to the .drone.star file
OCIS_MODULES = \
services/activitylog \
services/antivirus \
@@ -206,7 +205,10 @@ docs-serve:
docs-clean:
@$(MAKE) --no-print-directory -C docs docs-clean
# imitate a full drone run locally to build docs without pushing to the web.
# docs-local imitates the 3-step docs build pipeline (generate → copy → build)
# locally without pushing. docs-hugo-drone-prep creates the ../hugo symlink required by
# the Hugo Docker image (hugomods/hugo:base-0.129.0).
# imitate a full docs build run locally without pushing to the web.
# this can help identify uncaught issues when running `make docs-serve` only.
.PHONY: docs-local # run all steps as drone would do it (1, 2, 3)
docs-local:
@@ -365,9 +367,6 @@ l10n-write:
$(MAKE) -C $$extension l10n-write || exit 1; \
done
.PHONY: ci-format
ci-format: $(BUILDIFIER)
$(BUILDIFIER) --mode=fix .drone.star
.PHONY: test-php-style
test-php-style: vendor-bin/owncloud-codestyle/vendor vendor-bin/php_codesniffer/vendor

View File

@@ -1,7 +1,7 @@
# ownCloud Infinite Scale
[![Matrix](https://img.shields.io/matrix/ocis%3Amatrix.org?logo=matrix)](https://app.element.io/#/room/#ocis:matrix.org)
[![Build Status](https://drone.owncloud.com/api/badges/owncloud/ocis/status.svg)](https://drone.owncloud.com/owncloud/ocis)
[![Build Status](https://github.com/owncloud/ocis/actions/workflows/acceptance-tests.yml/badge.svg)](https://github.com/owncloud/ocis/actions/workflows/acceptance-tests.yml)
[![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=owncloud_ocis&metric=security_rating)](https://sonarcloud.io/dashboard?id=owncloud_ocis)
[![Coverage](https://sonarcloud.io/api/project_badges/measure?project=owncloud_ocis&metric=coverage)](https://sonarcloud.io/dashboard?id=owncloud_ocis)
[![Acceptance Test Coverage](https://sonarcloud.io/api/project_badges/measure?project=owncloud-1_ocis_acceptance-tests&metric=coverage)](https://sonarcloud.io/summary/new_code?id=owncloud-1_ocis_acceptance-tests)

4
ci.env Normal file
View File

@@ -0,0 +1,4 @@
# Pins the ownCloud Web frontend version used in CI tests.
# Read by: tests/acceptance/run-e2e.py and .github/workflows/acceptance-tests.yml (Playwright cache key).
WEB_COMMITID=6165b4e47a47e3a216107e96f90825190f10632e
WEB_BRANCH=stable-12.3

View File

@@ -37,7 +37,8 @@ help:
.PHONY:list
list: help
# for drone only. drone uses an image for hugo that needs the hugo folder in the root
# Creates a symlink ../hugo → hugo/ required by the Hugo Docker image
# (hugomods/hugo:base-0.129.0), which expects the hugo folder at the repo root.
# CURDIR will work for make only but not when running manually in the shell
.PHONY: docs-hugo-drone-prep
docs-hugo-drone-prep:

View File

@@ -40,7 +40,7 @@ variables:
description: ""
do_ignore: true
- rawname: GITHUB_TOKEN
path: tools/ci-reporter/cmd/list-failed-commits/main.go:116
path: tools/ci-reporter/cmd/list-flaky-tests/main.go:64
foundincode: true
name: GITHUB_TOKEN
type: ""

View File

@@ -15,7 +15,7 @@ The documentation:
* Is based on the [HUGO](https://gohugo.io/) framework written in Go. The underlaying language is markdown plus the extensions provided by hugo.
* The framwork including the theme used is maintained in the following [repo](https://github.com/owncloud/owncloud.github.io).
* The hugo version, currently at `0.129.0` must match the image version defined in the root makefile and the framework repo, defined in `.drone.yml`.\
* The hugo version, currently at `0.129.0` must match the image version defined in the root makefile and the framework repo.\
{{< hint info >}}
* Building the theme is part of the loaded framework while building the documentation is done using a separate container.
@@ -24,7 +24,7 @@ The documentation:
* The documentation sources are distributed over several repos such as `ocis`, `web` and others. When building, these sources act individually and only those sources are build and pushed to the web that come from the respective repo. The definition what is going where is defined in the [framework repo](https://github.com/owncloud/owncloud.github.io/blob/main/.batchfile). This means, each merge of a docs change in one of the sourcing repos only pushes that change to the site. There is no global build process and inter doc links must be treated therefore very carefully !
* The documentation setup is, with only a few exceptions, identical for both the `ocis` and the `web` repo. This means, for example, running commands locally are the same, the documentation process section below applies for both; only drone uses some specifics.
* The documentation setup is, with only a few exceptions, identical for both the `ocis` and the `web` repo. This means, for example, running commands locally are the same, the documentation process section below applies for both.
* {{< hint info >}}
Any files or folders that need to be excluded from the build process must be defined in the referenced framework repository above, in the file `config.yaml`.

View File

@@ -9,48 +9,47 @@ geekdocFilePath: continuous-integration.md
{{< toc >}}
oCIS uses [DRONE](https://www.drone.io/) as CI system. You can find the pipeline logs [here](https://drone.owncloud.com/owncloud/ocis) or in your PR.
## Overview
oCIS uses [GitHub Actions](https://github.com/owncloud/ocis/actions) as its CI system. Pipeline logs are visible directly in pull requests.
## Concepts
The pipeline is defined in [Starlark](https://github.com/bazelbuild/starlark) and transformed to YAML upon pipeline run. This enables us to do a highly dynamic and non repeating pipeline configuration. We enforce Starlark format guidelines with Bazel Buildifier. You can format the .drone.star file by running `make ci-format`.
Pipelines are defined in `.github/workflows/`. The main acceptance test workflow is `.github/workflows/acceptance-tests.yml`.
Upon running the pipeline, your branch gets merged to the master branch. This ensures that we always test your changeset if as it was applied to the master of oCIS. Please note that this does not apply to the pipeline definition (`.drone.star`).
Upon running the pipeline, tests run against the branch as-is. Make sure your branch is rebased onto the latest master before running CI to avoid false failures.
## Things done in CI
- static code analysis
- linting
- running UI tests
- running ownCloud 10 test suite against oCIS
- running API acceptance tests
- build and release docker images
- build and release binaries
- build and release documentation
## Flags in commit message and PR title
You may add flags to your commit message or PR title in order to speed up pipeline runs and take load from the CI runners.
You may add flags to your commit message or PR title to control pipeline runs.
- `[CI SKIP]`: no CI is run on the commit or PR
- `[full-ci]`: deactivates the fail early mechanism and runs all available test (as default only smoke tests are run)
- `[k6-test]`: enabled the run of the k6 with smoke and performance test
### Knowledge base
- My pipeline fails because some CI related files or commands are missing.
Please make sure to rebase your branch onto the latest master of oCIS. It could be that the pipeline definition (`.drone.star`) was changed on the master branch. This is the only file, that will not be auto merged to master upon pipeline run. So things could be out of sync.
Make sure your branch is rebased onto the latest master. Workflow files in `.github/workflows/` change over time and a stale branch may reference steps or configs that no longer exist.
- How can I see the YAML drone pipeline definition?
- How can I see what a workflow does?
In order to see the Yaml pipeline definition you can use the drone-cli to convert the Starlark file.
Workflow definitions are plain YAML in `.github/workflows/`. Open the relevant file directly in the repo — no CLI tool needed. The main acceptance test entry point is `.github/workflows/acceptance-tests.yml`.
- How can I re-run a failed job?
In the GitHub Actions UI, click **Re-run jobs** → **Re-run failed jobs** on the failed workflow run. You can also trigger a full re-run from `gh`:
```bash
gh run rerun <run-id> --failed --repo owncloud/ocis
```
drone starlark
```
{{< hint info >}}
If you experience a `"build" struct has no .title attribute` you need a newer version of drone-cli.
You currently need to build it yourself from this [source code](https://github.com/drone/drone-cli). If you are not using master as source, please ensure that this [PR](https://github.com/drone/drone-cli/pull/175) is included.
{{< /hint >}}

View File

@@ -581,11 +581,11 @@ THUMBNAILS_TXT_FONTMAP_FILE="/path/to/fontsMap.json"
ocis/bin/ocis server
```
The sample `fontsMap.json` file is located in `tests/config/drone/fontsMap.json`.
The sample `fontsMap.json` file is located in `tests/config/ci/fontsMap.json`.
```json
{
"defaultFont": "/path/to/ocis/tests/config/drone/NotoSans.ttf"
"defaultFont": "/path/to/ocis/tests/config/ci/NotoSans.ttf"
}
```

View File

@@ -108,10 +108,7 @@ https://owncloud.dev/ocis/releasing_guide/
* [ ] Create orphan branch: `git checkout --orphan docs-stable-7.2`
* [ ] Initial commit: `git commit --allow-empty -m "initial commit"`
* [ ] Push it: `git push`
* [ ] Adjust the `.drone.star` file to write to `docs-stable-7.2`
* [ ] Find target_branch value in the docs section and change it to `docs-stable-7.2`
* [ ] Example: https://github.com/owncloud/ocis/blame/56f7645f0b11c9112e15ce46f6effd2fea01d6be/.drone.star#L2249
* [ ] Add `stable-7.2` to the nightly cron jobs in drone (`Settings` -> `Cron Jobs`)
* [ ] Add `stable-7.2` to the nightly cron jobs in GitHub Actions (`Settings` -> `Schedules`)
```
### Steps for Patch Releases Only

View File

@@ -16,12 +16,7 @@ Services can have texts that need to be translated. These translations will be s
The process for _synchronisation_ with Transifex is already setup and nothing needs to be done here. For any translation, it is necessary to set it up in the respective service and tell to sync it.
**IMPORTANT**\
Translations are automatically synced on a daily basis in the night. To do so, there is an own repo that covers the process for ALL translations from all configured repos: [translation-sync](https://github.com/owncloud/translation-sync). If there is a manual "emergency" sync necessary, you only need to trigger [drone](https://drone.owncloud.com/owncloud/translation-sync) via cli
```bash
drone cron exec owncloud/translation-sync nightly
```
Note that you need to be logged on in drone to execute the command.
Translations are automatically synced on a daily basis in the night. To do so, there is an own repo that covers the process for ALL translations from all configured repos: [translation-sync](https://github.com/owncloud/translation-sync). If there is a manual "emergency" sync necessary, trigger the workflow manually via the [GitHub Actions UI](https://github.com/owncloud/translation-sync/actions) using `workflow_dispatch`.
## Implementing ocis Translations

View File

@@ -36,7 +36,7 @@ Use this checklist with copy/paste in your PR - right from the beginning. It ren
- Add the service config to `ocis-pkg/config/defaultconfig.go`
- [ ] If the service is using service accounts, add it to `ocis/pkg/init/init.go`
- [ ] Check that the service properly responds to `ocis <service-name> health` and has `/healthz` and `/readyz` endpoints
- [ ] Add the service to `.drone.star` to enable CI.
- [ ] Add the service suite to the GitHub Actions matrix in `.github/workflows/acceptance-tests.yml` to enable CI.
- [ ] Inform doc team in an *early stage* to review the readme AND the environment variables created.
- The description must reflect the behaviour AND usually has a positive code quality impact.
- [ ] Create proper description strings for envvars - see other services for examples, especially when it comes to multiple values. This must include:

View File

@@ -32,6 +32,6 @@ and take note of its release tag name.
2. Change into web package folder via `cd web`
3. Inside `web/`, update the `Makefile` so that the WEB_ASSETS_VERSION variable references the currently released version of https://github.com/owncloud/web
4. Move to the changelog (`cd ../changelog/`) and add a changelog file to the `unreleased/` folder (You can copy an old web release changelog item as a template)
5. Move to the repo root (`cd ..`)and update the WEB_COMMITID in the `/.drone.env` file to the commit id from the released version (unless the existing commit id is already newer)
5. Move to the repo root (`cd ..`) and update the `WEB_COMMITID` in `/ci.env` to the commit id from the released version (unless the existing commit id is already newer)
6. **Optional:** Test the changes locally by running `cd ocis && go run cmd/ocis/main.go server`, visiting [https://localhost:9200](https://localhost:9200) and confirming everything renders correctly
7. Commit your changes, push them and [create a PR](https://github.com/owncloud/ocis/pulls)

View File

@@ -1,10 +1,10 @@
services:
acceptance-tests:
image: owncloudci/php:8.2
working_dir: /drone/src
working_dir: /ocis
command: /bin/bash /test/run-tests.sh
environment:
OCIS_ROOT: /drone/src
OCIS_ROOT: /ocis
TEST_SERVER_URL: https://ocis-server:9200
OCIS_WRAPPER_URL: http://ocis-server:5200
STORAGE_DRIVER: $STORAGE_DRIVER
@@ -17,8 +17,8 @@ services:
EMAIL_HOST: email
EMAIL_PORT: 8025
env_file:
- ../../../../.drone.env
- ../../../../ci.env
volumes:
- ./run-tests.sh:/test/run-tests.sh
- ../../../../:/drone/src
- ../../../../:/ocis
- ../../../../vendor-bin/behat/composer.json:/tmp/vendor-bin/behat/composer.json

View File

@@ -15,7 +15,7 @@ services:
STORAGE_SYSTEM_DRIVER_OCIS_ROOT: /srv/app/tmp/ocis/storage/metadata
SHARING_USER_JSON_FILE: /srv/app/tmp/ocis/shares.json
PROXY_ENABLE_BASIC_AUTH: "true"
WEB_UI_CONFIG_FILE: /drone/src/tests/config/drone/ocis-config.json
WEB_UI_CONFIG_FILE: /ocis/tests/config/ci/ocis-config.json
OCIS_INSECURE: "true"
IDM_CREATE_DEMO_USERS: "true"
IDM_ADMIN_PASSWORD: "admin"
@@ -48,7 +48,7 @@ services:
SEARCH_EXTRACTOR_CS3SOURCE_INSECURE: "true"
# fonts map for txt thumbnails (including unicode support)
THUMBNAILS_TXT_FONTMAP_FILE: "/drone/src/tests/config/drone/fontsMap.json"
THUMBNAILS_TXT_FONTMAP_FILE: "/ocis/tests/config/ci/fontsMap.json"
volumes:
- ../../../config:/drone/src/tests/config
- ../../../config:/ocis/tests/config
- ../../../ociswrapper/bin/ociswrapper:/usr/bin/ociswrapper

View File

@@ -1,8 +1,5 @@
#!/bin/bash
#mkdir -p /drone/src/vendor-bin/behat
#cp /tmp/vendor-bin/behat/composer.json /drone/src/vendor-bin/behat/composer.json
git config --global advice.detachedHead false
## CONFIGURE TEST
@@ -12,7 +9,7 @@ if [ "$TEST_SOURCE" = "oc10" ]; then
if [ "$STORAGE_DRIVER" = "ocis" ]; then
export OCIS_REVA_DATA_ROOT=''
export BEHAT_FILTER_TAGS='~@skipOnOcis-OCIS-Storage'
export EXPECTED_FAILURES_FILE='/drone/src/tests/acceptance/expected-failures-API-on-OCIS-storage.md'
export EXPECTED_FAILURES_FILE='/ocis/tests/acceptance/expected-failures-API-on-OCIS-storage.md'
elif [ "$STORAGE_DRIVER" = "s3ng" ]; then
export BEHAT_FILTER_TAGS='~@skip&&~@skipOnOcis-S3NG-Storage'
export OCIS_REVA_DATA_ROOT=''

View File

@@ -286,7 +286,7 @@ Feature: Link sharing of project spaces
@env-config
Scenario: try to create a link share of a project-space with a password that is listed in the Banned-Password-List using permissions endpoint
Given using spaces DAV path
And the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/drone/banned-password-list.txt" for "sharing" service
And the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/ci/banned-password-list.txt" for "sharing" service
And the administrator has assigned the role "Space Admin" to user "Alice" using the Graph API
And user "Alice" has created a space "projectSpace" with the default quota using the Graph API
When user "Alice" creates the following space link share using permissions endpoint of the Graph API:
@@ -315,7 +315,7 @@ Feature: Link sharing of project spaces
@env-config
Scenario: try to create a link share of a project-space with a password that is listed in the Banned-Password-List using root endpoint
Given using spaces DAV path
And the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/drone/banned-password-list.txt" for "sharing" service
And the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/ci/banned-password-list.txt" for "sharing" service
And the administrator has assigned the role "Space Admin" to user "Alice" using the Graph API
And user "Alice" has created a space "projectSpace" with the default quota using the Graph API
When user "Alice" creates the following space link share using root endpoint of the Graph API:

View File

@@ -6,7 +6,7 @@ Feature: Link share with banned password
Given these users have been created with default attributes:
| username |
| Alice |
And the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/drone/banned-password-list.txt" for "sharing" service
And the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/ci/banned-password-list.txt" for "sharing" service
Scenario: try to create a folder link share with a password that is listed in the Banned-Password-List

View File

@@ -237,7 +237,7 @@ Feature: Update a link share for a resource
Scenario Outline: update a file's link share with a password that is listed in the Banned-Password-List using permissions endpoint
Given the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/drone/banned-password-list.txt" for "sharing" service
Given the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/ci/banned-password-list.txt" for "sharing" service
And user "Alice" has uploaded file with content "other data" to "text.txt"
And user "Alice" has created the following resource link share:
| resource | text.txt |

View File

@@ -283,8 +283,8 @@ Feature: enforce password on public link
Scenario Outline: update a public link with a password that is listed in the Banned-Password-List
Given the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/drone/banned-password-list.txt" for "sharing" service
And the config "FRONTEND_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/drone/banned-password-list.txt" for "frontend" service
Given the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/ci/banned-password-list.txt" for "sharing" service
And the config "FRONTEND_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/ci/banned-password-list.txt" for "frontend" service
And using OCS API version "2"
And user "Alice" has been created with default attributes
And user "Alice" has uploaded file with content "test file" to "/testfile.txt"
@@ -307,8 +307,8 @@ Feature: enforce password on public link
Scenario Outline: create a public link with a password that is listed in the Banned-Password-List
Given the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/drone/banned-password-list.txt" for "sharing" service
And the config "FRONTEND_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/drone/banned-password-list.txt" for "frontend" service
Given the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/ci/banned-password-list.txt" for "sharing" service
And the config "FRONTEND_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/ci/banned-password-list.txt" for "frontend" service
And using OCS API version "2"
And user "Alice" has been created with default attributes
And user "Alice" has uploaded file with content "test file" to "/testfile.txt"

View File

@@ -2,7 +2,6 @@
"""
Run CS3 API validator tests locally and in GitHub Actions CI.
Config sourced from .drone.star cs3ApiTests() — single source of truth.
Usage: python3 tests/acceptance/run-cs3api.py
"""
@@ -15,7 +14,7 @@ import time
from pathlib import Path
# ---------------------------------------------------------------------------
# Constants (mirroring .drone.star)
# Constants
# ---------------------------------------------------------------------------
# HTTPS — matching drone: ocis init generates a self-signed cert; proxy uses TLS by default.
@@ -54,7 +53,7 @@ def base_server_env(repo_root: Path, ocis_config_dir: str, ocis_public_url: str)
"NATS_NATS_PORT": "9233",
"OCIS_JWT_SECRET": "some-ocis-jwt-secret",
"EVENTHISTORY_STORE": "memory",
"WEB_UI_CONFIG_FILE": str(repo_root / "tests/config/drone/ocis-config.json"),
"WEB_UI_CONFIG_FILE": str(repo_root / "tests/config/ci/ocis-config.json"),
# cs3api_validator extras (drone ocisServer deploy_type="cs3api_validator")
"GATEWAY_GRPC_ADDR": "0.0.0.0:9142",
"OCIS_SHARING_PUBLIC_SHARE_MUST_HAVE_PASSWORD": "false",
@@ -102,7 +101,7 @@ def main() -> int:
check=True,
)
shutil.copy(
repo_root / "tests/config/drone/app-registry.yaml",
repo_root / "tests/config/ci/app-registry.yaml",
ocis_config_dir / "app-registry.yaml",
)

View File

@@ -73,15 +73,15 @@ def main() -> int:
# clone + install web only if not already provided (e.g. via artifact)
if not web_dir.exists():
drone_env = {}
drone_env_file = repo_root / ".drone.env"
if drone_env_file.exists():
for line in drone_env_file.read_text().splitlines():
ci_env = {}
ci_env_file = repo_root / "ci.env"
if ci_env_file.exists():
for line in ci_env_file.read_text().splitlines():
if "=" in line and not line.startswith("#"):
k, v = line.split("=", 1)
drone_env[k.strip()] = v.strip()
web_branch = drone_env.get("WEB_BRANCH", "master")
web_commitid = drone_env.get("WEB_COMMITID", "")
ci_env[k.strip()] = v.strip()
web_branch = ci_env.get("WEB_BRANCH", "master")
web_commitid = ci_env.get("WEB_COMMITID", "")
run(["git", "clone", "-b", web_branch, "--single-branch", "--no-tags",
WEB_REPO, str(web_dir)])
if web_commitid:
@@ -99,7 +99,7 @@ def main() -> int:
# init ocis
run([str(ocis_bin), "init", "--insecure", "true"])
shutil.copy(
repo_root / "tests/config/drone/app-registry.yaml",
repo_root / "tests/config/ci/app-registry.yaml",
ocis_config_dir / "app-registry.yaml",
)
@@ -115,7 +115,7 @@ def main() -> int:
# Patch web UI config: replace Drone Docker service name with our URL
drone_web_cfg = json.loads(
(repo_root / "tests/config/drone/ocis-config.json").read_text()
(repo_root / "tests/config/ci/ocis-config.json").read_text()
)
def _patch_urls(obj, old, new):
@@ -151,9 +151,9 @@ def main() -> int:
"EVENTHISTORY_STORE": "memory",
"OCIS_TRANSLATION_PATH": str(repo_root / "tests/config/translations"),
"WEB_UI_CONFIG_FILE": str(gha_web_cfg_path),
"THUMBNAILS_TXT_FONTMAP_FILE": str(repo_root / "tests/config/drone/fontsMap.json"),
"THUMBNAILS_TXT_FONTMAP_FILE": str(repo_root / "tests/config/ci/fontsMap.json"),
# extra_server_environment — matches drone e2eTestPipeline()
"OCIS_PASSWORD_POLICY_BANNED_PASSWORDS_LIST": str(repo_root / "tests/config/drone/banned-password-list.txt"),
"OCIS_PASSWORD_POLICY_BANNED_PASSWORDS_LIST": str(repo_root / "tests/config/ci/banned-password-list.txt"),
"GRAPH_AVAILABLE_ROLES": "b1e2218d-eef8-4d4c-b82d-0f1a1b48f3b5,a8d5fe5e-96e3-418d-825b-534dbdf22b99,fb6c3e19-e378-47e5-b277-9732f9de6e21,58c63c02-1d89-4572-916a-870abc5a1b7d,2d00ce52-1fc2-4dbc-8b95-a73b73395f5a,1c996275-f1c9-4e71-abdf-a42f6495e960,312c0871-5ef7-4b3a-85b6-0e4074c64049,aa97fe03-7980-45ac-9e50-b325749fd7e6,63e64e19-8d43-42ec-a738-2b6af2610efa",
"FRONTEND_CONFIGURABLE_NOTIFICATIONS": "true",
# debug addresses
@@ -215,7 +215,7 @@ def main() -> int:
"OCIS_ADMIN_USER_ID": "",
"GRAPH_ASSIGN_DEFAULT_USER_ROLE": "false",
"GRAPH_USERNAME_MATCH": "none",
"PROXY_CSP_CONFIG_FILE_LOCATION": str(repo_root / "tests/config/drone/csp.yaml"),
"PROXY_CSP_CONFIG_FILE_LOCATION": str(repo_root / "tests/config/ci/csp.yaml"),
"KEYCLOAK_DOMAIN": "localhost:8443",
"IDM_CREATE_DEMO_USERS": "false",
})

View File

@@ -2,7 +2,6 @@
"""
Run ocis acceptance tests locally and in GitHub Actions CI.
Config sourced from .drone.star localApiTests — single source of truth.
Usage: BEHAT_SUITES=apiGraph python3 tests/acceptance/run-github.py
"""
@@ -17,7 +16,7 @@ import shutil
from pathlib import Path
# ---------------------------------------------------------------------------
# Config sourced from .drone.star
# NOTE: EMAIL_SMTP_HOST is "email" (container name) in drone, "localhost" here
# ---------------------------------------------------------------------------
@@ -216,8 +215,8 @@ def base_server_env(repo_root: Path, ocis_url: str, ocis_config_dir: str) -> dic
"OCIS_JWT_SECRET": "some-ocis-jwt-secret",
"EVENTHISTORY_STORE": "memory",
"OCIS_TRANSLATION_PATH": str(repo_root / "tests/config/translations"),
"WEB_UI_CONFIG_FILE": str(repo_root / "tests/config/drone/ocis-config.json"),
"THUMBNAILS_TXT_FONTMAP_FILE": str(repo_root / "tests/config/drone/fontsMap.json"),
"WEB_UI_CONFIG_FILE": str(repo_root / "tests/config/ci/ocis-config.json"),
"THUMBNAILS_TXT_FONTMAP_FILE": str(repo_root / "tests/config/ci/fontsMap.json"),
# default tika off (overridden by search2 extraServerEnvironment)
"SEARCH_EXTRACTOR_TYPE": "basic",
"FRONTEND_FULL_TEXT_SEARCH_ENABLED": "false",
@@ -417,7 +416,7 @@ def main() -> int:
# OCM federation: rewrite providers.json with localhost URLs
if cfg["federationServer"]:
providers_src = repo_root / "tests/config/drone/providers.json"
providers_src = repo_root / "tests/config/ci/providers.json"
providers = json.loads(providers_src.read_text())
for p in providers:
# replace container DNS names with localhost
@@ -438,12 +437,12 @@ def main() -> int:
# init ocis
run([str(ocis_bin), "init", "--insecure", "true"])
shutil.copy(
repo_root / "tests/config/drone/app-registry.yaml",
repo_root / "tests/config/ci/app-registry.yaml",
ocis_config_dir / "app-registry.yaml",
)
# generate fontsMap.json with correct font path (drone hardcodes /drone/src/...)
font_path = str(repo_root / "tests/config/drone/NotoSans.ttf")
# generate fontsMap.json with the correct absolute font path for this runner
font_path = str(repo_root / "tests/config/ci/NotoSans.ttf")
fontmap_tmp = tempfile.NamedTemporaryFile(
mode="w", suffix=".json", prefix="fontsMap-", delete=False)
json.dump({"defaultFont": font_path}, fontmap_tmp)
@@ -491,7 +490,7 @@ def main() -> int:
run([str(ocis_bin), "init", "--insecure", "true",
"--config-path", str(fed_config_dir)])
shutil.copy(
repo_root / "tests/config/drone/app-registry.yaml",
repo_root / "tests/config/ci/app-registry.yaml",
fed_config_dir / "app-registry.yaml",
)
@@ -513,11 +512,11 @@ def main() -> int:
# → healthz never binds → 300s timeout. Use Python's built-in HTTP server
# instead; it handles concurrent connections without gaps.
run(["docker", "run", "-d", "--name", "fakeoffice", "--network", "host",
"-v", f"{repo_root}:/drone/src:ro",
"-v", f"{repo_root}:/ocis:ro",
"python:3-alpine",
"python3", "-c",
"import http.server, pathlib\n"
"body = pathlib.Path('/drone/src/tests/config/drone/hosting-discovery.xml').read_bytes()\n"
"body = pathlib.Path('/ocis/tests/config/ci/hosting-discovery.xml').read_bytes()\n"
"class H(http.server.BaseHTTPRequestHandler):\n"
" def do_GET(self):\n"
" self.send_response(200)\n"
@@ -552,7 +551,7 @@ def main() -> int:
# Drone avoids this because each service has its own network namespace.
subprocess.run(["sudo", "systemctl", "stop", "postgresql"],
capture_output=True)
only_office_json = repo_root / "tests/config/drone/only-office.json"
only_office_json = repo_root / "tests/config/ci/only-office.json"
run(["docker", "run", "-d", "--name", "onlyoffice", "--network", "host",
"-e", "WOPI_ENABLED=true",
"-e", "USE_UNAUTHORIZED_STORAGE=true",

View File

@@ -2,7 +2,6 @@
"""
Run litmus WebDAV compliance tests locally and in GitHub Actions CI.
Config sourced from .drone.star litmus() / setupForLitmus() — single source of truth.
Usage: python3 tests/acceptance/run-litmus.py
"""
@@ -17,7 +16,7 @@ import time
from pathlib import Path
# ---------------------------------------------------------------------------
# Constants (mirroring .drone.star)
# Constants
# ---------------------------------------------------------------------------
# HTTPS — matching drone: ocis init generates a self-signed cert; proxy uses TLS by default.
@@ -57,7 +56,7 @@ def base_server_env(repo_root: Path, ocis_config_dir: str, ocis_public_url: str)
"NATS_NATS_PORT": "9233",
"OCIS_JWT_SECRET": "some-ocis-jwt-secret",
"EVENTHISTORY_STORE": "memory",
"WEB_UI_CONFIG_FILE": str(repo_root / "tests/config/drone/ocis-config.json"),
"WEB_UI_CONFIG_FILE": str(repo_root / "tests/config/ci/ocis-config.json"),
}
@@ -82,7 +81,7 @@ def ocis_healthy(ocis_url: str) -> bool:
def setup_for_litmus(ocis_url: str) -> tuple:
"""
Translate tests/config/drone/setup-for-litmus.sh to Python.
Translate tests/config/ci/setup-for-litmus.sh to Python.
Returns (space_id, public_token).
"""
# get personal space ID
@@ -189,7 +188,7 @@ def main() -> int:
check=True,
)
shutil.copy(
repo_root / "tests/config/drone/app-registry.yaml",
repo_root / "tests/config/ci/app-registry.yaml",
ocis_config_dir / "app-registry.yaml",
)

View File

@@ -2,7 +2,6 @@
"""
Run WOPI validator tests locally and in GitHub Actions CI.
Config sourced from .drone.star wopiValidatorTests() — single source of truth.
Usage: python3 tests/acceptance/run-wopi.py --type builtin
python3 tests/acceptance/run-wopi.py --type cs3
"""
@@ -21,7 +20,7 @@ import urllib.parse
from pathlib import Path
# ---------------------------------------------------------------------------
# Constants (mirroring .drone.star)
# Constants
# ---------------------------------------------------------------------------
# HTTPS — matching drone; host-side curl calls use -k.
@@ -119,7 +118,7 @@ def base_server_env(repo_root: Path, ocis_config_dir: str, ocis_public_url: str,
"NATS_NATS_PORT": "9233",
"OCIS_JWT_SECRET": "some-ocis-jwt-secret",
"EVENTHISTORY_STORE": "memory",
"WEB_UI_CONFIG_FILE": str(repo_root / "tests/config/drone/ocis-config.json"),
"WEB_UI_CONFIG_FILE": str(repo_root / "tests/config/ci/ocis-config.json"),
# wopi_validator extras (drone ocisServer deploy_type="wopi_validator")
"GATEWAY_GRPC_ADDR": "0.0.0.0:9142",
"APP_PROVIDER_EXTERNAL_ADDR": "com.owncloud.api.app-provider",
@@ -265,16 +264,15 @@ def main() -> int:
try:
# --- fakeoffice: serves hosting-discovery.xml on :8080 ---
# Mirrors drone fakeOffice() — owncloudci/alpine running serve-hosting-discovery.sh.
# Repo is mounted at /drone/src (the path the script uses).
# Mirrors the old fakeOffice() step — owncloudci/alpine running serve-hosting-discovery.sh.
containers.append("wopi-fakeoffice-fake")
subprocess.run(["docker", "rm", "-f", "wopi-fakeoffice-fake"], capture_output=True)
subprocess.run([
"docker", "run", "-d", "--name", "wopi-fakeoffice-fake",
"-p", "8080:8080",
"-v", f"{repo_root}:/drone/src",
"-v", f"{repo_root}:/ocis",
FAKEOFFICE_IMAGE,
"sh", "/drone/src/tests/config/drone/serve-hosting-discovery.sh",
"sh", "/ocis/tests/config/ci/serve-hosting-discovery.sh",
], check=True)
wait_for(lambda: tcp_reachable(bridge_ip, 8080), 60, "fakeoffice:8080")
@@ -291,7 +289,7 @@ def main() -> int:
env=server_env, check=True,
)
shutil.copy(
repo_root / "tests/config/drone/app-registry.yaml",
repo_root / "tests/config/ci/app-registry.yaml",
ocis_config_dir / "app-registry.yaml",
)
@@ -324,7 +322,7 @@ def main() -> int:
else:
# cs3: patch wopiserver.conf (replace container hostname with bridge_ip),
# then run cs3org/wopiserver as a Docker container.
conf_text = (repo_root / "tests/config/drone/wopiserver.conf").read_text()
conf_text = (repo_root / "tests/config/ci/wopiserver.conf").read_text()
conf_text = conf_text.replace("ocis-server", bridge_ip)
conf_tmp = Path("/tmp/wopiserver-patched.conf")
conf_tmp.write_text(conf_text)

View File

@@ -677,18 +677,4 @@ then
${RED_COLOR}; printf "%s\n" "${UNEXPECTED_BEHAT_EXIT_STATUSES[@]}"
fi
# # sync the file-system so all output will be flushed to storage.
# # In drone we sometimes see that the last lines of output are missing from the
# # drone log.
# sync
# # If we are running in drone CI, then sleep for a bit to (hopefully) let the
# # drone agent send all the output to the drone server.
# NOTE: Not using this anymore, please enable it if needed
# if [ -n "${CI_REPO}" ]
# then
# echo "sleeping for 30 seconds at end of test run"
# sleep 30
# fi
exit ${FINAL_EXIT_STATUS}

View File

@@ -7,7 +7,7 @@
ROOT_PATH="$1"
if [ -z "$1" ]; then
ROOT_PATH="/drone/src"
ROOT_PATH="."
fi
BINGO_DIR="$ROOT_PATH/.bingo"
@@ -19,8 +19,6 @@ go_cache=$(mc find s3/$CACHE_BUCKET/ocis/go-bin/$BINGO_HASH/$2 2>&1 | grep 'Obje
if [[ -z "$go_cache" ]]
then
echo "[INFO] Go bin cache with has '$BINGO_HASH' exists."
# https://discourse.drone.io/t/how-to-exit-a-pipeline-early-without-failing/3951
# exit a Pipeline early without failing
exit 78
else
# stored hash of a .bingo folder to '.bingo_hash' file

View File

@@ -1,5 +1,7 @@
#!/bin/bash
source .drone.env
# Checks S3 MinIO cache for web test runner artifacts (acceptance/e2e/web).
# Sources ci.env to get WEB_COMMITID used as the cache key.
source ci.env
# if no $1 is supplied end the script
# Can be web, acceptance or e2e
@@ -13,8 +15,6 @@ web_cache=$(mc find s3/$CACHE_BUCKET/ocis/web-test-runner/$WEB_COMMITID/$1 2>&1
if [[ -z "$web_cache" ]]; then
echo "$1 cache with commit id $WEB_COMMITID already available."
# https://discourse.drone.io/t/how-to-exit-a-pipeline-early-without-failing/3951
# exit a Pipeline early without failing
exit 78
else
echo "$1 cache with commit id $WEB_COMMITID was not available."

View File

@@ -0,0 +1,3 @@
{
"defaultFont": "/ocis/tests/config/ci/NotoSans.ttf"
}

View File

@@ -0,0 +1,5 @@
#!/bin/sh
while true; do
echo -e "HTTP/1.1 200 OK\n\n$(cat /ocis/tests/config/ci/hosting-discovery.xml)" | nc -l -k -p 8080
done

View File

@@ -1,3 +0,0 @@
{
"defaultFont": "/drone/src/tests/config/drone/NotoSans.ttf"
}

View File

@@ -1,5 +0,0 @@
#!/bin/sh
while true; do
echo -e "HTTP/1.1 200 OK\n\n$(cat /drone/src/tests/config/drone/hosting-discovery.xml)" | nc -l -k -p 8080
done

View File

@@ -25,7 +25,7 @@ prepare-charts:
@test -d "$(CHART_REPO)" || git clone --single-branch -b main --depth 1 https://github.com/owncloud/ocis-charts.git $(CHART_REPO)
bash setup.sh $(CHART_REPO)
kubectl create namespace $(DOMAIN) || true
kubectl create configmap -n $(DOMAIN) ocis-fonts-ttf --from-file=../drone/NotoSans.ttf || true
kubectl create configmap -n $(DOMAIN) ocis-fonts-ttf --from-file=../ci/NotoSans.ttf || true
.PHONY: deploy-ocis
deploy-ocis:

View File

@@ -1,379 +0,0 @@
// Usage examples:
// go run ./cmd/list-failed-commits --max-commits 20
// go run ./cmd/list-failed-commits --max-commits 50 --out failed.json
// go run ./cmd/list-failed-commits --github-token $GITHUB_TOKEN --max-commits 100
// go run ./cmd/list-failed-commits --repo owncloud/ocis --branch master --max-commits 20
// go run ./cmd/list-failed-commits --since 7d --max-commits 50
// go run ./cmd/list-failed-commits --since 30d --out failed.json
// go run ./cmd/list-failed-commits --since 2026-01-01 --max-commits 100
// go run ./cmd/list-failed-commits --since 2026-02-15 --out recent.json
package main
import (
"encoding/json"
"flag"
"fmt"
"io"
"net/http"
"os"
"sort"
"strings"
"time"
"github.com/owncloud/ocis/v2/tools/ci-reporter/pkg/droneextractor"
"github.com/owncloud/ocis/v2/tools/ci-reporter/pkg/githubextractor"
"github.com/owncloud/ocis/v2/tools/ci-reporter/pkg/util"
)
const (
defaultRepo = "owncloud/ocis"
defaultBranch = "master"
requestDelay = 1 * time.Second
defaultLimit = 20
defaultUnlimited = 999999999
)
type RunConfig struct {
Repo string
Branch string
MaxCommits int
SinceRaw string
SinceParsed string
OutFile string
Token string
Command string
FetchFailedPipelineHistory bool
}
type FailureCount struct {
StageName string `json:"stage_name"`
StepName string `json:"step_name"`
Count int `json:"count"`
}
type DateRange struct {
StartDate string `json:"start_date"`
EndDate string `json:"end_date"`
}
type Raport struct {
Args string `json:"args"`
Repo string `json:"repo"`
Branch string `json:"branch"`
DateRange DateRange `json:"date_range"`
TotalCommits int `json:"total_commits"`
FailedCommits int `json:"failed_commits"`
Percentage float64 `json:"percentage"`
}
type RaportMetadata struct {
Args string
Repo string
Branch string
DateRange DateRange
TotalCommits int
}
type Report struct {
Raport *Raport `json:"raport"`
Count []FailureCount `json:"count"`
Pipelines []FailedCommit `json:"pipelines"`
}
type FailedCommit struct {
PR int `json:"pr,omitempty"`
SHA string `json:"sha"`
Date string `json:"date"`
Subject string `json:"subject"`
HTMLURL string `json:"html_url"`
CombinedState string `json:"combined_state"`
FailedContexts []StatusContext `json:"failed_contexts"`
PipelineInfo *droneextractor.PipelineInfo `json:"pipeline_info,omitempty"`
FailedPipelineHistory []droneextractor.PipelineInfoSummary `json:"failed_pipeline_history,omitempty"`
PrPipelineHistory []droneextractor.PipelineInfoSummary `json:"pr_pipeline_history,omitempty"`
}
type StatusContext struct {
Context string `json:"context"`
State string `json:"state"`
TargetURL string `json:"target_url"`
Description string `json:"description"`
}
func main() {
repo := flag.String("repo", defaultRepo, "GitHub repository (owner/repo)")
branch := flag.String("branch", defaultBranch, "Branch name")
maxCommits := flag.Int("max-commits", 0, "Maximum commits to check (default: 20 if no --since, unlimited if --since set)")
since := flag.String("since", "", "Date filter: YYYY-MM-DD, RFC3339 timestamp, or 'Nd' for last N days (e.g., '7d', '30d')")
outFile := flag.String("out", "", "Output file path (default: stdout)")
githubToken := flag.String("github-token", "", "GitHub token (optional; falls back to env GITHUB_TOKEN)")
failedPipelineHistory := flag.Bool("failed_pipeline_history", false, "if set, fetch failed_pipeline_history per commit (extra Drone API calls)")
flag.Parse()
token := strings.TrimSpace(*githubToken)
if token == "" {
token = strings.TrimSpace(os.Getenv("GITHUB_TOKEN"))
}
if token != "" {
fmt.Fprintf(os.Stderr, "Using GitHub token\n")
} else {
fmt.Fprintf(os.Stderr, "No token provided, unauthenticated, 60 req/hr limit\n")
}
command := strings.Join(os.Args[1:], " ")
command = util.RedactGitHubToken(command)
parsedSince, err := util.ParseSinceFlag(*since)
if err != nil {
fmt.Fprintf(os.Stderr, "Error parsing --since flag: %v\n", err)
os.Exit(1)
}
// Determine final max commits based on git-like UX:
// - No flags: default to 20
// - --since only: unlimited (all in range)
// - --max-commits only: use explicit value
// - Both: use explicit value (both filters apply)
finalMaxCommits := *maxCommits
if finalMaxCommits == 0 {
if *since != "" {
finalMaxCommits = defaultUnlimited // Unlimited when --since is used
} else {
finalMaxCommits = defaultLimit // Default for no flags
}
}
config := RunConfig{
Repo: *repo,
Branch: *branch,
MaxCommits: finalMaxCommits,
SinceRaw: *since,
SinceParsed: parsedSince,
OutFile: *outFile,
Token: token,
Command: command,
FetchFailedPipelineHistory: *failedPipelineHistory,
}
if err := run(config); err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
}
func run(cfg RunConfig) error {
// Configure Transport to prevent stale connection reuse during long pagination
transport := &http.Transport{
MaxIdleConns: 10,
IdleConnTimeout: 30 * time.Second,
DisableKeepAlives: false,
MaxIdleConnsPerHost: 2,
}
client := &http.Client{
Timeout: 60 * time.Second, // Increased for long-running pagination
Transport: transport,
}
githubExtractor := githubextractor.NewExtractor(client, cfg.Token)
droneExtractor := droneextractor.NewExtractor(client)
// Build scanning message
var scanMsg string
if cfg.SinceRaw != "" {
scanMsg = fmt.Sprintf("Scanning commits since %s from %s/%s", cfg.SinceRaw, cfg.Repo, cfg.Branch)
} else {
scanMsg = fmt.Sprintf("Scanning up to %d commits from %s/%s", cfg.MaxCommits, cfg.Repo, cfg.Branch)
}
fmt.Fprintf(os.Stderr, "%s\n", scanMsg)
commits, err := githubExtractor.GetCommits(cfg.Repo, cfg.Branch, cfg.MaxCommits, cfg.SinceParsed)
if err != nil {
return fmt.Errorf("fetching commits: %w", err)
}
var failedCommits []FailedCommit
for _, commit := range commits {
status, err := githubExtractor.GetCommitStatus(cfg.Repo, commit.SHA)
if err != nil {
fmt.Fprintf(os.Stderr, "⚠ %s: %v\n", commit.SHA[:7], err)
time.Sleep(requestDelay)
continue
}
if status.State == "failure" || status.State == "error" {
var failedCtxs []StatusContext
var pipelineInfo *droneextractor.PipelineInfo
for _, s := range status.Statuses {
if s.State != "success" {
// Skip Drone entries - they'll be in pipeline_info instead
if droneExtractor.IsDroneURL(s.TargetURL) {
info, err := droneExtractor.Extract(s.TargetURL)
if err == nil {
pipelineInfo = info
}
// Don't add to failed_contexts to avoid redundancy
continue
}
failedCtxs = append(failedCtxs, toStatusContext(s))
}
}
// TODO: Refactor redundant field assignment - prPipelineHistory always assigned, failedPipelineHistory conditional
var failedPipelineHistory []droneextractor.PipelineInfoSummary
var prPipelineHistory []droneextractor.PipelineInfoSummary
var prNumber int
var headRef string
// Get PR for this commit (master commit -> PR ID -> PR branch)
prs, err := githubExtractor.GetPRsForCommit(cfg.Repo, commit.SHA)
if err == nil && len(prs) > 0 {
prNumber = prs[0].Number
headRef = prs[0].Head.Ref
// Get failed builds by PR branch (black box: all logic in droneextractor)
failedBuilds, err := droneExtractor.GetFailedBuildsByPRBranch(droneextractor.BaseURL, cfg.Repo, headRef)
if err == nil {
prPipelineHistory = failedBuilds
if cfg.FetchFailedPipelineHistory {
failedPipelineHistory = failedBuilds
}
}
}
// Initialize empty slice if flag is set but no PR/builds found
if cfg.FetchFailedPipelineHistory && failedPipelineHistory == nil {
failedPipelineHistory = []droneextractor.PipelineInfoSummary{}
}
subject := util.ExtractSubject(commit.Commit.Message)
failedCommits = append(failedCommits, FailedCommit{
PR: prNumber,
SHA: commit.SHA,
Date: commit.Commit.Author.Date,
Subject: subject,
HTMLURL: commit.HTMLURL,
CombinedState: status.State,
FailedContexts: failedCtxs,
PipelineInfo: pipelineInfo,
FailedPipelineHistory: failedPipelineHistory,
PrPipelineHistory: prPipelineHistory,
})
// Single line output: ✗ sha subject -> PR #num (branch: name)
if prNumber > 0 && headRef != "" {
fmt.Fprintf(os.Stderr, "✗ %s %s -> PR #%d (branch: %s)\n", commit.SHA[:7], subject, prNumber, headRef)
} else {
fmt.Fprintf(os.Stderr, "✗ %s %s\n", commit.SHA[:7], subject)
}
}
time.Sleep(requestDelay)
}
fmt.Fprintf(os.Stderr, "\nResult: %d failed / %d total\n", len(failedCommits), len(commits))
var minDate, maxDate string
for _, c := range commits {
date := c.Commit.Author.Date
if minDate == "" || date < minDate {
minDate = date
}
if maxDate == "" || date > maxDate {
maxDate = date
}
}
metadata := RaportMetadata{
Args: cfg.Command,
Repo: cfg.Repo,
Branch: cfg.Branch,
DateRange: DateRange{StartDate: minDate, EndDate: maxDate},
TotalCommits: len(commits),
}
report := generateReport(failedCommits, metadata)
var w io.Writer = os.Stdout
if cfg.OutFile != "" {
f, err := os.Create(cfg.OutFile)
if err != nil {
return fmt.Errorf("creating output file: %w", err)
}
defer f.Close()
w = f
fmt.Fprintf(os.Stderr, "Writing JSON to %s...\n", cfg.OutFile)
}
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
if err := enc.Encode(report); err != nil {
return fmt.Errorf("encoding JSON: %w", err)
}
return nil
}
func toStatusContext(ctx githubextractor.StatusContext) StatusContext {
return StatusContext{
Context: ctx.Context,
State: ctx.State,
TargetURL: ctx.TargetURL,
Description: ctx.Description,
}
}
func generateReport(failedCommits []FailedCommit, metadata RaportMetadata) *Report {
countMap := make(map[string]int)
for _, commit := range failedCommits {
if commit.PipelineInfo == nil {
continue
}
for _, stage := range commit.PipelineInfo.PipelineStages {
for _, step := range stage.Steps {
if step.Status == "failure" {
key := stage.StageName + "/" + step.StepName
countMap[key]++
}
}
}
}
var counts []FailureCount
for key, count := range countMap {
parts := strings.Split(key, "/")
if len(parts) >= 2 {
counts = append(counts, FailureCount{
StageName: parts[0],
StepName: strings.Join(parts[1:], "/"),
Count: count,
})
}
}
sort.Slice(counts, func(i, j int) bool {
return counts[i].Count > counts[j].Count
})
percentage := 0.0
if metadata.TotalCommits > 0 {
percentage = float64(len(failedCommits)) / float64(metadata.TotalCommits) * 100.0
}
raport := &Raport{
Args: metadata.Args,
Repo: metadata.Repo,
Branch: metadata.Branch,
DateRange: metadata.DateRange,
TotalCommits: metadata.TotalCommits,
FailedCommits: len(failedCommits),
Percentage: percentage,
}
return &Report{
Raport: raport,
Count: counts,
Pipelines: failedCommits,
}
}

View File

@@ -0,0 +1,269 @@
// Usage examples:
// go run ./cmd/list-flaky-tests --since 14d
// go run ./cmd/list-flaky-tests --since 3m --workflow acceptance-tests.yml
// go run ./cmd/list-flaky-tests --github-token $GITHUB_TOKEN --since 30d
// go run ./cmd/list-flaky-tests --repo owncloud/ocis --since 7d --out report.json
package main
import (
"encoding/json"
"flag"
"fmt"
"io"
"net/http"
"os"
"sort"
"strings"
"time"
"github.com/owncloud/ocis/v2/tools/ci-reporter/pkg/ghaextractor"
"github.com/owncloud/ocis/v2/tools/ci-reporter/pkg/util"
)
const (
defaultRepo = "owncloud/ocis"
defaultWorkflow = "acceptance-tests.yml"
)
type Summary struct {
Args string `json:"args"`
Repo string `json:"repo"`
Workflow string `json:"workflow"`
DateRange Range `json:"date_range"`
TotalRuns int `json:"total_runs"`
RestartedRuns int `json:"restarted_runs"`
RestartRatePct float64 `json:"restart_rate_pct"`
TotalTimeLostMinutes float64 `json:"total_time_lost_minutes"`
}
type Range struct {
Start string `json:"start"`
End string `json:"end"`
}
type FlakyJob struct {
JobName string `json:"job_name"`
FailThenPassCount int `json:"fail_then_pass_count"`
FailureRatePct float64 `json:"failure_rate_pct"`
}
type Report struct {
Summary Summary `json:"summary"`
FlakyJobs []FlakyJob `json:"flaky_jobs"`
}
func main() {
repo := flag.String("repo", defaultRepo, "GitHub repository (owner/repo)")
workflow := flag.String("workflow", defaultWorkflow, "Workflow filename (e.g. acceptance-tests.yml)")
since := flag.String("since", "14d", "Date filter: YYYY-MM-DD, RFC3339, or 'Nd'/'Nm' (e.g. '7d', '3m')")
outFile := flag.String("out", "", "Output file path (default: stdout)")
githubToken := flag.String("github-token", "", "GitHub token (falls back to GITHUB_TOKEN env)")
flag.Parse()
token := strings.TrimSpace(*githubToken)
if token == "" {
token = strings.TrimSpace(os.Getenv("GITHUB_TOKEN"))
}
if token != "" {
fmt.Fprintf(os.Stderr, "Using GitHub token\n")
} else {
fmt.Fprintf(os.Stderr, "No token provided, unauthenticated (60 req/hr limit)\n")
}
sinceRaw := *since
// Support Nm month notation (not in ParseSinceFlag)
sinceRFC, err := parseSince(sinceRaw)
if err != nil {
fmt.Fprintf(os.Stderr, "Error parsing --since: %v\n", err)
os.Exit(1)
}
command := strings.Join(os.Args[1:], " ")
command = util.RedactGitHubToken(command)
transport := &http.Transport{
MaxIdleConns: 10,
IdleConnTimeout: 30 * time.Second,
MaxIdleConnsPerHost: 2,
}
client := &http.Client{
Timeout: 60 * time.Second,
Transport: transport,
}
extractor := ghaextractor.NewExtractor(client, token)
fmt.Fprintf(os.Stderr, "Fetching workflow runs for %s/%s since %s...\n", *repo, *workflow, sinceRaw)
runs, err := extractor.GetWorkflowRuns(*repo, *workflow, sinceRFC)
if err != nil {
fmt.Fprintf(os.Stderr, "Error fetching runs: %v\n", err)
os.Exit(1)
}
fmt.Fprintf(os.Stderr, "Found %d completed runs\n", len(runs))
report, err := buildReport(extractor, *repo, *workflow, runs, command)
if err != nil {
fmt.Fprintf(os.Stderr, "Error building report: %v\n", err)
os.Exit(1)
}
var w io.Writer = os.Stdout
if *outFile != "" {
f, err := os.Create(*outFile)
if err != nil {
fmt.Fprintf(os.Stderr, "Error creating output file: %v\n", err)
os.Exit(1)
}
defer f.Close()
w = f
fmt.Fprintf(os.Stderr, "Writing JSON to %s...\n", *outFile)
}
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
if err := enc.Encode(report); err != nil {
fmt.Fprintf(os.Stderr, "Error encoding JSON: %v\n", err)
os.Exit(1)
}
}
func buildReport(extractor *ghaextractor.Extractor, repo, workflow string, runs []ghaextractor.WorkflowRun, command string) (*Report, error) {
// The API returns one record per attempt, so a pipeline retried twice yields 3 records.
// Deduplicate by head_sha, keeping the highest-attempt record for each unique pipeline.
type pipelineRecord struct {
latest ghaextractor.WorkflowRun
maxAttempt int
}
bySHA := make(map[string]*pipelineRecord)
for _, run := range runs {
r := bySHA[run.HeadSHA]
if r == nil {
bySHA[run.HeadSHA] = &pipelineRecord{latest: run, maxAttempt: run.RunAttempt}
} else if run.RunAttempt > r.maxAttempt {
r.latest = run
r.maxAttempt = run.RunAttempt
}
}
pipelines := make([]pipelineRecord, 0, len(bySHA))
for _, r := range bySHA {
pipelines = append(pipelines, *r)
}
var minDate, maxDate string
for _, r := range pipelines {
d := r.latest.CreatedAt
if minDate == "" || d < minDate {
minDate = d
}
if maxDate == "" || d > maxDate {
maxDate = d
}
}
totalPipelines := len(pipelines)
restartedRuns := 0
var timeLostMinutes float64
// job name -> count of pipelines where that job was retried (any re-run, regardless of outcome)
flakyCount := make(map[string]int)
for i, r := range pipelines {
run := r.latest
fmt.Fprintf(os.Stderr, " [%d/%d] pipeline %d (max attempt %d)...\n", i+1, len(pipelines), run.ID, r.maxAttempt)
wasRestarted := r.maxAttempt > 1
if !wasRestarted {
continue
}
restartedRuns++
// Fetch all jobs across all attempts for this run to find which jobs were re-run
// and measure actual wall-clock time lost on retry attempts.
jobs, err := extractor.GetRunJobs(repo, run.ID)
if err != nil {
fmt.Fprintf(os.Stderr, " ⚠ run %d: %v\n", run.ID, err)
continue
}
// Track which job names appeared in attempt > 1 (= developer had to wait for them again)
retriedJobNames := make(map[string]bool)
var retryStart, retryEnd time.Time
for _, j := range jobs {
if j.RunAttempt <= 1 || j.Name == "all-acceptance-tests" {
continue
}
retriedJobNames[j.Name] = true
if j.StartedAt != "" && j.CompletedAt != "" {
if s, err := time.Parse(time.RFC3339, j.StartedAt); err == nil {
if retryStart.IsZero() || s.Before(retryStart) {
retryStart = s
}
}
if e, err := time.Parse(time.RFC3339, j.CompletedAt); err == nil {
if e.After(retryEnd) {
retryEnd = e
}
}
}
}
for name := range retriedJobNames {
flakyCount[name]++
}
if !retryStart.IsZero() && retryEnd.After(retryStart) {
timeLostMinutes += retryEnd.Sub(retryStart).Minutes()
}
}
restartRatePct := 0.0
if totalPipelines > 0 {
restartRatePct = float64(restartedRuns) / float64(totalPipelines) * 100.0
}
flakyJobs := make([]FlakyJob, 0, len(flakyCount))
for name, count := range flakyCount {
ratePct := 0.0
if totalPipelines > 0 {
ratePct = float64(count) / float64(totalPipelines) * 100.0
}
flakyJobs = append(flakyJobs, FlakyJob{
JobName: name,
FailThenPassCount: count,
FailureRatePct: ratePct,
})
}
sort.Slice(flakyJobs, func(i, j int) bool {
return flakyJobs[i].FailThenPassCount > flakyJobs[j].FailThenPassCount
})
return &Report{
Summary: Summary{
Args: command,
Repo: repo,
Workflow: workflow,
DateRange: Range{Start: minDate, End: maxDate},
TotalRuns: totalPipelines,
RestartedRuns: restartedRuns,
RestartRatePct: restartRatePct,
TotalTimeLostMinutes: timeLostMinutes,
},
FlakyJobs: flakyJobs,
}, nil
}
// parseSince extends util.ParseSinceFlag with Nm (months) and Nw (weeks) support.
func parseSince(s string) (string, error) {
if numStr, ok := strings.CutSuffix(s, "m"); ok {
var months int
if _, err := fmt.Sscanf(numStr, "%d", &months); err == nil {
return time.Now().AddDate(0, -months, 0).Format(time.RFC3339), nil
}
}
if numStr, ok := strings.CutSuffix(s, "w"); ok {
var weeks int
if _, err := fmt.Sscanf(numStr, "%d", &weeks); err == nil {
return time.Now().AddDate(0, 0, -weeks*7).Format(time.RFC3339), nil
}
}
return util.ParseSinceFlag(s)
}

View File

@@ -1,5 +1,3 @@
module github.com/owncloud/ocis/v2/tools/ci-reporter
go 1.24.0
require golang.org/x/net v0.50.0 // indirect

View File

@@ -1,2 +0,0 @@
golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=

View File

@@ -1,631 +0,0 @@
package droneextractor
import (
"encoding/json"
"fmt"
"io"
"net/http"
"regexp"
"sort"
"strings"
"golang.org/x/net/html"
)
const (
// BaseURL is the default Drone instance URL
BaseURL = "https://drone.owncloud.com"
)
type Extractor struct {
client *http.Client
}
func NewExtractor(client *http.Client) *Extractor {
return &Extractor{client: client}
}
// IsDroneURL checks if a URL is a Drone CI URL
func (e *Extractor) IsDroneURL(url string) bool {
return strings.Contains(url, "drone.owncloud.com")
}
func (e *Extractor) Extract(buildURL string) (*PipelineInfo, error) {
apiURL := e.buildURLToAPIURL(buildURL)
if apiURL == "" {
return nil, fmt.Errorf("invalid build URL: %s", buildURL)
}
buildData, err := e.fetchDroneAPI(apiURL)
if err != nil {
return nil, fmt.Errorf("fetching Drone API: %w", err)
}
stages := e.parseStagesFromAPI(buildData, buildURL)
var durationMinutes float64
if buildData.Finished > 0 && buildData.Started > 0 {
durationMinutes = float64(buildData.Finished-buildData.Started) / 60.0
}
return &PipelineInfo{
BuildURL: buildURL,
Started: buildData.Started,
Finished: buildData.Finished,
DurationMinutes: durationMinutes,
PipelineStages: stages,
}, nil
}
func (e *Extractor) buildURLToAPIURL(buildURL string) string {
if !strings.Contains(buildURL, "drone.owncloud.com") {
return ""
}
parts := strings.Split(buildURL, "/")
if len(parts) < 5 {
return ""
}
repo := parts[3] + "/" + parts[4]
buildNum := parts[5]
return fmt.Sprintf("https://drone.owncloud.com/api/repos/%s/builds/%s", repo, buildNum)
}
func (e *Extractor) fetchDroneAPI(url string) (*droneBuildResponse, error) {
resp, err := e.client.Get(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body))
}
var buildData droneBuildResponse
if err := json.NewDecoder(resp.Body).Decode(&buildData); err != nil {
return nil, err
}
return &buildData, nil
}
func (e *Extractor) parseStagesFromAPI(buildData *droneBuildResponse, buildURL string) []PipelineStage {
var stages []PipelineStage
for _, apiStage := range buildData.Stages {
normalizedStatus := e.normalizeStatus(apiStage.Status)
if normalizedStatus == "success" || normalizedStatus == "skipped" {
continue
}
stage := PipelineStage{
StageNumber: apiStage.Number,
StageName: apiStage.Name,
Status: normalizedStatus,
Steps: []PipelineStep{},
}
for _, apiStep := range apiStage.Steps {
normalizedStepStatus := e.normalizeStatus(apiStep.Status)
if normalizedStepStatus == "skipped" {
continue
}
stepURL := fmt.Sprintf("%s/%d/%d", buildURL, apiStage.Number, apiStep.Number)
step := PipelineStep{
StepNumber: apiStep.Number,
StepName: apiStep.Name,
Status: normalizedStepStatus,
URL: stepURL,
}
if normalizedStepStatus == "failure" {
logs, err := e.fetchStepLogs(stepURL, buildURL, apiStage.Number, apiStep.Number)
if err == nil {
lines := strings.Split(logs, "\n")
if len(lines) > 500 && (strings.Contains(logs, "--- Failed scenarios:") || strings.Contains(logs, "Scenario:")) {
logs = e.extractBehatFailures(logs)
}
step.Logs = logs
}
}
stage.Steps = append(stage.Steps, step)
}
stages = append(stages, stage)
}
return stages
}
func (e *Extractor) normalizeStatus(status string) string {
switch strings.ToLower(status) {
case "success", "passed":
return "success"
case "failure", "failed", "error":
return "failure"
case "running", "pending", "started":
return "running"
case "skipped":
return "skipped"
default:
return status
}
}
// droneBuildResponse matches the structure returned by Drone API:
// GET /api/repos/{owner}/{repo}/builds/{buildNumber}
// Verified against: https://drone.owncloud.com/api/repos/owncloud/ocis/builds/51629
type droneBuildResponse struct {
Started int64 `json:"started"`
Finished int64 `json:"finished"`
Stages []droneStage `json:"stages"`
}
type droneStage struct {
Number int `json:"number"` // Stage number (1, 2, 3, ...)
Name string `json:"name"` // Stage name (e.g., "API-wopi", "coding-standard-php8.4")
Status string `json:"status"` // "success", "failure", "running", etc.
Steps []droneStep `json:"steps"` // Array of steps in this stage
}
type droneStep struct {
Number int `json:"number"` // Step number within stage
Name string `json:"name"` // Step name (e.g., "clone", "test-acceptance-api")
Status string `json:"status"` // "success", "failure", "skipped", etc.
}
type droneLogEntry struct {
Pos int `json:"pos"`
Out string `json:"out"`
Time int64 `json:"time"`
}
type BuildInfo struct {
BuildNumber int `json:"build_number"`
Status string `json:"status"` // "success", "failure", "error"
Started int64 `json:"started"`
Finished int64 `json:"finished"`
CommitSHA string `json:"commit_sha"`
}
type droneBuildListItem struct {
Number int `json:"number"`
Status string `json:"status"`
Event string `json:"event"` // "pull_request", "push", etc.
Source string `json:"source"` // Source branch for PR builds
Target string `json:"target"` // Target branch for PR builds
After string `json:"after"` // Commit SHA
Started int64 `json:"started"`
Finished int64 `json:"finished"`
}
func (e *Extractor) fetchStepLogs(stepURL, buildURL string, stageNum, stepNum int) (string, error) {
apiLogsURL := e.buildLogsAPIURL(buildURL, stageNum, stepNum)
if apiLogsURL != "" {
logs, err := e.fetchLogsFromAPI(apiLogsURL)
if err == nil {
return logs, nil
}
if err != nil && !strings.Contains(err.Error(), "401") && !strings.Contains(err.Error(), "403") {
return "", err
}
}
return e.fetchLogsFromHTML(stepURL)
}
func (e *Extractor) buildLogsAPIURL(buildURL string, stageNum, stepNum int) string {
if !strings.Contains(buildURL, "drone.owncloud.com") {
return ""
}
parts := strings.Split(buildURL, "/")
if len(parts) < 5 {
return ""
}
repo := parts[3] + "/" + parts[4]
buildNum := parts[5]
return fmt.Sprintf("https://drone.owncloud.com/api/repos/%s/builds/%s/logs/%d/%d", repo, buildNum, stageNum, stepNum)
}
func (e *Extractor) fetchLogsFromAPI(apiURL string) (string, error) {
resp, err := e.client.Get(apiURL)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("HTTP %d", resp.StatusCode)
}
var logEntries []droneLogEntry
if err := json.NewDecoder(resp.Body).Decode(&logEntries); err != nil {
return "", err
}
var logs strings.Builder
for _, entry := range logEntries {
logs.WriteString(entry.Out)
}
return logs.String(), nil
}
func (e *Extractor) fetchLogsFromHTML(stepURL string) (string, error) {
resp, err := e.client.Get(stepURL)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("HTTP %d", resp.StatusCode)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}
return e.extractLogsFromHTML(string(body))
}
func (e *Extractor) extractLogsFromHTML(htmlContent string) (string, error) {
doc, err := html.Parse(strings.NewReader(htmlContent))
if err != nil {
return "", err
}
var logs strings.Builder
var extractText func(*html.Node)
extractText = func(n *html.Node) {
if n.Type == html.ElementNode {
if n.Data == "pre" || n.Data == "code" {
text := e.getTextContent(n)
if text != "" {
logs.WriteString(text)
logs.WriteString("\n")
}
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
extractText(c)
}
}
extractText(doc)
return strings.TrimSpace(logs.String()), nil
}
func (e *Extractor) getTextContent(n *html.Node) string {
var text strings.Builder
var collectText func(*html.Node)
collectText = func(n *html.Node) {
if n.Type == html.TextNode {
text.WriteString(n.Data)
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
collectText(c)
}
}
collectText(n)
return text.String()
}
func (e *Extractor) extractBehatFailures(logs string) string {
var result strings.Builder
// ANSI red code pattern
redCodePattern := regexp.MustCompile(`\x1b\[31m|\033\[31m|\[31m`)
ansiStripPattern := regexp.MustCompile(`\x1b\[[0-9;]*m|\033\[[0-9;]*m`)
// Split by Scenario: or Scenario Outline: (case-insensitive)
// Use regex to find all scenario starts while preserving the marker
scenarioPattern := regexp.MustCompile(`(?i)(Scenario(?:\s+Outline)?:\s*)`)
matches := scenarioPattern.FindAllStringIndex(logs, -1)
if len(matches) == 0 {
// No scenarios found, try to extract just "--- Failed scenarios:" section
return e.extractFailedScenariosSection(logs, ansiStripPattern)
}
// Extract each scenario section
for i, match := range matches {
start := match[0]
end := len(logs)
if i+1 < len(matches) {
end = matches[i+1][0]
}
scenarioSection := logs[start:end]
// Check if this section contains ANSI red codes
if redCodePattern.MatchString(scenarioSection) {
// Strip ANSI codes and add to result
cleaned := ansiStripPattern.ReplaceAllString(scenarioSection, "")
if result.Len() > 0 {
result.WriteString("\n\n")
}
result.WriteString(cleaned)
}
}
// Extract "--- Failed scenarios:" section
failedScenariosSection := e.extractFailedScenariosSection(logs, ansiStripPattern)
if failedScenariosSection != "" {
if result.Len() > 0 {
result.WriteString("\n\n")
}
result.WriteString(failedScenariosSection)
}
extracted := strings.TrimSpace(result.String())
if extracted == "" {
// If extraction failed, return original logs
return logs
}
return extracted
}
func (e *Extractor) extractFailedScenariosSection(logs string, ansiStripPattern *regexp.Regexp) string {
failedScenariosMarker := "--- Failed scenarios:"
if idx := strings.Index(logs, failedScenariosMarker); idx != -1 {
failedScenariosSection := logs[idx:]
cleaned := ansiStripPattern.ReplaceAllString(failedScenariosSection, "")
return cleaned
}
return ""
}
// GetBuildsByCommitSHA queries Drone API for all builds matching a specific commit SHA
func (e *Extractor) GetBuildsByCommitSHA(baseURL, repo, commitSHA string) ([]BuildInfo, error) {
var allBuilds []BuildInfo
page := 1
maxPages := 200 // Safety limit (25 items per page = 5000 builds max)
for page <= maxPages {
url := fmt.Sprintf("%s/api/repos/%s/builds?page=%d", baseURL, repo, page)
resp, err := e.client.Get(url)
if err != nil {
return nil, fmt.Errorf("fetching build list: %w", err)
}
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body))
}
var buildList []droneBuildListItem
if err := json.NewDecoder(resp.Body).Decode(&buildList); err != nil {
resp.Body.Close()
return nil, fmt.Errorf("decoding build list: %w", err)
}
resp.Body.Close()
// Stop when we get an empty page
if len(buildList) == 0 {
break
}
// Filter builds by commit SHA
for _, item := range buildList {
if item.After == commitSHA {
allBuilds = append(allBuilds, BuildInfo{
BuildNumber: item.Number,
Status: item.Status,
Started: item.Started,
Finished: item.Finished,
CommitSHA: item.After,
})
}
}
page++
}
// Sort by build number (ascending = chronological)
sort.Slice(allBuilds, func(i, j int) bool {
return allBuilds[i].BuildNumber < allBuilds[j].BuildNumber
})
return allBuilds, nil
}
// GetBuildsByPRBranch queries Drone API for all builds matching PR event and source branch
func (e *Extractor) GetBuildsByPRBranch(baseURL, repo, sourceBranch string) ([]BuildInfo, error) {
var allBuilds []BuildInfo
page := 1
maxPages := 200 // Safety limit (25 items per page = 5000 builds max)
for page <= maxPages {
url := fmt.Sprintf("%s/api/repos/%s/builds?page=%d", baseURL, repo, page)
resp, err := e.client.Get(url)
if err != nil {
return nil, fmt.Errorf("fetching build list: %w", err)
}
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body))
}
var buildList []droneBuildListItem
if err := json.NewDecoder(resp.Body).Decode(&buildList); err != nil {
resp.Body.Close()
return nil, fmt.Errorf("decoding build list: %w", err)
}
resp.Body.Close()
// Stop when we get an empty page
if len(buildList) == 0 {
break
}
// Filter builds by event="pull_request" and source branch
for _, item := range buildList {
if item.Event == "pull_request" && item.Source == sourceBranch {
allBuilds = append(allBuilds, BuildInfo{
BuildNumber: item.Number,
Status: item.Status,
Started: item.Started,
Finished: item.Finished,
CommitSHA: item.After,
})
}
}
page++
}
// Sort by build number (ascending = chronological)
sort.Slice(allBuilds, func(i, j int) bool {
return allBuilds[i].BuildNumber < allBuilds[j].BuildNumber
})
return allBuilds, nil
}
// GetBuildsByPushBranch queries Drone API for all builds matching push/cron events to a specific branch (e.g., master).
// This is specifically for merged commits to main branches, not PR builds.
// It includes both "push" events (actual merges) and "cron" events (scheduled/nightly builds).
func (e *Extractor) GetBuildsByPushBranch(baseURL, repo, targetBranch string) ([]BuildInfo, error) {
var allBuilds []BuildInfo
page := 1
maxPages := 200 // Safety limit (25 items per page = 5000 builds max)
for page <= maxPages {
url := fmt.Sprintf("%s/api/repos/%s/builds?page=%d", baseURL, repo, page)
resp, err := e.client.Get(url)
if err != nil {
return nil, fmt.Errorf("fetching build list: %w", err)
}
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body))
}
var buildList []droneBuildListItem
if err := json.NewDecoder(resp.Body).Decode(&buildList); err != nil {
resp.Body.Close()
return nil, fmt.Errorf("decoding build list: %w", err)
}
resp.Body.Close()
// Stop when we get an empty page
if len(buildList) == 0 {
break
}
// Filter builds by event="push" or event="cron" and target branch
// "push" = actual merges to branch
// "cron" = scheduled/nightly builds running on the branch
for _, item := range buildList {
if (item.Event == "push" || item.Event == "cron") && item.Target == targetBranch {
allBuilds = append(allBuilds, BuildInfo{
BuildNumber: item.Number,
Status: item.Status,
Started: item.Started,
Finished: item.Finished,
CommitSHA: item.After,
})
}
}
page++
}
// Sort by build number (ascending = chronological)
sort.Slice(allBuilds, func(i, j int) bool {
return allBuilds[i].BuildNumber < allBuilds[j].BuildNumber
})
return allBuilds, nil
}
// extractBaseURL extracts the base URL (e.g., "https://drone.owncloud.com") from a build URL
func extractBaseURL(buildURL string) string {
if !strings.Contains(buildURL, "://") {
return ""
}
parts := strings.Split(buildURL, "/")
if len(parts) >= 3 {
return parts[0] + "//" + parts[2]
}
return ""
}
// BuildInfoToPipelineSummary converts a BuildInfo to PipelineInfoSummary
func BuildInfoToPipelineSummary(build BuildInfo, baseURL, repo string) PipelineInfoSummary {
var durationMinutes float64
if build.Finished > 0 && build.Started > 0 {
durationMinutes = float64(build.Finished-build.Started) / 60.0
}
// URL format: https://drone.owncloud.com/owncloud/ocis/{buildNumber}
// repo format: "owncloud/ocis"
buildURL := fmt.Sprintf("%s/%s/%d", baseURL, repo, build.BuildNumber)
return PipelineInfoSummary{
BuildURL: buildURL,
Started: build.Started,
Finished: build.Finished,
DurationMinutes: durationMinutes,
Status: build.Status,
}
}
// BuildInfoToPipelineSummaryWithStages converts a BuildInfo to PipelineInfoSummary with failed stages
// by fetching full pipeline details from the build URL
func (e *Extractor) BuildInfoToPipelineSummaryWithStages(build BuildInfo, baseURL, repo string) PipelineInfoSummary {
summary := BuildInfoToPipelineSummary(build, baseURL, repo)
// Fetch full pipeline info to extract failed stages
pipelineInfo, err := e.Extract(summary.BuildURL)
if err == nil && pipelineInfo != nil {
var failedStages []FailedStage
for _, stage := range pipelineInfo.PipelineStages {
if stage.Status == "failure" || stage.Status == "error" {
failedStages = append(failedStages, FailedStage{
StageNumber: stage.StageNumber,
StageName: stage.StageName,
Status: stage.Status,
})
}
}
summary.FailedStages = failedStages
}
return summary
}
// IsFailedBuild checks if a build status indicates failure
func IsFailedBuild(status string) bool {
normalized := strings.ToLower(status)
return normalized == "failure" || normalized == "failed" || normalized == "error"
}
// GetFailedBuildsByPRBranch returns failed builds for a PR branch with pipeline summaries
func (e *Extractor) GetFailedBuildsByPRBranch(baseURL, repo, sourceBranch string) ([]PipelineInfoSummary, error) {
builds, err := e.GetBuildsByPRBranch(baseURL, repo, sourceBranch)
if err != nil {
return nil, err
}
var failedSummaries []PipelineInfoSummary
for _, build := range builds {
if IsFailedBuild(build.Status) {
summary := e.BuildInfoToPipelineSummaryWithStages(build, baseURL, repo)
failedSummaries = append(failedSummaries, summary)
// Rate limiting delay for API calls in BuildInfoToPipelineSummaryWithStages
}
}
return failedSummaries, nil
}

View File

@@ -1,78 +0,0 @@
package droneextractor
import (
"fmt"
"io"
"time"
"github.com/owncloud/ocis/v2/tools/ci-reporter/pkg/githubextractor"
)
const requestDelay = 1 * time.Second
// FetchPRBuilds fetches all Drone builds for a PR by querying:
// 1. Builds by PR branch (handles force-pushes)
// 2. Builds by individual commit SHAs (fallback for merged/deleted branches)
//
// Returns a map of commit SHA -> builds
func FetchPRBuilds(
extractor *Extractor,
baseURL string,
repo string,
prBranch string,
commits []githubextractor.PRCommit,
logWriter io.Writer,
) (map[string][]BuildInfo, error) {
buildsByCommit := make(map[string][]BuildInfo)
// Query builds by PR branch (handles force-pushes)
if logWriter != nil {
fmt.Fprintf(logWriter, "Fetching Drone builds for PR (by event and branch)...\n")
}
prBuilds, err := extractor.GetBuildsByPRBranch(baseURL, repo, prBranch)
if err != nil {
if logWriter != nil {
fmt.Fprintf(logWriter, "⚠ Failed to get builds by branch: %v\n", err)
}
} else {
if logWriter != nil {
fmt.Fprintf(logWriter, "Found %d builds for PR branch %s\n", len(prBuilds), prBranch)
}
// Group builds by commit SHA
for _, build := range prBuilds {
buildsByCommit[build.CommitSHA] = append(buildsByCommit[build.CommitSHA], build)
}
}
// Also query by current commit SHAs (for completeness)
if logWriter != nil {
fmt.Fprintf(logWriter, "Fetching Drone builds for current commits...\n")
}
for i, commit := range commits {
if _, exists := buildsByCommit[commit.SHA]; exists {
if logWriter != nil {
fmt.Fprintf(logWriter, " [%d/%d] %s: already found via PR query\n", i+1, len(commits), commit.SHA[:7])
}
continue
}
if logWriter != nil {
fmt.Fprintf(logWriter, " [%d/%d] Fetching builds for %s...\n", i+1, len(commits), commit.SHA[:7])
}
builds, err := extractor.GetBuildsByCommitSHA(baseURL, repo, commit.SHA)
if err != nil {
if logWriter != nil {
fmt.Fprintf(logWriter, " ⚠ Failed to get builds: %v\n", err)
}
continue
}
if len(builds) > 0 {
buildsByCommit[commit.SHA] = append(buildsByCommit[commit.SHA], builds...)
if logWriter != nil {
fmt.Fprintf(logWriter, " Found %d builds\n", len(builds))
}
}
time.Sleep(requestDelay)
}
return buildsByCommit, nil
}

View File

@@ -1,39 +0,0 @@
package droneextractor
type PipelineInfo struct {
BuildURL string `json:"build_url"`
Started int64 `json:"started,omitempty"` // Unix timestamp
Finished int64 `json:"finished,omitempty"` // Unix timestamp
DurationMinutes float64 `json:"duration_minutes,omitempty"` // Calculated: (finished - started) / 60
PipelineStages []PipelineStage `json:"pipeline_stages"`
}
type PipelineStage struct {
StageNumber int `json:"stage_number"`
StageName string `json:"stage_name,omitempty"`
Status string `json:"status,omitempty"`
Steps []PipelineStep `json:"steps"`
}
type PipelineStep struct {
StepNumber int `json:"step_number"`
StepName string `json:"step_name"`
Status string `json:"status"` // "success", "failure", "error", "running", etc.
URL string `json:"url,omitempty"`
Logs string `json:"logs,omitempty"` // Console logs content
}
type PipelineInfoSummary struct {
BuildURL string `json:"build_url"`
Started int64 `json:"started,omitempty"`
Finished int64 `json:"finished,omitempty"`
DurationMinutes float64 `json:"duration_minutes,omitempty"`
Status string `json:"status,omitempty"` // Overall build status
FailedStages []FailedStage `json:"failed_stages,omitempty"`
}
type FailedStage struct {
StageNumber int `json:"stage_number"`
StageName string `json:"stage_name"`
Status string `json:"status"`
}

View File

@@ -0,0 +1,147 @@
package ghaextractor
import (
"encoding/json"
"fmt"
"io"
"net/http"
"time"
)
const (
apiBase = "https://api.github.com"
requestDelay = 1 * time.Second
retryDelay = 2 * time.Second
maxRetries = 3
)
type Extractor struct {
client *http.Client
token string
}
func NewExtractor(client *http.Client, token string) *Extractor {
return &Extractor{client: client, token: token}
}
type WorkflowRun struct {
ID int64 `json:"id"`
RunAttempt int `json:"run_attempt"`
Status string `json:"status"`
Conclusion string `json:"conclusion"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
HTMLURL string `json:"html_url"`
HeadBranch string `json:"head_branch"`
HeadSHA string `json:"head_sha"`
Event string `json:"event"`
}
type workflowRunsPage struct {
TotalCount int `json:"total_count"`
WorkflowRuns []WorkflowRun `json:"workflow_runs"`
}
type Job struct {
ID int64 `json:"id"`
RunID int64 `json:"run_id"`
RunAttempt int `json:"run_attempt"`
Name string `json:"name"`
Status string `json:"status"`
Conclusion string `json:"conclusion"`
StartedAt string `json:"started_at"`
CompletedAt string `json:"completed_at"`
}
type jobsPage struct {
TotalCount int `json:"total_count"`
Jobs []Job `json:"jobs"`
}
// GetWorkflowRuns returns all runs for a workflow file created at or after `since` (RFC3339).
func (e *Extractor) GetWorkflowRuns(repo, workflow, since string) ([]WorkflowRun, error) {
var all []WorkflowRun
page := 1
for {
url := fmt.Sprintf("%s/repos/%s/actions/workflows/%s/runs?per_page=100&page=%d&status=completed", apiBase, repo, workflow, page)
if since != "" {
url += "&created=>=" + since
}
var result workflowRunsPage
if err := e.get(url, &result); err != nil {
return nil, err
}
all = append(all, result.WorkflowRuns...)
if len(result.WorkflowRuns) < 100 {
break
}
page++
}
return all, nil
}
// GetRunJobs returns all jobs for a given run across all attempts (filter=all).
func (e *Extractor) GetRunJobs(repo string, runID int64) ([]Job, error) {
var all []Job
page := 1
for {
url := fmt.Sprintf("%s/repos/%s/actions/runs/%d/jobs?filter=all&per_page=100&page=%d", apiBase, repo, runID, page)
var result jobsPage
if err := e.get(url, &result); err != nil {
return nil, err
}
all = append(all, result.Jobs...)
if len(result.Jobs) < 100 {
break
}
page++
}
return all, nil
}
func (e *Extractor) get(url string, result any) error {
resp, err := e.request(url)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("API error %d: %s", resp.StatusCode, string(body))
}
if err := json.NewDecoder(resp.Body).Decode(result); err != nil {
return err
}
time.Sleep(requestDelay)
return nil
}
func (e *Extractor) request(url string) (*http.Response, error) {
var lastErr error
for attempt := 0; attempt < maxRetries; attempt++ {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "application/vnd.github.v3+json")
if e.token != "" {
req.Header.Set("Authorization", "Bearer "+e.token)
}
resp, err := e.client.Do(req)
if err != nil {
lastErr = err
if attempt < maxRetries-1 {
time.Sleep(retryDelay)
continue
}
return nil, err
}
return resp, nil
}
return nil, lastErr
}

19
vendor/github.com/RoaringBitmap/roaring/v2/.drone.yml generated vendored Normal file
View File

@@ -0,0 +1,19 @@
kind: pipeline
name: default
workspace:
base: /go
path: src/github.com/RoaringBitmap/roaring
steps:
- name: test
image: golang
commands:
- go get -t
- go test
- go build -tags appengine
- go test -tags appengine
- GOARCH=386 go build
- GOARCH=386 go test
- GOARCH=arm go build
- GOARCH=arm64 go build