mirror of
https://github.com/owncloud/ocis.git
synced 2026-10-23 23:29:02 +03:00
feat: drop drone (#12243)
This commit is contained in:
@@ -1,3 +0,0 @@
|
||||
# The test runner source for UI tests
|
||||
WEB_COMMITID=39d6215da6daa184cd2365966915a659bdbd2c8b
|
||||
WEB_BRANCH=stable-12.3
|
||||
4091
.drone.star
4091
.drone.star
File diff suppressed because it is too large
Load Diff
1
.github/CODEOWNERS
vendored
1
.github/CODEOWNERS
vendored
@@ -1,4 +1,3 @@
|
||||
services/settings/ @lukashirt
|
||||
services/web/ @lukashirt
|
||||
.drone.star @kobergj @phil-davis @individual-it
|
||||
assets/End-User-License-Agreement-for-ownCloud-Infinite-Scale.pdf @kobergj @hodyroff
|
||||
|
||||
1
.github/release_template.md
vendored
1
.github/release_template.md
vendored
@@ -8,7 +8,6 @@
|
||||
* [ ] DEV/QA: Check new strings and align with clients
|
||||
* [ ] DEV/DOCS: Create list of pending docs tasks
|
||||
* [ ] DEV: Create branch `release-x.x.x-rc.x` -> CODEFREEZE
|
||||
* [ ] DEV: specify if it is a production release in [.drone.star](https://github.com/owncloud/ocis/blob/b4cf38fa1ba180c58519026dfe762b7c45695466/.drone.star#L8) (if needed)
|
||||
* [ ] DEV: bump ocis version in necessary files
|
||||
* [ ] DEV: `changelog/CHANGELOG.tmpl`
|
||||
* [ ] DEV: `ocis-pkg/version/version.go`
|
||||
|
||||
9
.github/workflows/acceptance-tests.yml
vendored
9
.github/workflows/acceptance-tests.yml
vendored
@@ -219,7 +219,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
suite:
|
||||
- cliCommands,apiServiceAvailability # grouped like drone; needs ClamAV + email
|
||||
- cliCommands,apiServiceAvailability # grouped: both need ClamAV + email services
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
@@ -375,7 +375,7 @@ jobs:
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: ~/.cache/ms-playwright
|
||||
key: playwright-chromium-${{ hashFiles('.drone.env') }}
|
||||
key: playwright-chromium-${{ hashFiles('ci.env') }}
|
||||
|
||||
# --- Tika (search suite only) ---
|
||||
- name: Start Tika
|
||||
@@ -409,9 +409,10 @@ jobs:
|
||||
- name: Start Keycloak
|
||||
if: matrix.keycloak == true
|
||||
run: |
|
||||
# Patch realm: replace Drone Docker hostname with localhost IP
|
||||
# Patch realm: ocis-ci-realm.dist.json uses ocis-server:9200 as hostname.
|
||||
# GitHub Actions runs on the host network so patch it to 127.0.0.1.
|
||||
sed 's|https://ocis-server:9200|https://127.0.0.1:9200|g' \
|
||||
tests/config/drone/ocis-ci-realm.dist.json > /tmp/ocis-realm.json
|
||||
tests/config/ci/ocis-ci-realm.dist.json > /tmp/ocis-realm.json
|
||||
docker run -d --name keycloak --network host \
|
||||
-e OCIS_DOMAIN=https://127.0.0.1:9200 \
|
||||
-e KC_HOSTNAME=localhost \
|
||||
|
||||
7
.github/workflows/weekly_pipeline_report.yml
vendored
7
.github/workflows/weekly_pipeline_report.yml
vendored
@@ -24,9 +24,8 @@ jobs:
|
||||
working-directory: tools/ci-reporter
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
# Review the pipelines in last 14 days before the dorne logs expire
|
||||
run: |
|
||||
GOWORK=off go run ./cmd/list-failed-commits \
|
||||
GOWORK=off go run ./cmd/list-flaky-tests \
|
||||
--github-token "$GITHUB_TOKEN" \
|
||||
--since 14d \
|
||||
--max-commits 9999999
|
||||
--workflow acceptance-tests.yml \
|
||||
--since 4w
|
||||
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -45,8 +45,6 @@ vendor-php
|
||||
# QA activity reports
|
||||
tests/qa-activity-report/reports/
|
||||
|
||||
# drone CI is in .drone.star, do not let someone accidentally commit a local .drone.yml
|
||||
.drone.yml
|
||||
|
||||
**/l10n/template.pot
|
||||
|
||||
|
||||
9
Makefile
9
Makefile
@@ -20,7 +20,6 @@ L10N_MODULES := \
|
||||
services/userlog \
|
||||
services/settings
|
||||
|
||||
# if you add a module here please also add it to the .drone.star file
|
||||
OCIS_MODULES = \
|
||||
services/activitylog \
|
||||
services/antivirus \
|
||||
@@ -206,7 +205,10 @@ docs-serve:
|
||||
docs-clean:
|
||||
@$(MAKE) --no-print-directory -C docs docs-clean
|
||||
|
||||
# imitate a full drone run locally to build docs without pushing to the web.
|
||||
# docs-local imitates the 3-step docs build pipeline (generate → copy → build)
|
||||
# locally without pushing. docs-hugo-drone-prep creates the ../hugo symlink required by
|
||||
# the Hugo Docker image (hugomods/hugo:base-0.129.0).
|
||||
# imitate a full docs build run locally without pushing to the web.
|
||||
# this can help identify uncaught issues when running `make docs-serve` only.
|
||||
.PHONY: docs-local # run all steps as drone would do it (1, 2, 3)
|
||||
docs-local:
|
||||
@@ -365,9 +367,6 @@ l10n-write:
|
||||
$(MAKE) -C $$extension l10n-write || exit 1; \
|
||||
done
|
||||
|
||||
.PHONY: ci-format
|
||||
ci-format: $(BUILDIFIER)
|
||||
$(BUILDIFIER) --mode=fix .drone.star
|
||||
|
||||
.PHONY: test-php-style
|
||||
test-php-style: vendor-bin/owncloud-codestyle/vendor vendor-bin/php_codesniffer/vendor
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# ownCloud Infinite Scale
|
||||
|
||||
[](https://app.element.io/#/room/#ocis:matrix.org)
|
||||
[](https://drone.owncloud.com/owncloud/ocis)
|
||||
[](https://github.com/owncloud/ocis/actions/workflows/acceptance-tests.yml)
|
||||
[](https://sonarcloud.io/dashboard?id=owncloud_ocis)
|
||||
[](https://sonarcloud.io/dashboard?id=owncloud_ocis)
|
||||
[](https://sonarcloud.io/summary/new_code?id=owncloud-1_ocis_acceptance-tests)
|
||||
|
||||
4
ci.env
Normal file
4
ci.env
Normal file
@@ -0,0 +1,4 @@
|
||||
# Pins the ownCloud Web frontend version used in CI tests.
|
||||
# Read by: tests/acceptance/run-e2e.py and .github/workflows/acceptance-tests.yml (Playwright cache key).
|
||||
WEB_COMMITID=6165b4e47a47e3a216107e96f90825190f10632e
|
||||
WEB_BRANCH=stable-12.3
|
||||
@@ -37,7 +37,8 @@ help:
|
||||
.PHONY:list
|
||||
list: help
|
||||
|
||||
# for drone only. drone uses an image for hugo that needs the hugo folder in the root
|
||||
# Creates a symlink ../hugo → hugo/ required by the Hugo Docker image
|
||||
# (hugomods/hugo:base-0.129.0), which expects the hugo folder at the repo root.
|
||||
# CURDIR will work for make only but not when running manually in the shell
|
||||
.PHONY: docs-hugo-drone-prep
|
||||
docs-hugo-drone-prep:
|
||||
|
||||
@@ -40,7 +40,7 @@ variables:
|
||||
description: ""
|
||||
do_ignore: true
|
||||
- rawname: GITHUB_TOKEN
|
||||
path: tools/ci-reporter/cmd/list-failed-commits/main.go:116
|
||||
path: tools/ci-reporter/cmd/list-flaky-tests/main.go:64
|
||||
foundincode: true
|
||||
name: GITHUB_TOKEN
|
||||
type: ""
|
||||
|
||||
@@ -15,7 +15,7 @@ The documentation:
|
||||
|
||||
* Is based on the [HUGO](https://gohugo.io/) framework written in Go. The underlaying language is markdown plus the extensions provided by hugo.
|
||||
* The framwork including the theme used is maintained in the following [repo](https://github.com/owncloud/owncloud.github.io).
|
||||
* The hugo version, currently at `0.129.0` must match the image version defined in the root makefile and the framework repo, defined in `.drone.yml`.\
|
||||
* The hugo version, currently at `0.129.0` must match the image version defined in the root makefile and the framework repo.\
|
||||
{{< hint info >}}
|
||||
* Building the theme is part of the loaded framework while building the documentation is done using a separate container.
|
||||
|
||||
@@ -24,7 +24,7 @@ The documentation:
|
||||
|
||||
* The documentation sources are distributed over several repos such as `ocis`, `web` and others. When building, these sources act individually and only those sources are build and pushed to the web that come from the respective repo. The definition what is going where is defined in the [framework repo](https://github.com/owncloud/owncloud.github.io/blob/main/.batchfile). This means, each merge of a docs change in one of the sourcing repos only pushes that change to the site. There is no global build process and inter doc links must be treated therefore very carefully !
|
||||
|
||||
* The documentation setup is, with only a few exceptions, identical for both the `ocis` and the `web` repo. This means, for example, running commands locally are the same, the documentation process section below applies for both; only drone uses some specifics.
|
||||
* The documentation setup is, with only a few exceptions, identical for both the `ocis` and the `web` repo. This means, for example, running commands locally are the same, the documentation process section below applies for both.
|
||||
|
||||
* {{< hint info >}}
|
||||
Any files or folders that need to be excluded from the build process must be defined in the referenced framework repository above, in the file `config.yaml`.
|
||||
|
||||
@@ -9,48 +9,47 @@ geekdocFilePath: continuous-integration.md
|
||||
|
||||
{{< toc >}}
|
||||
|
||||
oCIS uses [DRONE](https://www.drone.io/) as CI system. You can find the pipeline logs [here](https://drone.owncloud.com/owncloud/ocis) or in your PR.
|
||||
## Overview
|
||||
|
||||
oCIS uses [GitHub Actions](https://github.com/owncloud/ocis/actions) as its CI system. Pipeline logs are visible directly in pull requests.
|
||||
|
||||
## Concepts
|
||||
|
||||
The pipeline is defined in [Starlark](https://github.com/bazelbuild/starlark) and transformed to YAML upon pipeline run. This enables us to do a highly dynamic and non repeating pipeline configuration. We enforce Starlark format guidelines with Bazel Buildifier. You can format the .drone.star file by running `make ci-format`.
|
||||
Pipelines are defined in `.github/workflows/`. The main acceptance test workflow is `.github/workflows/acceptance-tests.yml`.
|
||||
|
||||
Upon running the pipeline, your branch gets merged to the master branch. This ensures that we always test your changeset if as it was applied to the master of oCIS. Please note that this does not apply to the pipeline definition (`.drone.star`).
|
||||
Upon running the pipeline, tests run against the branch as-is. Make sure your branch is rebased onto the latest master before running CI to avoid false failures.
|
||||
|
||||
## Things done in CI
|
||||
|
||||
- static code analysis
|
||||
- linting
|
||||
- running UI tests
|
||||
- running ownCloud 10 test suite against oCIS
|
||||
- running API acceptance tests
|
||||
- build and release docker images
|
||||
- build and release binaries
|
||||
- build and release documentation
|
||||
|
||||
## Flags in commit message and PR title
|
||||
|
||||
You may add flags to your commit message or PR title in order to speed up pipeline runs and take load from the CI runners.
|
||||
You may add flags to your commit message or PR title to control pipeline runs.
|
||||
|
||||
- `[CI SKIP]`: no CI is run on the commit or PR
|
||||
|
||||
- `[full-ci]`: deactivates the fail early mechanism and runs all available test (as default only smoke tests are run)
|
||||
- `[k6-test]`: enabled the run of the k6 with smoke and performance test
|
||||
|
||||
### Knowledge base
|
||||
|
||||
- My pipeline fails because some CI related files or commands are missing.
|
||||
|
||||
Please make sure to rebase your branch onto the latest master of oCIS. It could be that the pipeline definition (`.drone.star`) was changed on the master branch. This is the only file, that will not be auto merged to master upon pipeline run. So things could be out of sync.
|
||||
Make sure your branch is rebased onto the latest master. Workflow files in `.github/workflows/` change over time and a stale branch may reference steps or configs that no longer exist.
|
||||
|
||||
- How can I see the YAML drone pipeline definition?
|
||||
- How can I see what a workflow does?
|
||||
|
||||
In order to see the Yaml pipeline definition you can use the drone-cli to convert the Starlark file.
|
||||
Workflow definitions are plain YAML in `.github/workflows/`. Open the relevant file directly in the repo — no CLI tool needed. The main acceptance test entry point is `.github/workflows/acceptance-tests.yml`.
|
||||
|
||||
- How can I re-run a failed job?
|
||||
|
||||
In the GitHub Actions UI, click **Re-run jobs** → **Re-run failed jobs** on the failed workflow run. You can also trigger a full re-run from `gh`:
|
||||
|
||||
```bash
|
||||
gh run rerun <run-id> --failed --repo owncloud/ocis
|
||||
```
|
||||
drone starlark
|
||||
```
|
||||
|
||||
{{< hint info >}}
|
||||
If you experience a `"build" struct has no .title attribute` you need a newer version of drone-cli.
|
||||
|
||||
You currently need to build it yourself from this [source code](https://github.com/drone/drone-cli). If you are not using master as source, please ensure that this [PR](https://github.com/drone/drone-cli/pull/175) is included.
|
||||
{{< /hint >}}
|
||||
|
||||
@@ -581,11 +581,11 @@ THUMBNAILS_TXT_FONTMAP_FILE="/path/to/fontsMap.json"
|
||||
ocis/bin/ocis server
|
||||
```
|
||||
|
||||
The sample `fontsMap.json` file is located in `tests/config/drone/fontsMap.json`.
|
||||
The sample `fontsMap.json` file is located in `tests/config/ci/fontsMap.json`.
|
||||
|
||||
```json
|
||||
{
|
||||
"defaultFont": "/path/to/ocis/tests/config/drone/NotoSans.ttf"
|
||||
"defaultFont": "/path/to/ocis/tests/config/ci/NotoSans.ttf"
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -108,10 +108,7 @@ https://owncloud.dev/ocis/releasing_guide/
|
||||
* [ ] Create orphan branch: `git checkout --orphan docs-stable-7.2`
|
||||
* [ ] Initial commit: `git commit --allow-empty -m "initial commit"`
|
||||
* [ ] Push it: `git push`
|
||||
* [ ] Adjust the `.drone.star` file to write to `docs-stable-7.2`
|
||||
* [ ] Find target_branch value in the docs section and change it to `docs-stable-7.2`
|
||||
* [ ] Example: https://github.com/owncloud/ocis/blame/56f7645f0b11c9112e15ce46f6effd2fea01d6be/.drone.star#L2249
|
||||
* [ ] Add `stable-7.2` to the nightly cron jobs in drone (`Settings` -> `Cron Jobs`)
|
||||
* [ ] Add `stable-7.2` to the nightly cron jobs in GitHub Actions (`Settings` -> `Schedules`)
|
||||
```
|
||||
|
||||
### Steps for Patch Releases Only
|
||||
|
||||
@@ -16,12 +16,7 @@ Services can have texts that need to be translated. These translations will be s
|
||||
The process for _synchronisation_ with Transifex is already setup and nothing needs to be done here. For any translation, it is necessary to set it up in the respective service and tell to sync it.
|
||||
|
||||
**IMPORTANT**\
|
||||
Translations are automatically synced on a daily basis in the night. To do so, there is an own repo that covers the process for ALL translations from all configured repos: [translation-sync](https://github.com/owncloud/translation-sync). If there is a manual "emergency" sync necessary, you only need to trigger [drone](https://drone.owncloud.com/owncloud/translation-sync) via cli
|
||||
|
||||
```bash
|
||||
drone cron exec owncloud/translation-sync nightly
|
||||
```
|
||||
Note that you need to be logged on in drone to execute the command.
|
||||
Translations are automatically synced on a daily basis in the night. To do so, there is an own repo that covers the process for ALL translations from all configured repos: [translation-sync](https://github.com/owncloud/translation-sync). If there is a manual "emergency" sync necessary, trigger the workflow manually via the [GitHub Actions UI](https://github.com/owncloud/translation-sync/actions) using `workflow_dispatch`.
|
||||
|
||||
## Implementing ocis Translations
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ Use this checklist with copy/paste in your PR - right from the beginning. It ren
|
||||
- Add the service config to `ocis-pkg/config/defaultconfig.go`
|
||||
- [ ] If the service is using service accounts, add it to `ocis/pkg/init/init.go`
|
||||
- [ ] Check that the service properly responds to `ocis <service-name> health` and has `/healthz` and `/readyz` endpoints
|
||||
- [ ] Add the service to `.drone.star` to enable CI.
|
||||
- [ ] Add the service suite to the GitHub Actions matrix in `.github/workflows/acceptance-tests.yml` to enable CI.
|
||||
- [ ] Inform doc team in an *early stage* to review the readme AND the environment variables created.
|
||||
- The description must reflect the behaviour AND usually has a positive code quality impact.
|
||||
- [ ] Create proper description strings for envvars - see other services for examples, especially when it comes to multiple values. This must include:
|
||||
|
||||
@@ -32,6 +32,6 @@ and take note of its release tag name.
|
||||
2. Change into web package folder via `cd web`
|
||||
3. Inside `web/`, update the `Makefile` so that the WEB_ASSETS_VERSION variable references the currently released version of https://github.com/owncloud/web
|
||||
4. Move to the changelog (`cd ../changelog/`) and add a changelog file to the `unreleased/` folder (You can copy an old web release changelog item as a template)
|
||||
5. Move to the repo root (`cd ..`)and update the WEB_COMMITID in the `/.drone.env` file to the commit id from the released version (unless the existing commit id is already newer)
|
||||
5. Move to the repo root (`cd ..`) and update the `WEB_COMMITID` in `/ci.env` to the commit id from the released version (unless the existing commit id is already newer)
|
||||
6. **Optional:** Test the changes locally by running `cd ocis && go run cmd/ocis/main.go server`, visiting [https://localhost:9200](https://localhost:9200) and confirming everything renders correctly
|
||||
7. Commit your changes, push them and [create a PR](https://github.com/owncloud/ocis/pulls)
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
services:
|
||||
acceptance-tests:
|
||||
image: owncloudci/php:8.2
|
||||
working_dir: /drone/src
|
||||
working_dir: /ocis
|
||||
command: /bin/bash /test/run-tests.sh
|
||||
environment:
|
||||
OCIS_ROOT: /drone/src
|
||||
OCIS_ROOT: /ocis
|
||||
TEST_SERVER_URL: https://ocis-server:9200
|
||||
OCIS_WRAPPER_URL: http://ocis-server:5200
|
||||
STORAGE_DRIVER: $STORAGE_DRIVER
|
||||
@@ -17,8 +17,8 @@ services:
|
||||
EMAIL_HOST: email
|
||||
EMAIL_PORT: 8025
|
||||
env_file:
|
||||
- ../../../../.drone.env
|
||||
- ../../../../ci.env
|
||||
volumes:
|
||||
- ./run-tests.sh:/test/run-tests.sh
|
||||
- ../../../../:/drone/src
|
||||
- ../../../../:/ocis
|
||||
- ../../../../vendor-bin/behat/composer.json:/tmp/vendor-bin/behat/composer.json
|
||||
|
||||
@@ -15,7 +15,7 @@ services:
|
||||
STORAGE_SYSTEM_DRIVER_OCIS_ROOT: /srv/app/tmp/ocis/storage/metadata
|
||||
SHARING_USER_JSON_FILE: /srv/app/tmp/ocis/shares.json
|
||||
PROXY_ENABLE_BASIC_AUTH: "true"
|
||||
WEB_UI_CONFIG_FILE: /drone/src/tests/config/drone/ocis-config.json
|
||||
WEB_UI_CONFIG_FILE: /ocis/tests/config/ci/ocis-config.json
|
||||
OCIS_INSECURE: "true"
|
||||
IDM_CREATE_DEMO_USERS: "true"
|
||||
IDM_ADMIN_PASSWORD: "admin"
|
||||
@@ -48,7 +48,7 @@ services:
|
||||
SEARCH_EXTRACTOR_CS3SOURCE_INSECURE: "true"
|
||||
|
||||
# fonts map for txt thumbnails (including unicode support)
|
||||
THUMBNAILS_TXT_FONTMAP_FILE: "/drone/src/tests/config/drone/fontsMap.json"
|
||||
THUMBNAILS_TXT_FONTMAP_FILE: "/ocis/tests/config/ci/fontsMap.json"
|
||||
volumes:
|
||||
- ../../../config:/drone/src/tests/config
|
||||
- ../../../config:/ocis/tests/config
|
||||
- ../../../ociswrapper/bin/ociswrapper:/usr/bin/ociswrapper
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
#!/bin/bash
|
||||
|
||||
#mkdir -p /drone/src/vendor-bin/behat
|
||||
#cp /tmp/vendor-bin/behat/composer.json /drone/src/vendor-bin/behat/composer.json
|
||||
|
||||
git config --global advice.detachedHead false
|
||||
|
||||
## CONFIGURE TEST
|
||||
@@ -12,7 +9,7 @@ if [ "$TEST_SOURCE" = "oc10" ]; then
|
||||
if [ "$STORAGE_DRIVER" = "ocis" ]; then
|
||||
export OCIS_REVA_DATA_ROOT=''
|
||||
export BEHAT_FILTER_TAGS='~@skipOnOcis-OCIS-Storage'
|
||||
export EXPECTED_FAILURES_FILE='/drone/src/tests/acceptance/expected-failures-API-on-OCIS-storage.md'
|
||||
export EXPECTED_FAILURES_FILE='/ocis/tests/acceptance/expected-failures-API-on-OCIS-storage.md'
|
||||
elif [ "$STORAGE_DRIVER" = "s3ng" ]; then
|
||||
export BEHAT_FILTER_TAGS='~@skip&&~@skipOnOcis-S3NG-Storage'
|
||||
export OCIS_REVA_DATA_ROOT=''
|
||||
|
||||
@@ -286,7 +286,7 @@ Feature: Link sharing of project spaces
|
||||
@env-config
|
||||
Scenario: try to create a link share of a project-space with a password that is listed in the Banned-Password-List using permissions endpoint
|
||||
Given using spaces DAV path
|
||||
And the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/drone/banned-password-list.txt" for "sharing" service
|
||||
And the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/ci/banned-password-list.txt" for "sharing" service
|
||||
And the administrator has assigned the role "Space Admin" to user "Alice" using the Graph API
|
||||
And user "Alice" has created a space "projectSpace" with the default quota using the Graph API
|
||||
When user "Alice" creates the following space link share using permissions endpoint of the Graph API:
|
||||
@@ -315,7 +315,7 @@ Feature: Link sharing of project spaces
|
||||
@env-config
|
||||
Scenario: try to create a link share of a project-space with a password that is listed in the Banned-Password-List using root endpoint
|
||||
Given using spaces DAV path
|
||||
And the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/drone/banned-password-list.txt" for "sharing" service
|
||||
And the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/ci/banned-password-list.txt" for "sharing" service
|
||||
And the administrator has assigned the role "Space Admin" to user "Alice" using the Graph API
|
||||
And user "Alice" has created a space "projectSpace" with the default quota using the Graph API
|
||||
When user "Alice" creates the following space link share using root endpoint of the Graph API:
|
||||
|
||||
@@ -6,7 +6,7 @@ Feature: Link share with banned password
|
||||
Given these users have been created with default attributes:
|
||||
| username |
|
||||
| Alice |
|
||||
And the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/drone/banned-password-list.txt" for "sharing" service
|
||||
And the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/ci/banned-password-list.txt" for "sharing" service
|
||||
|
||||
|
||||
Scenario: try to create a folder link share with a password that is listed in the Banned-Password-List
|
||||
|
||||
@@ -237,7 +237,7 @@ Feature: Update a link share for a resource
|
||||
|
||||
|
||||
Scenario Outline: update a file's link share with a password that is listed in the Banned-Password-List using permissions endpoint
|
||||
Given the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/drone/banned-password-list.txt" for "sharing" service
|
||||
Given the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/ci/banned-password-list.txt" for "sharing" service
|
||||
And user "Alice" has uploaded file with content "other data" to "text.txt"
|
||||
And user "Alice" has created the following resource link share:
|
||||
| resource | text.txt |
|
||||
|
||||
@@ -283,8 +283,8 @@ Feature: enforce password on public link
|
||||
|
||||
|
||||
Scenario Outline: update a public link with a password that is listed in the Banned-Password-List
|
||||
Given the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/drone/banned-password-list.txt" for "sharing" service
|
||||
And the config "FRONTEND_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/drone/banned-password-list.txt" for "frontend" service
|
||||
Given the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/ci/banned-password-list.txt" for "sharing" service
|
||||
And the config "FRONTEND_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/ci/banned-password-list.txt" for "frontend" service
|
||||
And using OCS API version "2"
|
||||
And user "Alice" has been created with default attributes
|
||||
And user "Alice" has uploaded file with content "test file" to "/testfile.txt"
|
||||
@@ -307,8 +307,8 @@ Feature: enforce password on public link
|
||||
|
||||
|
||||
Scenario Outline: create a public link with a password that is listed in the Banned-Password-List
|
||||
Given the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/drone/banned-password-list.txt" for "sharing" service
|
||||
And the config "FRONTEND_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/drone/banned-password-list.txt" for "frontend" service
|
||||
Given the config "SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/ci/banned-password-list.txt" for "sharing" service
|
||||
And the config "FRONTEND_PASSWORD_POLICY_BANNED_PASSWORDS_LIST" has been set to path "config/ci/banned-password-list.txt" for "frontend" service
|
||||
And using OCS API version "2"
|
||||
And user "Alice" has been created with default attributes
|
||||
And user "Alice" has uploaded file with content "test file" to "/testfile.txt"
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
"""
|
||||
Run CS3 API validator tests locally and in GitHub Actions CI.
|
||||
|
||||
Config sourced from .drone.star cs3ApiTests() — single source of truth.
|
||||
Usage: python3 tests/acceptance/run-cs3api.py
|
||||
"""
|
||||
|
||||
@@ -15,7 +14,7 @@ import time
|
||||
from pathlib import Path
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants (mirroring .drone.star)
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# HTTPS — matching drone: ocis init generates a self-signed cert; proxy uses TLS by default.
|
||||
@@ -54,7 +53,7 @@ def base_server_env(repo_root: Path, ocis_config_dir: str, ocis_public_url: str)
|
||||
"NATS_NATS_PORT": "9233",
|
||||
"OCIS_JWT_SECRET": "some-ocis-jwt-secret",
|
||||
"EVENTHISTORY_STORE": "memory",
|
||||
"WEB_UI_CONFIG_FILE": str(repo_root / "tests/config/drone/ocis-config.json"),
|
||||
"WEB_UI_CONFIG_FILE": str(repo_root / "tests/config/ci/ocis-config.json"),
|
||||
# cs3api_validator extras (drone ocisServer deploy_type="cs3api_validator")
|
||||
"GATEWAY_GRPC_ADDR": "0.0.0.0:9142",
|
||||
"OCIS_SHARING_PUBLIC_SHARE_MUST_HAVE_PASSWORD": "false",
|
||||
@@ -102,7 +101,7 @@ def main() -> int:
|
||||
check=True,
|
||||
)
|
||||
shutil.copy(
|
||||
repo_root / "tests/config/drone/app-registry.yaml",
|
||||
repo_root / "tests/config/ci/app-registry.yaml",
|
||||
ocis_config_dir / "app-registry.yaml",
|
||||
)
|
||||
|
||||
|
||||
@@ -73,15 +73,15 @@ def main() -> int:
|
||||
|
||||
# clone + install web only if not already provided (e.g. via artifact)
|
||||
if not web_dir.exists():
|
||||
drone_env = {}
|
||||
drone_env_file = repo_root / ".drone.env"
|
||||
if drone_env_file.exists():
|
||||
for line in drone_env_file.read_text().splitlines():
|
||||
ci_env = {}
|
||||
ci_env_file = repo_root / "ci.env"
|
||||
if ci_env_file.exists():
|
||||
for line in ci_env_file.read_text().splitlines():
|
||||
if "=" in line and not line.startswith("#"):
|
||||
k, v = line.split("=", 1)
|
||||
drone_env[k.strip()] = v.strip()
|
||||
web_branch = drone_env.get("WEB_BRANCH", "master")
|
||||
web_commitid = drone_env.get("WEB_COMMITID", "")
|
||||
ci_env[k.strip()] = v.strip()
|
||||
web_branch = ci_env.get("WEB_BRANCH", "master")
|
||||
web_commitid = ci_env.get("WEB_COMMITID", "")
|
||||
run(["git", "clone", "-b", web_branch, "--single-branch", "--no-tags",
|
||||
WEB_REPO, str(web_dir)])
|
||||
if web_commitid:
|
||||
@@ -99,7 +99,7 @@ def main() -> int:
|
||||
# init ocis
|
||||
run([str(ocis_bin), "init", "--insecure", "true"])
|
||||
shutil.copy(
|
||||
repo_root / "tests/config/drone/app-registry.yaml",
|
||||
repo_root / "tests/config/ci/app-registry.yaml",
|
||||
ocis_config_dir / "app-registry.yaml",
|
||||
)
|
||||
|
||||
@@ -115,7 +115,7 @@ def main() -> int:
|
||||
|
||||
# Patch web UI config: replace Drone Docker service name with our URL
|
||||
drone_web_cfg = json.loads(
|
||||
(repo_root / "tests/config/drone/ocis-config.json").read_text()
|
||||
(repo_root / "tests/config/ci/ocis-config.json").read_text()
|
||||
)
|
||||
|
||||
def _patch_urls(obj, old, new):
|
||||
@@ -151,9 +151,9 @@ def main() -> int:
|
||||
"EVENTHISTORY_STORE": "memory",
|
||||
"OCIS_TRANSLATION_PATH": str(repo_root / "tests/config/translations"),
|
||||
"WEB_UI_CONFIG_FILE": str(gha_web_cfg_path),
|
||||
"THUMBNAILS_TXT_FONTMAP_FILE": str(repo_root / "tests/config/drone/fontsMap.json"),
|
||||
"THUMBNAILS_TXT_FONTMAP_FILE": str(repo_root / "tests/config/ci/fontsMap.json"),
|
||||
# extra_server_environment — matches drone e2eTestPipeline()
|
||||
"OCIS_PASSWORD_POLICY_BANNED_PASSWORDS_LIST": str(repo_root / "tests/config/drone/banned-password-list.txt"),
|
||||
"OCIS_PASSWORD_POLICY_BANNED_PASSWORDS_LIST": str(repo_root / "tests/config/ci/banned-password-list.txt"),
|
||||
"GRAPH_AVAILABLE_ROLES": "b1e2218d-eef8-4d4c-b82d-0f1a1b48f3b5,a8d5fe5e-96e3-418d-825b-534dbdf22b99,fb6c3e19-e378-47e5-b277-9732f9de6e21,58c63c02-1d89-4572-916a-870abc5a1b7d,2d00ce52-1fc2-4dbc-8b95-a73b73395f5a,1c996275-f1c9-4e71-abdf-a42f6495e960,312c0871-5ef7-4b3a-85b6-0e4074c64049,aa97fe03-7980-45ac-9e50-b325749fd7e6,63e64e19-8d43-42ec-a738-2b6af2610efa",
|
||||
"FRONTEND_CONFIGURABLE_NOTIFICATIONS": "true",
|
||||
# debug addresses
|
||||
@@ -215,7 +215,7 @@ def main() -> int:
|
||||
"OCIS_ADMIN_USER_ID": "",
|
||||
"GRAPH_ASSIGN_DEFAULT_USER_ROLE": "false",
|
||||
"GRAPH_USERNAME_MATCH": "none",
|
||||
"PROXY_CSP_CONFIG_FILE_LOCATION": str(repo_root / "tests/config/drone/csp.yaml"),
|
||||
"PROXY_CSP_CONFIG_FILE_LOCATION": str(repo_root / "tests/config/ci/csp.yaml"),
|
||||
"KEYCLOAK_DOMAIN": "localhost:8443",
|
||||
"IDM_CREATE_DEMO_USERS": "false",
|
||||
})
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
"""
|
||||
Run ocis acceptance tests locally and in GitHub Actions CI.
|
||||
|
||||
Config sourced from .drone.star localApiTests — single source of truth.
|
||||
Usage: BEHAT_SUITES=apiGraph python3 tests/acceptance/run-github.py
|
||||
"""
|
||||
|
||||
@@ -17,7 +16,7 @@ import shutil
|
||||
from pathlib import Path
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config sourced from .drone.star
|
||||
|
||||
# NOTE: EMAIL_SMTP_HOST is "email" (container name) in drone, "localhost" here
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -216,8 +215,8 @@ def base_server_env(repo_root: Path, ocis_url: str, ocis_config_dir: str) -> dic
|
||||
"OCIS_JWT_SECRET": "some-ocis-jwt-secret",
|
||||
"EVENTHISTORY_STORE": "memory",
|
||||
"OCIS_TRANSLATION_PATH": str(repo_root / "tests/config/translations"),
|
||||
"WEB_UI_CONFIG_FILE": str(repo_root / "tests/config/drone/ocis-config.json"),
|
||||
"THUMBNAILS_TXT_FONTMAP_FILE": str(repo_root / "tests/config/drone/fontsMap.json"),
|
||||
"WEB_UI_CONFIG_FILE": str(repo_root / "tests/config/ci/ocis-config.json"),
|
||||
"THUMBNAILS_TXT_FONTMAP_FILE": str(repo_root / "tests/config/ci/fontsMap.json"),
|
||||
# default tika off (overridden by search2 extraServerEnvironment)
|
||||
"SEARCH_EXTRACTOR_TYPE": "basic",
|
||||
"FRONTEND_FULL_TEXT_SEARCH_ENABLED": "false",
|
||||
@@ -417,7 +416,7 @@ def main() -> int:
|
||||
|
||||
# OCM federation: rewrite providers.json with localhost URLs
|
||||
if cfg["federationServer"]:
|
||||
providers_src = repo_root / "tests/config/drone/providers.json"
|
||||
providers_src = repo_root / "tests/config/ci/providers.json"
|
||||
providers = json.loads(providers_src.read_text())
|
||||
for p in providers:
|
||||
# replace container DNS names with localhost
|
||||
@@ -438,12 +437,12 @@ def main() -> int:
|
||||
# init ocis
|
||||
run([str(ocis_bin), "init", "--insecure", "true"])
|
||||
shutil.copy(
|
||||
repo_root / "tests/config/drone/app-registry.yaml",
|
||||
repo_root / "tests/config/ci/app-registry.yaml",
|
||||
ocis_config_dir / "app-registry.yaml",
|
||||
)
|
||||
|
||||
# generate fontsMap.json with correct font path (drone hardcodes /drone/src/...)
|
||||
font_path = str(repo_root / "tests/config/drone/NotoSans.ttf")
|
||||
# generate fontsMap.json with the correct absolute font path for this runner
|
||||
font_path = str(repo_root / "tests/config/ci/NotoSans.ttf")
|
||||
fontmap_tmp = tempfile.NamedTemporaryFile(
|
||||
mode="w", suffix=".json", prefix="fontsMap-", delete=False)
|
||||
json.dump({"defaultFont": font_path}, fontmap_tmp)
|
||||
@@ -491,7 +490,7 @@ def main() -> int:
|
||||
run([str(ocis_bin), "init", "--insecure", "true",
|
||||
"--config-path", str(fed_config_dir)])
|
||||
shutil.copy(
|
||||
repo_root / "tests/config/drone/app-registry.yaml",
|
||||
repo_root / "tests/config/ci/app-registry.yaml",
|
||||
fed_config_dir / "app-registry.yaml",
|
||||
)
|
||||
|
||||
@@ -513,11 +512,11 @@ def main() -> int:
|
||||
# → healthz never binds → 300s timeout. Use Python's built-in HTTP server
|
||||
# instead; it handles concurrent connections without gaps.
|
||||
run(["docker", "run", "-d", "--name", "fakeoffice", "--network", "host",
|
||||
"-v", f"{repo_root}:/drone/src:ro",
|
||||
"-v", f"{repo_root}:/ocis:ro",
|
||||
"python:3-alpine",
|
||||
"python3", "-c",
|
||||
"import http.server, pathlib\n"
|
||||
"body = pathlib.Path('/drone/src/tests/config/drone/hosting-discovery.xml').read_bytes()\n"
|
||||
"body = pathlib.Path('/ocis/tests/config/ci/hosting-discovery.xml').read_bytes()\n"
|
||||
"class H(http.server.BaseHTTPRequestHandler):\n"
|
||||
" def do_GET(self):\n"
|
||||
" self.send_response(200)\n"
|
||||
@@ -552,7 +551,7 @@ def main() -> int:
|
||||
# Drone avoids this because each service has its own network namespace.
|
||||
subprocess.run(["sudo", "systemctl", "stop", "postgresql"],
|
||||
capture_output=True)
|
||||
only_office_json = repo_root / "tests/config/drone/only-office.json"
|
||||
only_office_json = repo_root / "tests/config/ci/only-office.json"
|
||||
run(["docker", "run", "-d", "--name", "onlyoffice", "--network", "host",
|
||||
"-e", "WOPI_ENABLED=true",
|
||||
"-e", "USE_UNAUTHORIZED_STORAGE=true",
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
"""
|
||||
Run litmus WebDAV compliance tests locally and in GitHub Actions CI.
|
||||
|
||||
Config sourced from .drone.star litmus() / setupForLitmus() — single source of truth.
|
||||
Usage: python3 tests/acceptance/run-litmus.py
|
||||
"""
|
||||
|
||||
@@ -17,7 +16,7 @@ import time
|
||||
from pathlib import Path
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants (mirroring .drone.star)
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# HTTPS — matching drone: ocis init generates a self-signed cert; proxy uses TLS by default.
|
||||
@@ -57,7 +56,7 @@ def base_server_env(repo_root: Path, ocis_config_dir: str, ocis_public_url: str)
|
||||
"NATS_NATS_PORT": "9233",
|
||||
"OCIS_JWT_SECRET": "some-ocis-jwt-secret",
|
||||
"EVENTHISTORY_STORE": "memory",
|
||||
"WEB_UI_CONFIG_FILE": str(repo_root / "tests/config/drone/ocis-config.json"),
|
||||
"WEB_UI_CONFIG_FILE": str(repo_root / "tests/config/ci/ocis-config.json"),
|
||||
}
|
||||
|
||||
|
||||
@@ -82,7 +81,7 @@ def ocis_healthy(ocis_url: str) -> bool:
|
||||
|
||||
def setup_for_litmus(ocis_url: str) -> tuple:
|
||||
"""
|
||||
Translate tests/config/drone/setup-for-litmus.sh to Python.
|
||||
Translate tests/config/ci/setup-for-litmus.sh to Python.
|
||||
Returns (space_id, public_token).
|
||||
"""
|
||||
# get personal space ID
|
||||
@@ -189,7 +188,7 @@ def main() -> int:
|
||||
check=True,
|
||||
)
|
||||
shutil.copy(
|
||||
repo_root / "tests/config/drone/app-registry.yaml",
|
||||
repo_root / "tests/config/ci/app-registry.yaml",
|
||||
ocis_config_dir / "app-registry.yaml",
|
||||
)
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
"""
|
||||
Run WOPI validator tests locally and in GitHub Actions CI.
|
||||
|
||||
Config sourced from .drone.star wopiValidatorTests() — single source of truth.
|
||||
Usage: python3 tests/acceptance/run-wopi.py --type builtin
|
||||
python3 tests/acceptance/run-wopi.py --type cs3
|
||||
"""
|
||||
@@ -21,7 +20,7 @@ import urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants (mirroring .drone.star)
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# HTTPS — matching drone; host-side curl calls use -k.
|
||||
@@ -119,7 +118,7 @@ def base_server_env(repo_root: Path, ocis_config_dir: str, ocis_public_url: str,
|
||||
"NATS_NATS_PORT": "9233",
|
||||
"OCIS_JWT_SECRET": "some-ocis-jwt-secret",
|
||||
"EVENTHISTORY_STORE": "memory",
|
||||
"WEB_UI_CONFIG_FILE": str(repo_root / "tests/config/drone/ocis-config.json"),
|
||||
"WEB_UI_CONFIG_FILE": str(repo_root / "tests/config/ci/ocis-config.json"),
|
||||
# wopi_validator extras (drone ocisServer deploy_type="wopi_validator")
|
||||
"GATEWAY_GRPC_ADDR": "0.0.0.0:9142",
|
||||
"APP_PROVIDER_EXTERNAL_ADDR": "com.owncloud.api.app-provider",
|
||||
@@ -265,16 +264,15 @@ def main() -> int:
|
||||
|
||||
try:
|
||||
# --- fakeoffice: serves hosting-discovery.xml on :8080 ---
|
||||
# Mirrors drone fakeOffice() — owncloudci/alpine running serve-hosting-discovery.sh.
|
||||
# Repo is mounted at /drone/src (the path the script uses).
|
||||
# Mirrors the old fakeOffice() step — owncloudci/alpine running serve-hosting-discovery.sh.
|
||||
containers.append("wopi-fakeoffice-fake")
|
||||
subprocess.run(["docker", "rm", "-f", "wopi-fakeoffice-fake"], capture_output=True)
|
||||
subprocess.run([
|
||||
"docker", "run", "-d", "--name", "wopi-fakeoffice-fake",
|
||||
"-p", "8080:8080",
|
||||
"-v", f"{repo_root}:/drone/src",
|
||||
"-v", f"{repo_root}:/ocis",
|
||||
FAKEOFFICE_IMAGE,
|
||||
"sh", "/drone/src/tests/config/drone/serve-hosting-discovery.sh",
|
||||
"sh", "/ocis/tests/config/ci/serve-hosting-discovery.sh",
|
||||
], check=True)
|
||||
|
||||
wait_for(lambda: tcp_reachable(bridge_ip, 8080), 60, "fakeoffice:8080")
|
||||
@@ -291,7 +289,7 @@ def main() -> int:
|
||||
env=server_env, check=True,
|
||||
)
|
||||
shutil.copy(
|
||||
repo_root / "tests/config/drone/app-registry.yaml",
|
||||
repo_root / "tests/config/ci/app-registry.yaml",
|
||||
ocis_config_dir / "app-registry.yaml",
|
||||
)
|
||||
|
||||
@@ -324,7 +322,7 @@ def main() -> int:
|
||||
else:
|
||||
# cs3: patch wopiserver.conf (replace container hostname with bridge_ip),
|
||||
# then run cs3org/wopiserver as a Docker container.
|
||||
conf_text = (repo_root / "tests/config/drone/wopiserver.conf").read_text()
|
||||
conf_text = (repo_root / "tests/config/ci/wopiserver.conf").read_text()
|
||||
conf_text = conf_text.replace("ocis-server", bridge_ip)
|
||||
conf_tmp = Path("/tmp/wopiserver-patched.conf")
|
||||
conf_tmp.write_text(conf_text)
|
||||
|
||||
@@ -677,18 +677,4 @@ then
|
||||
${RED_COLOR}; printf "%s\n" "${UNEXPECTED_BEHAT_EXIT_STATUSES[@]}"
|
||||
fi
|
||||
|
||||
# # sync the file-system so all output will be flushed to storage.
|
||||
# # In drone we sometimes see that the last lines of output are missing from the
|
||||
# # drone log.
|
||||
# sync
|
||||
|
||||
# # If we are running in drone CI, then sleep for a bit to (hopefully) let the
|
||||
# # drone agent send all the output to the drone server.
|
||||
# NOTE: Not using this anymore, please enable it if needed
|
||||
# if [ -n "${CI_REPO}" ]
|
||||
# then
|
||||
# echo "sleeping for 30 seconds at end of test run"
|
||||
# sleep 30
|
||||
# fi
|
||||
|
||||
exit ${FINAL_EXIT_STATUS}
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
ROOT_PATH="$1"
|
||||
if [ -z "$1" ]; then
|
||||
ROOT_PATH="/drone/src"
|
||||
ROOT_PATH="."
|
||||
fi
|
||||
BINGO_DIR="$ROOT_PATH/.bingo"
|
||||
|
||||
@@ -19,8 +19,6 @@ go_cache=$(mc find s3/$CACHE_BUCKET/ocis/go-bin/$BINGO_HASH/$2 2>&1 | grep 'Obje
|
||||
if [[ -z "$go_cache" ]]
|
||||
then
|
||||
echo "[INFO] Go bin cache with has '$BINGO_HASH' exists."
|
||||
# https://discourse.drone.io/t/how-to-exit-a-pipeline-early-without-failing/3951
|
||||
# exit a Pipeline early without failing
|
||||
exit 78
|
||||
else
|
||||
# stored hash of a .bingo folder to '.bingo_hash' file
|
||||
@@ -1,5 +1,7 @@
|
||||
#!/bin/bash
|
||||
source .drone.env
|
||||
# Checks S3 MinIO cache for web test runner artifacts (acceptance/e2e/web).
|
||||
# Sources ci.env to get WEB_COMMITID used as the cache key.
|
||||
source ci.env
|
||||
|
||||
# if no $1 is supplied end the script
|
||||
# Can be web, acceptance or e2e
|
||||
@@ -13,8 +15,6 @@ web_cache=$(mc find s3/$CACHE_BUCKET/ocis/web-test-runner/$WEB_COMMITID/$1 2>&1
|
||||
|
||||
if [[ -z "$web_cache" ]]; then
|
||||
echo "$1 cache with commit id $WEB_COMMITID already available."
|
||||
# https://discourse.drone.io/t/how-to-exit-a-pipeline-early-without-failing/3951
|
||||
# exit a Pipeline early without failing
|
||||
exit 78
|
||||
else
|
||||
echo "$1 cache with commit id $WEB_COMMITID was not available."
|
||||
3
tests/config/ci/fontsMap.json
Normal file
3
tests/config/ci/fontsMap.json
Normal file
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"defaultFont": "/ocis/tests/config/ci/NotoSans.ttf"
|
||||
}
|
||||
5
tests/config/ci/serve-hosting-discovery.sh
Normal file
5
tests/config/ci/serve-hosting-discovery.sh
Normal file
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
|
||||
while true; do
|
||||
echo -e "HTTP/1.1 200 OK\n\n$(cat /ocis/tests/config/ci/hosting-discovery.xml)" | nc -l -k -p 8080
|
||||
done
|
||||
@@ -1,3 +0,0 @@
|
||||
{
|
||||
"defaultFont": "/drone/src/tests/config/drone/NotoSans.ttf"
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
while true; do
|
||||
echo -e "HTTP/1.1 200 OK\n\n$(cat /drone/src/tests/config/drone/hosting-discovery.xml)" | nc -l -k -p 8080
|
||||
done
|
||||
@@ -25,7 +25,7 @@ prepare-charts:
|
||||
@test -d "$(CHART_REPO)" || git clone --single-branch -b main --depth 1 https://github.com/owncloud/ocis-charts.git $(CHART_REPO)
|
||||
bash setup.sh $(CHART_REPO)
|
||||
kubectl create namespace $(DOMAIN) || true
|
||||
kubectl create configmap -n $(DOMAIN) ocis-fonts-ttf --from-file=../drone/NotoSans.ttf || true
|
||||
kubectl create configmap -n $(DOMAIN) ocis-fonts-ttf --from-file=../ci/NotoSans.ttf || true
|
||||
|
||||
.PHONY: deploy-ocis
|
||||
deploy-ocis:
|
||||
|
||||
@@ -1,379 +0,0 @@
|
||||
// Usage examples:
|
||||
// go run ./cmd/list-failed-commits --max-commits 20
|
||||
// go run ./cmd/list-failed-commits --max-commits 50 --out failed.json
|
||||
// go run ./cmd/list-failed-commits --github-token $GITHUB_TOKEN --max-commits 100
|
||||
// go run ./cmd/list-failed-commits --repo owncloud/ocis --branch master --max-commits 20
|
||||
// go run ./cmd/list-failed-commits --since 7d --max-commits 50
|
||||
// go run ./cmd/list-failed-commits --since 30d --out failed.json
|
||||
// go run ./cmd/list-failed-commits --since 2026-01-01 --max-commits 100
|
||||
// go run ./cmd/list-failed-commits --since 2026-02-15 --out recent.json
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/owncloud/ocis/v2/tools/ci-reporter/pkg/droneextractor"
|
||||
"github.com/owncloud/ocis/v2/tools/ci-reporter/pkg/githubextractor"
|
||||
"github.com/owncloud/ocis/v2/tools/ci-reporter/pkg/util"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultRepo = "owncloud/ocis"
|
||||
defaultBranch = "master"
|
||||
requestDelay = 1 * time.Second
|
||||
defaultLimit = 20
|
||||
defaultUnlimited = 999999999
|
||||
)
|
||||
|
||||
type RunConfig struct {
|
||||
Repo string
|
||||
Branch string
|
||||
MaxCommits int
|
||||
SinceRaw string
|
||||
SinceParsed string
|
||||
OutFile string
|
||||
Token string
|
||||
Command string
|
||||
FetchFailedPipelineHistory bool
|
||||
}
|
||||
|
||||
type FailureCount struct {
|
||||
StageName string `json:"stage_name"`
|
||||
StepName string `json:"step_name"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
type DateRange struct {
|
||||
StartDate string `json:"start_date"`
|
||||
EndDate string `json:"end_date"`
|
||||
}
|
||||
|
||||
type Raport struct {
|
||||
Args string `json:"args"`
|
||||
Repo string `json:"repo"`
|
||||
Branch string `json:"branch"`
|
||||
DateRange DateRange `json:"date_range"`
|
||||
TotalCommits int `json:"total_commits"`
|
||||
FailedCommits int `json:"failed_commits"`
|
||||
Percentage float64 `json:"percentage"`
|
||||
}
|
||||
|
||||
type RaportMetadata struct {
|
||||
Args string
|
||||
Repo string
|
||||
Branch string
|
||||
DateRange DateRange
|
||||
TotalCommits int
|
||||
}
|
||||
|
||||
type Report struct {
|
||||
Raport *Raport `json:"raport"`
|
||||
Count []FailureCount `json:"count"`
|
||||
Pipelines []FailedCommit `json:"pipelines"`
|
||||
}
|
||||
|
||||
type FailedCommit struct {
|
||||
PR int `json:"pr,omitempty"`
|
||||
SHA string `json:"sha"`
|
||||
Date string `json:"date"`
|
||||
Subject string `json:"subject"`
|
||||
HTMLURL string `json:"html_url"`
|
||||
CombinedState string `json:"combined_state"`
|
||||
FailedContexts []StatusContext `json:"failed_contexts"`
|
||||
PipelineInfo *droneextractor.PipelineInfo `json:"pipeline_info,omitempty"`
|
||||
FailedPipelineHistory []droneextractor.PipelineInfoSummary `json:"failed_pipeline_history,omitempty"`
|
||||
PrPipelineHistory []droneextractor.PipelineInfoSummary `json:"pr_pipeline_history,omitempty"`
|
||||
}
|
||||
|
||||
type StatusContext struct {
|
||||
Context string `json:"context"`
|
||||
State string `json:"state"`
|
||||
TargetURL string `json:"target_url"`
|
||||
Description string `json:"description"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
repo := flag.String("repo", defaultRepo, "GitHub repository (owner/repo)")
|
||||
branch := flag.String("branch", defaultBranch, "Branch name")
|
||||
maxCommits := flag.Int("max-commits", 0, "Maximum commits to check (default: 20 if no --since, unlimited if --since set)")
|
||||
since := flag.String("since", "", "Date filter: YYYY-MM-DD, RFC3339 timestamp, or 'Nd' for last N days (e.g., '7d', '30d')")
|
||||
outFile := flag.String("out", "", "Output file path (default: stdout)")
|
||||
githubToken := flag.String("github-token", "", "GitHub token (optional; falls back to env GITHUB_TOKEN)")
|
||||
failedPipelineHistory := flag.Bool("failed_pipeline_history", false, "if set, fetch failed_pipeline_history per commit (extra Drone API calls)")
|
||||
flag.Parse()
|
||||
|
||||
token := strings.TrimSpace(*githubToken)
|
||||
if token == "" {
|
||||
token = strings.TrimSpace(os.Getenv("GITHUB_TOKEN"))
|
||||
}
|
||||
|
||||
if token != "" {
|
||||
fmt.Fprintf(os.Stderr, "Using GitHub token\n")
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "No token provided, unauthenticated, 60 req/hr limit\n")
|
||||
}
|
||||
|
||||
command := strings.Join(os.Args[1:], " ")
|
||||
command = util.RedactGitHubToken(command)
|
||||
|
||||
parsedSince, err := util.ParseSinceFlag(*since)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error parsing --since flag: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Determine final max commits based on git-like UX:
|
||||
// - No flags: default to 20
|
||||
// - --since only: unlimited (all in range)
|
||||
// - --max-commits only: use explicit value
|
||||
// - Both: use explicit value (both filters apply)
|
||||
finalMaxCommits := *maxCommits
|
||||
if finalMaxCommits == 0 {
|
||||
if *since != "" {
|
||||
finalMaxCommits = defaultUnlimited // Unlimited when --since is used
|
||||
} else {
|
||||
finalMaxCommits = defaultLimit // Default for no flags
|
||||
}
|
||||
}
|
||||
|
||||
config := RunConfig{
|
||||
Repo: *repo,
|
||||
Branch: *branch,
|
||||
MaxCommits: finalMaxCommits,
|
||||
SinceRaw: *since,
|
||||
SinceParsed: parsedSince,
|
||||
OutFile: *outFile,
|
||||
Token: token,
|
||||
Command: command,
|
||||
FetchFailedPipelineHistory: *failedPipelineHistory,
|
||||
}
|
||||
|
||||
if err := run(config); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(cfg RunConfig) error {
|
||||
// Configure Transport to prevent stale connection reuse during long pagination
|
||||
transport := &http.Transport{
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 30 * time.Second,
|
||||
DisableKeepAlives: false,
|
||||
MaxIdleConnsPerHost: 2,
|
||||
}
|
||||
client := &http.Client{
|
||||
Timeout: 60 * time.Second, // Increased for long-running pagination
|
||||
Transport: transport,
|
||||
}
|
||||
githubExtractor := githubextractor.NewExtractor(client, cfg.Token)
|
||||
droneExtractor := droneextractor.NewExtractor(client)
|
||||
|
||||
// Build scanning message
|
||||
var scanMsg string
|
||||
if cfg.SinceRaw != "" {
|
||||
scanMsg = fmt.Sprintf("Scanning commits since %s from %s/%s", cfg.SinceRaw, cfg.Repo, cfg.Branch)
|
||||
} else {
|
||||
scanMsg = fmt.Sprintf("Scanning up to %d commits from %s/%s", cfg.MaxCommits, cfg.Repo, cfg.Branch)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "%s\n", scanMsg)
|
||||
commits, err := githubExtractor.GetCommits(cfg.Repo, cfg.Branch, cfg.MaxCommits, cfg.SinceParsed)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetching commits: %w", err)
|
||||
}
|
||||
|
||||
var failedCommits []FailedCommit
|
||||
for _, commit := range commits {
|
||||
status, err := githubExtractor.GetCommitStatus(cfg.Repo, commit.SHA)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "⚠ %s: %v\n", commit.SHA[:7], err)
|
||||
time.Sleep(requestDelay)
|
||||
continue
|
||||
}
|
||||
|
||||
if status.State == "failure" || status.State == "error" {
|
||||
var failedCtxs []StatusContext
|
||||
var pipelineInfo *droneextractor.PipelineInfo
|
||||
|
||||
for _, s := range status.Statuses {
|
||||
if s.State != "success" {
|
||||
// Skip Drone entries - they'll be in pipeline_info instead
|
||||
if droneExtractor.IsDroneURL(s.TargetURL) {
|
||||
info, err := droneExtractor.Extract(s.TargetURL)
|
||||
if err == nil {
|
||||
pipelineInfo = info
|
||||
}
|
||||
// Don't add to failed_contexts to avoid redundancy
|
||||
continue
|
||||
}
|
||||
failedCtxs = append(failedCtxs, toStatusContext(s))
|
||||
}
|
||||
}
|
||||
// TODO: Refactor redundant field assignment - prPipelineHistory always assigned, failedPipelineHistory conditional
|
||||
var failedPipelineHistory []droneextractor.PipelineInfoSummary
|
||||
var prPipelineHistory []droneextractor.PipelineInfoSummary
|
||||
var prNumber int
|
||||
var headRef string
|
||||
|
||||
// Get PR for this commit (master commit -> PR ID -> PR branch)
|
||||
prs, err := githubExtractor.GetPRsForCommit(cfg.Repo, commit.SHA)
|
||||
if err == nil && len(prs) > 0 {
|
||||
prNumber = prs[0].Number
|
||||
headRef = prs[0].Head.Ref
|
||||
|
||||
// Get failed builds by PR branch (black box: all logic in droneextractor)
|
||||
failedBuilds, err := droneExtractor.GetFailedBuildsByPRBranch(droneextractor.BaseURL, cfg.Repo, headRef)
|
||||
if err == nil {
|
||||
prPipelineHistory = failedBuilds
|
||||
if cfg.FetchFailedPipelineHistory {
|
||||
failedPipelineHistory = failedBuilds
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize empty slice if flag is set but no PR/builds found
|
||||
if cfg.FetchFailedPipelineHistory && failedPipelineHistory == nil {
|
||||
failedPipelineHistory = []droneextractor.PipelineInfoSummary{}
|
||||
}
|
||||
|
||||
subject := util.ExtractSubject(commit.Commit.Message)
|
||||
failedCommits = append(failedCommits, FailedCommit{
|
||||
PR: prNumber,
|
||||
SHA: commit.SHA,
|
||||
Date: commit.Commit.Author.Date,
|
||||
Subject: subject,
|
||||
HTMLURL: commit.HTMLURL,
|
||||
CombinedState: status.State,
|
||||
FailedContexts: failedCtxs,
|
||||
PipelineInfo: pipelineInfo,
|
||||
FailedPipelineHistory: failedPipelineHistory,
|
||||
PrPipelineHistory: prPipelineHistory,
|
||||
})
|
||||
|
||||
// Single line output: ✗ sha subject -> PR #num (branch: name)
|
||||
if prNumber > 0 && headRef != "" {
|
||||
fmt.Fprintf(os.Stderr, "✗ %s %s -> PR #%d (branch: %s)\n", commit.SHA[:7], subject, prNumber, headRef)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "✗ %s %s\n", commit.SHA[:7], subject)
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(requestDelay)
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "\nResult: %d failed / %d total\n", len(failedCommits), len(commits))
|
||||
|
||||
var minDate, maxDate string
|
||||
for _, c := range commits {
|
||||
date := c.Commit.Author.Date
|
||||
if minDate == "" || date < minDate {
|
||||
minDate = date
|
||||
}
|
||||
if maxDate == "" || date > maxDate {
|
||||
maxDate = date
|
||||
}
|
||||
}
|
||||
|
||||
metadata := RaportMetadata{
|
||||
Args: cfg.Command,
|
||||
Repo: cfg.Repo,
|
||||
Branch: cfg.Branch,
|
||||
DateRange: DateRange{StartDate: minDate, EndDate: maxDate},
|
||||
TotalCommits: len(commits),
|
||||
}
|
||||
|
||||
report := generateReport(failedCommits, metadata)
|
||||
|
||||
var w io.Writer = os.Stdout
|
||||
if cfg.OutFile != "" {
|
||||
f, err := os.Create(cfg.OutFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating output file: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
w = f
|
||||
fmt.Fprintf(os.Stderr, "Writing JSON to %s...\n", cfg.OutFile)
|
||||
}
|
||||
|
||||
enc := json.NewEncoder(w)
|
||||
enc.SetIndent("", " ")
|
||||
if err := enc.Encode(report); err != nil {
|
||||
return fmt.Errorf("encoding JSON: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func toStatusContext(ctx githubextractor.StatusContext) StatusContext {
|
||||
return StatusContext{
|
||||
Context: ctx.Context,
|
||||
State: ctx.State,
|
||||
TargetURL: ctx.TargetURL,
|
||||
Description: ctx.Description,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
func generateReport(failedCommits []FailedCommit, metadata RaportMetadata) *Report {
|
||||
countMap := make(map[string]int)
|
||||
|
||||
for _, commit := range failedCommits {
|
||||
if commit.PipelineInfo == nil {
|
||||
continue
|
||||
}
|
||||
for _, stage := range commit.PipelineInfo.PipelineStages {
|
||||
for _, step := range stage.Steps {
|
||||
if step.Status == "failure" {
|
||||
key := stage.StageName + "/" + step.StepName
|
||||
countMap[key]++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var counts []FailureCount
|
||||
for key, count := range countMap {
|
||||
parts := strings.Split(key, "/")
|
||||
if len(parts) >= 2 {
|
||||
counts = append(counts, FailureCount{
|
||||
StageName: parts[0],
|
||||
StepName: strings.Join(parts[1:], "/"),
|
||||
Count: count,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(counts, func(i, j int) bool {
|
||||
return counts[i].Count > counts[j].Count
|
||||
})
|
||||
|
||||
percentage := 0.0
|
||||
if metadata.TotalCommits > 0 {
|
||||
percentage = float64(len(failedCommits)) / float64(metadata.TotalCommits) * 100.0
|
||||
}
|
||||
|
||||
raport := &Raport{
|
||||
Args: metadata.Args,
|
||||
Repo: metadata.Repo,
|
||||
Branch: metadata.Branch,
|
||||
DateRange: metadata.DateRange,
|
||||
TotalCommits: metadata.TotalCommits,
|
||||
FailedCommits: len(failedCommits),
|
||||
Percentage: percentage,
|
||||
}
|
||||
|
||||
return &Report{
|
||||
Raport: raport,
|
||||
Count: counts,
|
||||
Pipelines: failedCommits,
|
||||
}
|
||||
}
|
||||
269
tools/ci-reporter/cmd/list-flaky-tests/main.go
Normal file
269
tools/ci-reporter/cmd/list-flaky-tests/main.go
Normal file
@@ -0,0 +1,269 @@
|
||||
// Usage examples:
|
||||
// go run ./cmd/list-flaky-tests --since 14d
|
||||
// go run ./cmd/list-flaky-tests --since 3m --workflow acceptance-tests.yml
|
||||
// go run ./cmd/list-flaky-tests --github-token $GITHUB_TOKEN --since 30d
|
||||
// go run ./cmd/list-flaky-tests --repo owncloud/ocis --since 7d --out report.json
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/owncloud/ocis/v2/tools/ci-reporter/pkg/ghaextractor"
|
||||
"github.com/owncloud/ocis/v2/tools/ci-reporter/pkg/util"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultRepo = "owncloud/ocis"
|
||||
defaultWorkflow = "acceptance-tests.yml"
|
||||
)
|
||||
|
||||
type Summary struct {
|
||||
Args string `json:"args"`
|
||||
Repo string `json:"repo"`
|
||||
Workflow string `json:"workflow"`
|
||||
DateRange Range `json:"date_range"`
|
||||
TotalRuns int `json:"total_runs"`
|
||||
RestartedRuns int `json:"restarted_runs"`
|
||||
RestartRatePct float64 `json:"restart_rate_pct"`
|
||||
TotalTimeLostMinutes float64 `json:"total_time_lost_minutes"`
|
||||
}
|
||||
|
||||
type Range struct {
|
||||
Start string `json:"start"`
|
||||
End string `json:"end"`
|
||||
}
|
||||
|
||||
type FlakyJob struct {
|
||||
JobName string `json:"job_name"`
|
||||
FailThenPassCount int `json:"fail_then_pass_count"`
|
||||
FailureRatePct float64 `json:"failure_rate_pct"`
|
||||
}
|
||||
|
||||
type Report struct {
|
||||
Summary Summary `json:"summary"`
|
||||
FlakyJobs []FlakyJob `json:"flaky_jobs"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
repo := flag.String("repo", defaultRepo, "GitHub repository (owner/repo)")
|
||||
workflow := flag.String("workflow", defaultWorkflow, "Workflow filename (e.g. acceptance-tests.yml)")
|
||||
since := flag.String("since", "14d", "Date filter: YYYY-MM-DD, RFC3339, or 'Nd'/'Nm' (e.g. '7d', '3m')")
|
||||
outFile := flag.String("out", "", "Output file path (default: stdout)")
|
||||
githubToken := flag.String("github-token", "", "GitHub token (falls back to GITHUB_TOKEN env)")
|
||||
flag.Parse()
|
||||
|
||||
token := strings.TrimSpace(*githubToken)
|
||||
if token == "" {
|
||||
token = strings.TrimSpace(os.Getenv("GITHUB_TOKEN"))
|
||||
}
|
||||
if token != "" {
|
||||
fmt.Fprintf(os.Stderr, "Using GitHub token\n")
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "No token provided, unauthenticated (60 req/hr limit)\n")
|
||||
}
|
||||
|
||||
sinceRaw := *since
|
||||
// Support Nm month notation (not in ParseSinceFlag)
|
||||
sinceRFC, err := parseSince(sinceRaw)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error parsing --since: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
command := strings.Join(os.Args[1:], " ")
|
||||
command = util.RedactGitHubToken(command)
|
||||
|
||||
transport := &http.Transport{
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 30 * time.Second,
|
||||
MaxIdleConnsPerHost: 2,
|
||||
}
|
||||
client := &http.Client{
|
||||
Timeout: 60 * time.Second,
|
||||
Transport: transport,
|
||||
}
|
||||
extractor := ghaextractor.NewExtractor(client, token)
|
||||
|
||||
fmt.Fprintf(os.Stderr, "Fetching workflow runs for %s/%s since %s...\n", *repo, *workflow, sinceRaw)
|
||||
runs, err := extractor.GetWorkflowRuns(*repo, *workflow, sinceRFC)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error fetching runs: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "Found %d completed runs\n", len(runs))
|
||||
|
||||
report, err := buildReport(extractor, *repo, *workflow, runs, command)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error building report: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
var w io.Writer = os.Stdout
|
||||
if *outFile != "" {
|
||||
f, err := os.Create(*outFile)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error creating output file: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer f.Close()
|
||||
w = f
|
||||
fmt.Fprintf(os.Stderr, "Writing JSON to %s...\n", *outFile)
|
||||
}
|
||||
|
||||
enc := json.NewEncoder(w)
|
||||
enc.SetIndent("", " ")
|
||||
if err := enc.Encode(report); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error encoding JSON: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func buildReport(extractor *ghaextractor.Extractor, repo, workflow string, runs []ghaextractor.WorkflowRun, command string) (*Report, error) {
|
||||
// The API returns one record per attempt, so a pipeline retried twice yields 3 records.
|
||||
// Deduplicate by head_sha, keeping the highest-attempt record for each unique pipeline.
|
||||
type pipelineRecord struct {
|
||||
latest ghaextractor.WorkflowRun
|
||||
maxAttempt int
|
||||
}
|
||||
bySHA := make(map[string]*pipelineRecord)
|
||||
for _, run := range runs {
|
||||
r := bySHA[run.HeadSHA]
|
||||
if r == nil {
|
||||
bySHA[run.HeadSHA] = &pipelineRecord{latest: run, maxAttempt: run.RunAttempt}
|
||||
} else if run.RunAttempt > r.maxAttempt {
|
||||
r.latest = run
|
||||
r.maxAttempt = run.RunAttempt
|
||||
}
|
||||
}
|
||||
|
||||
pipelines := make([]pipelineRecord, 0, len(bySHA))
|
||||
for _, r := range bySHA {
|
||||
pipelines = append(pipelines, *r)
|
||||
}
|
||||
|
||||
var minDate, maxDate string
|
||||
for _, r := range pipelines {
|
||||
d := r.latest.CreatedAt
|
||||
if minDate == "" || d < minDate {
|
||||
minDate = d
|
||||
}
|
||||
if maxDate == "" || d > maxDate {
|
||||
maxDate = d
|
||||
}
|
||||
}
|
||||
|
||||
totalPipelines := len(pipelines)
|
||||
restartedRuns := 0
|
||||
var timeLostMinutes float64
|
||||
// job name -> count of pipelines where that job was retried (any re-run, regardless of outcome)
|
||||
flakyCount := make(map[string]int)
|
||||
for i, r := range pipelines {
|
||||
run := r.latest
|
||||
fmt.Fprintf(os.Stderr, " [%d/%d] pipeline %d (max attempt %d)...\n", i+1, len(pipelines), run.ID, r.maxAttempt)
|
||||
|
||||
wasRestarted := r.maxAttempt > 1
|
||||
if !wasRestarted {
|
||||
continue
|
||||
}
|
||||
restartedRuns++
|
||||
|
||||
// Fetch all jobs across all attempts for this run to find which jobs were re-run
|
||||
// and measure actual wall-clock time lost on retry attempts.
|
||||
jobs, err := extractor.GetRunJobs(repo, run.ID)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, " ⚠ run %d: %v\n", run.ID, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Track which job names appeared in attempt > 1 (= developer had to wait for them again)
|
||||
retriedJobNames := make(map[string]bool)
|
||||
var retryStart, retryEnd time.Time
|
||||
for _, j := range jobs {
|
||||
if j.RunAttempt <= 1 || j.Name == "all-acceptance-tests" {
|
||||
continue
|
||||
}
|
||||
retriedJobNames[j.Name] = true
|
||||
if j.StartedAt != "" && j.CompletedAt != "" {
|
||||
if s, err := time.Parse(time.RFC3339, j.StartedAt); err == nil {
|
||||
if retryStart.IsZero() || s.Before(retryStart) {
|
||||
retryStart = s
|
||||
}
|
||||
}
|
||||
if e, err := time.Parse(time.RFC3339, j.CompletedAt); err == nil {
|
||||
if e.After(retryEnd) {
|
||||
retryEnd = e
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for name := range retriedJobNames {
|
||||
flakyCount[name]++
|
||||
}
|
||||
if !retryStart.IsZero() && retryEnd.After(retryStart) {
|
||||
timeLostMinutes += retryEnd.Sub(retryStart).Minutes()
|
||||
}
|
||||
}
|
||||
|
||||
restartRatePct := 0.0
|
||||
if totalPipelines > 0 {
|
||||
restartRatePct = float64(restartedRuns) / float64(totalPipelines) * 100.0
|
||||
}
|
||||
|
||||
flakyJobs := make([]FlakyJob, 0, len(flakyCount))
|
||||
for name, count := range flakyCount {
|
||||
ratePct := 0.0
|
||||
if totalPipelines > 0 {
|
||||
ratePct = float64(count) / float64(totalPipelines) * 100.0
|
||||
}
|
||||
flakyJobs = append(flakyJobs, FlakyJob{
|
||||
JobName: name,
|
||||
FailThenPassCount: count,
|
||||
FailureRatePct: ratePct,
|
||||
})
|
||||
}
|
||||
sort.Slice(flakyJobs, func(i, j int) bool {
|
||||
return flakyJobs[i].FailThenPassCount > flakyJobs[j].FailThenPassCount
|
||||
})
|
||||
|
||||
return &Report{
|
||||
Summary: Summary{
|
||||
Args: command,
|
||||
Repo: repo,
|
||||
Workflow: workflow,
|
||||
DateRange: Range{Start: minDate, End: maxDate},
|
||||
TotalRuns: totalPipelines,
|
||||
RestartedRuns: restartedRuns,
|
||||
RestartRatePct: restartRatePct,
|
||||
TotalTimeLostMinutes: timeLostMinutes,
|
||||
},
|
||||
FlakyJobs: flakyJobs,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
// parseSince extends util.ParseSinceFlag with Nm (months) and Nw (weeks) support.
|
||||
func parseSince(s string) (string, error) {
|
||||
if numStr, ok := strings.CutSuffix(s, "m"); ok {
|
||||
var months int
|
||||
if _, err := fmt.Sscanf(numStr, "%d", &months); err == nil {
|
||||
return time.Now().AddDate(0, -months, 0).Format(time.RFC3339), nil
|
||||
}
|
||||
}
|
||||
if numStr, ok := strings.CutSuffix(s, "w"); ok {
|
||||
var weeks int
|
||||
if _, err := fmt.Sscanf(numStr, "%d", &weeks); err == nil {
|
||||
return time.Now().AddDate(0, 0, -weeks*7).Format(time.RFC3339), nil
|
||||
}
|
||||
}
|
||||
return util.ParseSinceFlag(s)
|
||||
}
|
||||
@@ -1,5 +1,3 @@
|
||||
module github.com/owncloud/ocis/v2/tools/ci-reporter
|
||||
|
||||
go 1.24.0
|
||||
|
||||
require golang.org/x/net v0.50.0 // indirect
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
|
||||
golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
|
||||
|
||||
@@ -1,631 +0,0 @@
|
||||
package droneextractor
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
const (
|
||||
// BaseURL is the default Drone instance URL
|
||||
BaseURL = "https://drone.owncloud.com"
|
||||
)
|
||||
|
||||
type Extractor struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func NewExtractor(client *http.Client) *Extractor {
|
||||
return &Extractor{client: client}
|
||||
}
|
||||
|
||||
// IsDroneURL checks if a URL is a Drone CI URL
|
||||
func (e *Extractor) IsDroneURL(url string) bool {
|
||||
return strings.Contains(url, "drone.owncloud.com")
|
||||
}
|
||||
|
||||
func (e *Extractor) Extract(buildURL string) (*PipelineInfo, error) {
|
||||
apiURL := e.buildURLToAPIURL(buildURL)
|
||||
if apiURL == "" {
|
||||
return nil, fmt.Errorf("invalid build URL: %s", buildURL)
|
||||
}
|
||||
|
||||
buildData, err := e.fetchDroneAPI(apiURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetching Drone API: %w", err)
|
||||
}
|
||||
|
||||
stages := e.parseStagesFromAPI(buildData, buildURL)
|
||||
|
||||
var durationMinutes float64
|
||||
if buildData.Finished > 0 && buildData.Started > 0 {
|
||||
durationMinutes = float64(buildData.Finished-buildData.Started) / 60.0
|
||||
}
|
||||
|
||||
return &PipelineInfo{
|
||||
BuildURL: buildURL,
|
||||
Started: buildData.Started,
|
||||
Finished: buildData.Finished,
|
||||
DurationMinutes: durationMinutes,
|
||||
PipelineStages: stages,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (e *Extractor) buildURLToAPIURL(buildURL string) string {
|
||||
if !strings.Contains(buildURL, "drone.owncloud.com") {
|
||||
return ""
|
||||
}
|
||||
parts := strings.Split(buildURL, "/")
|
||||
if len(parts) < 5 {
|
||||
return ""
|
||||
}
|
||||
repo := parts[3] + "/" + parts[4]
|
||||
buildNum := parts[5]
|
||||
return fmt.Sprintf("https://drone.owncloud.com/api/repos/%s/builds/%s", repo, buildNum)
|
||||
}
|
||||
|
||||
func (e *Extractor) fetchDroneAPI(url string) (*droneBuildResponse, error) {
|
||||
resp, err := e.client.Get(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var buildData droneBuildResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&buildData); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &buildData, nil
|
||||
}
|
||||
|
||||
func (e *Extractor) parseStagesFromAPI(buildData *droneBuildResponse, buildURL string) []PipelineStage {
|
||||
var stages []PipelineStage
|
||||
|
||||
for _, apiStage := range buildData.Stages {
|
||||
normalizedStatus := e.normalizeStatus(apiStage.Status)
|
||||
if normalizedStatus == "success" || normalizedStatus == "skipped" {
|
||||
continue
|
||||
}
|
||||
|
||||
stage := PipelineStage{
|
||||
StageNumber: apiStage.Number,
|
||||
StageName: apiStage.Name,
|
||||
Status: normalizedStatus,
|
||||
Steps: []PipelineStep{},
|
||||
}
|
||||
|
||||
for _, apiStep := range apiStage.Steps {
|
||||
normalizedStepStatus := e.normalizeStatus(apiStep.Status)
|
||||
if normalizedStepStatus == "skipped" {
|
||||
continue
|
||||
}
|
||||
stepURL := fmt.Sprintf("%s/%d/%d", buildURL, apiStage.Number, apiStep.Number)
|
||||
step := PipelineStep{
|
||||
StepNumber: apiStep.Number,
|
||||
StepName: apiStep.Name,
|
||||
Status: normalizedStepStatus,
|
||||
URL: stepURL,
|
||||
}
|
||||
|
||||
if normalizedStepStatus == "failure" {
|
||||
logs, err := e.fetchStepLogs(stepURL, buildURL, apiStage.Number, apiStep.Number)
|
||||
if err == nil {
|
||||
lines := strings.Split(logs, "\n")
|
||||
if len(lines) > 500 && (strings.Contains(logs, "--- Failed scenarios:") || strings.Contains(logs, "Scenario:")) {
|
||||
logs = e.extractBehatFailures(logs)
|
||||
}
|
||||
step.Logs = logs
|
||||
}
|
||||
}
|
||||
|
||||
stage.Steps = append(stage.Steps, step)
|
||||
}
|
||||
|
||||
stages = append(stages, stage)
|
||||
}
|
||||
|
||||
return stages
|
||||
}
|
||||
|
||||
func (e *Extractor) normalizeStatus(status string) string {
|
||||
switch strings.ToLower(status) {
|
||||
case "success", "passed":
|
||||
return "success"
|
||||
case "failure", "failed", "error":
|
||||
return "failure"
|
||||
case "running", "pending", "started":
|
||||
return "running"
|
||||
case "skipped":
|
||||
return "skipped"
|
||||
default:
|
||||
return status
|
||||
}
|
||||
}
|
||||
|
||||
// droneBuildResponse matches the structure returned by Drone API:
|
||||
// GET /api/repos/{owner}/{repo}/builds/{buildNumber}
|
||||
// Verified against: https://drone.owncloud.com/api/repos/owncloud/ocis/builds/51629
|
||||
type droneBuildResponse struct {
|
||||
Started int64 `json:"started"`
|
||||
Finished int64 `json:"finished"`
|
||||
Stages []droneStage `json:"stages"`
|
||||
}
|
||||
|
||||
type droneStage struct {
|
||||
Number int `json:"number"` // Stage number (1, 2, 3, ...)
|
||||
Name string `json:"name"` // Stage name (e.g., "API-wopi", "coding-standard-php8.4")
|
||||
Status string `json:"status"` // "success", "failure", "running", etc.
|
||||
Steps []droneStep `json:"steps"` // Array of steps in this stage
|
||||
}
|
||||
|
||||
type droneStep struct {
|
||||
Number int `json:"number"` // Step number within stage
|
||||
Name string `json:"name"` // Step name (e.g., "clone", "test-acceptance-api")
|
||||
Status string `json:"status"` // "success", "failure", "skipped", etc.
|
||||
}
|
||||
|
||||
type droneLogEntry struct {
|
||||
Pos int `json:"pos"`
|
||||
Out string `json:"out"`
|
||||
Time int64 `json:"time"`
|
||||
}
|
||||
|
||||
type BuildInfo struct {
|
||||
BuildNumber int `json:"build_number"`
|
||||
Status string `json:"status"` // "success", "failure", "error"
|
||||
Started int64 `json:"started"`
|
||||
Finished int64 `json:"finished"`
|
||||
CommitSHA string `json:"commit_sha"`
|
||||
}
|
||||
|
||||
type droneBuildListItem struct {
|
||||
Number int `json:"number"`
|
||||
Status string `json:"status"`
|
||||
Event string `json:"event"` // "pull_request", "push", etc.
|
||||
Source string `json:"source"` // Source branch for PR builds
|
||||
Target string `json:"target"` // Target branch for PR builds
|
||||
After string `json:"after"` // Commit SHA
|
||||
Started int64 `json:"started"`
|
||||
Finished int64 `json:"finished"`
|
||||
}
|
||||
|
||||
func (e *Extractor) fetchStepLogs(stepURL, buildURL string, stageNum, stepNum int) (string, error) {
|
||||
apiLogsURL := e.buildLogsAPIURL(buildURL, stageNum, stepNum)
|
||||
if apiLogsURL != "" {
|
||||
logs, err := e.fetchLogsFromAPI(apiLogsURL)
|
||||
if err == nil {
|
||||
return logs, nil
|
||||
}
|
||||
if err != nil && !strings.Contains(err.Error(), "401") && !strings.Contains(err.Error(), "403") {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
return e.fetchLogsFromHTML(stepURL)
|
||||
}
|
||||
|
||||
func (e *Extractor) buildLogsAPIURL(buildURL string, stageNum, stepNum int) string {
|
||||
if !strings.Contains(buildURL, "drone.owncloud.com") {
|
||||
return ""
|
||||
}
|
||||
parts := strings.Split(buildURL, "/")
|
||||
if len(parts) < 5 {
|
||||
return ""
|
||||
}
|
||||
repo := parts[3] + "/" + parts[4]
|
||||
buildNum := parts[5]
|
||||
return fmt.Sprintf("https://drone.owncloud.com/api/repos/%s/builds/%s/logs/%d/%d", repo, buildNum, stageNum, stepNum)
|
||||
}
|
||||
|
||||
func (e *Extractor) fetchLogsFromAPI(apiURL string) (string, error) {
|
||||
resp, err := e.client.Get(apiURL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var logEntries []droneLogEntry
|
||||
if err := json.NewDecoder(resp.Body).Decode(&logEntries); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var logs strings.Builder
|
||||
for _, entry := range logEntries {
|
||||
logs.WriteString(entry.Out)
|
||||
}
|
||||
|
||||
return logs.String(), nil
|
||||
}
|
||||
|
||||
func (e *Extractor) fetchLogsFromHTML(stepURL string) (string, error) {
|
||||
resp, err := e.client.Get(stepURL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return e.extractLogsFromHTML(string(body))
|
||||
}
|
||||
|
||||
func (e *Extractor) extractLogsFromHTML(htmlContent string) (string, error) {
|
||||
doc, err := html.Parse(strings.NewReader(htmlContent))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var logs strings.Builder
|
||||
var extractText func(*html.Node)
|
||||
extractText = func(n *html.Node) {
|
||||
if n.Type == html.ElementNode {
|
||||
if n.Data == "pre" || n.Data == "code" {
|
||||
text := e.getTextContent(n)
|
||||
if text != "" {
|
||||
logs.WriteString(text)
|
||||
logs.WriteString("\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
extractText(c)
|
||||
}
|
||||
}
|
||||
|
||||
extractText(doc)
|
||||
return strings.TrimSpace(logs.String()), nil
|
||||
}
|
||||
|
||||
func (e *Extractor) getTextContent(n *html.Node) string {
|
||||
var text strings.Builder
|
||||
var collectText func(*html.Node)
|
||||
collectText = func(n *html.Node) {
|
||||
if n.Type == html.TextNode {
|
||||
text.WriteString(n.Data)
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
collectText(c)
|
||||
}
|
||||
}
|
||||
collectText(n)
|
||||
return text.String()
|
||||
}
|
||||
|
||||
func (e *Extractor) extractBehatFailures(logs string) string {
|
||||
var result strings.Builder
|
||||
|
||||
// ANSI red code pattern
|
||||
redCodePattern := regexp.MustCompile(`\x1b\[31m|\033\[31m|\[31m`)
|
||||
ansiStripPattern := regexp.MustCompile(`\x1b\[[0-9;]*m|\033\[[0-9;]*m`)
|
||||
|
||||
// Split by Scenario: or Scenario Outline: (case-insensitive)
|
||||
// Use regex to find all scenario starts while preserving the marker
|
||||
scenarioPattern := regexp.MustCompile(`(?i)(Scenario(?:\s+Outline)?:\s*)`)
|
||||
matches := scenarioPattern.FindAllStringIndex(logs, -1)
|
||||
|
||||
if len(matches) == 0 {
|
||||
// No scenarios found, try to extract just "--- Failed scenarios:" section
|
||||
return e.extractFailedScenariosSection(logs, ansiStripPattern)
|
||||
}
|
||||
|
||||
// Extract each scenario section
|
||||
for i, match := range matches {
|
||||
start := match[0]
|
||||
end := len(logs)
|
||||
if i+1 < len(matches) {
|
||||
end = matches[i+1][0]
|
||||
}
|
||||
|
||||
scenarioSection := logs[start:end]
|
||||
|
||||
// Check if this section contains ANSI red codes
|
||||
if redCodePattern.MatchString(scenarioSection) {
|
||||
// Strip ANSI codes and add to result
|
||||
cleaned := ansiStripPattern.ReplaceAllString(scenarioSection, "")
|
||||
if result.Len() > 0 {
|
||||
result.WriteString("\n\n")
|
||||
}
|
||||
result.WriteString(cleaned)
|
||||
}
|
||||
}
|
||||
|
||||
// Extract "--- Failed scenarios:" section
|
||||
failedScenariosSection := e.extractFailedScenariosSection(logs, ansiStripPattern)
|
||||
if failedScenariosSection != "" {
|
||||
if result.Len() > 0 {
|
||||
result.WriteString("\n\n")
|
||||
}
|
||||
result.WriteString(failedScenariosSection)
|
||||
}
|
||||
|
||||
extracted := strings.TrimSpace(result.String())
|
||||
if extracted == "" {
|
||||
// If extraction failed, return original logs
|
||||
return logs
|
||||
}
|
||||
|
||||
return extracted
|
||||
}
|
||||
|
||||
func (e *Extractor) extractFailedScenariosSection(logs string, ansiStripPattern *regexp.Regexp) string {
|
||||
failedScenariosMarker := "--- Failed scenarios:"
|
||||
if idx := strings.Index(logs, failedScenariosMarker); idx != -1 {
|
||||
failedScenariosSection := logs[idx:]
|
||||
cleaned := ansiStripPattern.ReplaceAllString(failedScenariosSection, "")
|
||||
return cleaned
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// GetBuildsByCommitSHA queries Drone API for all builds matching a specific commit SHA
|
||||
func (e *Extractor) GetBuildsByCommitSHA(baseURL, repo, commitSHA string) ([]BuildInfo, error) {
|
||||
var allBuilds []BuildInfo
|
||||
page := 1
|
||||
maxPages := 200 // Safety limit (25 items per page = 5000 builds max)
|
||||
|
||||
for page <= maxPages {
|
||||
url := fmt.Sprintf("%s/api/repos/%s/builds?page=%d", baseURL, repo, page)
|
||||
resp, err := e.client.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetching build list: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var buildList []droneBuildListItem
|
||||
if err := json.NewDecoder(resp.Body).Decode(&buildList); err != nil {
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("decoding build list: %w", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
// Stop when we get an empty page
|
||||
if len(buildList) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// Filter builds by commit SHA
|
||||
for _, item := range buildList {
|
||||
if item.After == commitSHA {
|
||||
allBuilds = append(allBuilds, BuildInfo{
|
||||
BuildNumber: item.Number,
|
||||
Status: item.Status,
|
||||
Started: item.Started,
|
||||
Finished: item.Finished,
|
||||
CommitSHA: item.After,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
page++
|
||||
}
|
||||
|
||||
// Sort by build number (ascending = chronological)
|
||||
sort.Slice(allBuilds, func(i, j int) bool {
|
||||
return allBuilds[i].BuildNumber < allBuilds[j].BuildNumber
|
||||
})
|
||||
|
||||
return allBuilds, nil
|
||||
}
|
||||
|
||||
// GetBuildsByPRBranch queries Drone API for all builds matching PR event and source branch
|
||||
func (e *Extractor) GetBuildsByPRBranch(baseURL, repo, sourceBranch string) ([]BuildInfo, error) {
|
||||
var allBuilds []BuildInfo
|
||||
page := 1
|
||||
maxPages := 200 // Safety limit (25 items per page = 5000 builds max)
|
||||
|
||||
for page <= maxPages {
|
||||
url := fmt.Sprintf("%s/api/repos/%s/builds?page=%d", baseURL, repo, page)
|
||||
resp, err := e.client.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetching build list: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var buildList []droneBuildListItem
|
||||
if err := json.NewDecoder(resp.Body).Decode(&buildList); err != nil {
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("decoding build list: %w", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
// Stop when we get an empty page
|
||||
if len(buildList) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// Filter builds by event="pull_request" and source branch
|
||||
for _, item := range buildList {
|
||||
if item.Event == "pull_request" && item.Source == sourceBranch {
|
||||
allBuilds = append(allBuilds, BuildInfo{
|
||||
BuildNumber: item.Number,
|
||||
Status: item.Status,
|
||||
Started: item.Started,
|
||||
Finished: item.Finished,
|
||||
CommitSHA: item.After,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
page++
|
||||
}
|
||||
|
||||
// Sort by build number (ascending = chronological)
|
||||
sort.Slice(allBuilds, func(i, j int) bool {
|
||||
return allBuilds[i].BuildNumber < allBuilds[j].BuildNumber
|
||||
})
|
||||
|
||||
return allBuilds, nil
|
||||
}
|
||||
|
||||
// GetBuildsByPushBranch queries Drone API for all builds matching push/cron events to a specific branch (e.g., master).
|
||||
// This is specifically for merged commits to main branches, not PR builds.
|
||||
// It includes both "push" events (actual merges) and "cron" events (scheduled/nightly builds).
|
||||
func (e *Extractor) GetBuildsByPushBranch(baseURL, repo, targetBranch string) ([]BuildInfo, error) {
|
||||
var allBuilds []BuildInfo
|
||||
page := 1
|
||||
maxPages := 200 // Safety limit (25 items per page = 5000 builds max)
|
||||
|
||||
for page <= maxPages {
|
||||
url := fmt.Sprintf("%s/api/repos/%s/builds?page=%d", baseURL, repo, page)
|
||||
resp, err := e.client.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetching build list: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var buildList []droneBuildListItem
|
||||
if err := json.NewDecoder(resp.Body).Decode(&buildList); err != nil {
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("decoding build list: %w", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
// Stop when we get an empty page
|
||||
if len(buildList) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// Filter builds by event="push" or event="cron" and target branch
|
||||
// "push" = actual merges to branch
|
||||
// "cron" = scheduled/nightly builds running on the branch
|
||||
for _, item := range buildList {
|
||||
if (item.Event == "push" || item.Event == "cron") && item.Target == targetBranch {
|
||||
allBuilds = append(allBuilds, BuildInfo{
|
||||
BuildNumber: item.Number,
|
||||
Status: item.Status,
|
||||
Started: item.Started,
|
||||
Finished: item.Finished,
|
||||
CommitSHA: item.After,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
page++
|
||||
}
|
||||
|
||||
// Sort by build number (ascending = chronological)
|
||||
sort.Slice(allBuilds, func(i, j int) bool {
|
||||
return allBuilds[i].BuildNumber < allBuilds[j].BuildNumber
|
||||
})
|
||||
|
||||
return allBuilds, nil
|
||||
}
|
||||
|
||||
// extractBaseURL extracts the base URL (e.g., "https://drone.owncloud.com") from a build URL
|
||||
func extractBaseURL(buildURL string) string {
|
||||
if !strings.Contains(buildURL, "://") {
|
||||
return ""
|
||||
}
|
||||
parts := strings.Split(buildURL, "/")
|
||||
if len(parts) >= 3 {
|
||||
return parts[0] + "//" + parts[2]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// BuildInfoToPipelineSummary converts a BuildInfo to PipelineInfoSummary
|
||||
func BuildInfoToPipelineSummary(build BuildInfo, baseURL, repo string) PipelineInfoSummary {
|
||||
var durationMinutes float64
|
||||
if build.Finished > 0 && build.Started > 0 {
|
||||
durationMinutes = float64(build.Finished-build.Started) / 60.0
|
||||
}
|
||||
|
||||
// URL format: https://drone.owncloud.com/owncloud/ocis/{buildNumber}
|
||||
// repo format: "owncloud/ocis"
|
||||
buildURL := fmt.Sprintf("%s/%s/%d", baseURL, repo, build.BuildNumber)
|
||||
|
||||
return PipelineInfoSummary{
|
||||
BuildURL: buildURL,
|
||||
Started: build.Started,
|
||||
Finished: build.Finished,
|
||||
DurationMinutes: durationMinutes,
|
||||
Status: build.Status,
|
||||
}
|
||||
}
|
||||
|
||||
// BuildInfoToPipelineSummaryWithStages converts a BuildInfo to PipelineInfoSummary with failed stages
|
||||
// by fetching full pipeline details from the build URL
|
||||
func (e *Extractor) BuildInfoToPipelineSummaryWithStages(build BuildInfo, baseURL, repo string) PipelineInfoSummary {
|
||||
summary := BuildInfoToPipelineSummary(build, baseURL, repo)
|
||||
|
||||
// Fetch full pipeline info to extract failed stages
|
||||
pipelineInfo, err := e.Extract(summary.BuildURL)
|
||||
if err == nil && pipelineInfo != nil {
|
||||
var failedStages []FailedStage
|
||||
for _, stage := range pipelineInfo.PipelineStages {
|
||||
if stage.Status == "failure" || stage.Status == "error" {
|
||||
failedStages = append(failedStages, FailedStage{
|
||||
StageNumber: stage.StageNumber,
|
||||
StageName: stage.StageName,
|
||||
Status: stage.Status,
|
||||
})
|
||||
}
|
||||
}
|
||||
summary.FailedStages = failedStages
|
||||
}
|
||||
|
||||
return summary
|
||||
}
|
||||
|
||||
// IsFailedBuild checks if a build status indicates failure
|
||||
func IsFailedBuild(status string) bool {
|
||||
normalized := strings.ToLower(status)
|
||||
return normalized == "failure" || normalized == "failed" || normalized == "error"
|
||||
}
|
||||
|
||||
// GetFailedBuildsByPRBranch returns failed builds for a PR branch with pipeline summaries
|
||||
func (e *Extractor) GetFailedBuildsByPRBranch(baseURL, repo, sourceBranch string) ([]PipelineInfoSummary, error) {
|
||||
builds, err := e.GetBuildsByPRBranch(baseURL, repo, sourceBranch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var failedSummaries []PipelineInfoSummary
|
||||
for _, build := range builds {
|
||||
if IsFailedBuild(build.Status) {
|
||||
summary := e.BuildInfoToPipelineSummaryWithStages(build, baseURL, repo)
|
||||
failedSummaries = append(failedSummaries, summary)
|
||||
// Rate limiting delay for API calls in BuildInfoToPipelineSummaryWithStages
|
||||
}
|
||||
}
|
||||
|
||||
return failedSummaries, nil
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package droneextractor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/owncloud/ocis/v2/tools/ci-reporter/pkg/githubextractor"
|
||||
)
|
||||
|
||||
const requestDelay = 1 * time.Second
|
||||
|
||||
// FetchPRBuilds fetches all Drone builds for a PR by querying:
|
||||
// 1. Builds by PR branch (handles force-pushes)
|
||||
// 2. Builds by individual commit SHAs (fallback for merged/deleted branches)
|
||||
//
|
||||
// Returns a map of commit SHA -> builds
|
||||
func FetchPRBuilds(
|
||||
extractor *Extractor,
|
||||
baseURL string,
|
||||
repo string,
|
||||
prBranch string,
|
||||
commits []githubextractor.PRCommit,
|
||||
logWriter io.Writer,
|
||||
) (map[string][]BuildInfo, error) {
|
||||
buildsByCommit := make(map[string][]BuildInfo)
|
||||
|
||||
// Query builds by PR branch (handles force-pushes)
|
||||
if logWriter != nil {
|
||||
fmt.Fprintf(logWriter, "Fetching Drone builds for PR (by event and branch)...\n")
|
||||
}
|
||||
prBuilds, err := extractor.GetBuildsByPRBranch(baseURL, repo, prBranch)
|
||||
if err != nil {
|
||||
if logWriter != nil {
|
||||
fmt.Fprintf(logWriter, "⚠ Failed to get builds by branch: %v\n", err)
|
||||
}
|
||||
} else {
|
||||
if logWriter != nil {
|
||||
fmt.Fprintf(logWriter, "Found %d builds for PR branch %s\n", len(prBuilds), prBranch)
|
||||
}
|
||||
// Group builds by commit SHA
|
||||
for _, build := range prBuilds {
|
||||
buildsByCommit[build.CommitSHA] = append(buildsByCommit[build.CommitSHA], build)
|
||||
}
|
||||
}
|
||||
|
||||
// Also query by current commit SHAs (for completeness)
|
||||
if logWriter != nil {
|
||||
fmt.Fprintf(logWriter, "Fetching Drone builds for current commits...\n")
|
||||
}
|
||||
for i, commit := range commits {
|
||||
if _, exists := buildsByCommit[commit.SHA]; exists {
|
||||
if logWriter != nil {
|
||||
fmt.Fprintf(logWriter, " [%d/%d] %s: already found via PR query\n", i+1, len(commits), commit.SHA[:7])
|
||||
}
|
||||
continue
|
||||
}
|
||||
if logWriter != nil {
|
||||
fmt.Fprintf(logWriter, " [%d/%d] Fetching builds for %s...\n", i+1, len(commits), commit.SHA[:7])
|
||||
}
|
||||
builds, err := extractor.GetBuildsByCommitSHA(baseURL, repo, commit.SHA)
|
||||
if err != nil {
|
||||
if logWriter != nil {
|
||||
fmt.Fprintf(logWriter, " ⚠ Failed to get builds: %v\n", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if len(builds) > 0 {
|
||||
buildsByCommit[commit.SHA] = append(buildsByCommit[commit.SHA], builds...)
|
||||
if logWriter != nil {
|
||||
fmt.Fprintf(logWriter, " Found %d builds\n", len(builds))
|
||||
}
|
||||
}
|
||||
time.Sleep(requestDelay)
|
||||
}
|
||||
|
||||
return buildsByCommit, nil
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
package droneextractor
|
||||
|
||||
type PipelineInfo struct {
|
||||
BuildURL string `json:"build_url"`
|
||||
Started int64 `json:"started,omitempty"` // Unix timestamp
|
||||
Finished int64 `json:"finished,omitempty"` // Unix timestamp
|
||||
DurationMinutes float64 `json:"duration_minutes,omitempty"` // Calculated: (finished - started) / 60
|
||||
PipelineStages []PipelineStage `json:"pipeline_stages"`
|
||||
}
|
||||
|
||||
type PipelineStage struct {
|
||||
StageNumber int `json:"stage_number"`
|
||||
StageName string `json:"stage_name,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
Steps []PipelineStep `json:"steps"`
|
||||
}
|
||||
|
||||
type PipelineStep struct {
|
||||
StepNumber int `json:"step_number"`
|
||||
StepName string `json:"step_name"`
|
||||
Status string `json:"status"` // "success", "failure", "error", "running", etc.
|
||||
URL string `json:"url,omitempty"`
|
||||
Logs string `json:"logs,omitempty"` // Console logs content
|
||||
}
|
||||
|
||||
type PipelineInfoSummary struct {
|
||||
BuildURL string `json:"build_url"`
|
||||
Started int64 `json:"started,omitempty"`
|
||||
Finished int64 `json:"finished,omitempty"`
|
||||
DurationMinutes float64 `json:"duration_minutes,omitempty"`
|
||||
Status string `json:"status,omitempty"` // Overall build status
|
||||
FailedStages []FailedStage `json:"failed_stages,omitempty"`
|
||||
}
|
||||
|
||||
type FailedStage struct {
|
||||
StageNumber int `json:"stage_number"`
|
||||
StageName string `json:"stage_name"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
147
tools/ci-reporter/pkg/ghaextractor/extractor.go
Normal file
147
tools/ci-reporter/pkg/ghaextractor/extractor.go
Normal file
@@ -0,0 +1,147 @@
|
||||
package ghaextractor
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
apiBase = "https://api.github.com"
|
||||
requestDelay = 1 * time.Second
|
||||
retryDelay = 2 * time.Second
|
||||
maxRetries = 3
|
||||
)
|
||||
|
||||
type Extractor struct {
|
||||
client *http.Client
|
||||
token string
|
||||
}
|
||||
|
||||
func NewExtractor(client *http.Client, token string) *Extractor {
|
||||
return &Extractor{client: client, token: token}
|
||||
}
|
||||
|
||||
type WorkflowRun struct {
|
||||
ID int64 `json:"id"`
|
||||
RunAttempt int `json:"run_attempt"`
|
||||
Status string `json:"status"`
|
||||
Conclusion string `json:"conclusion"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
HTMLURL string `json:"html_url"`
|
||||
HeadBranch string `json:"head_branch"`
|
||||
HeadSHA string `json:"head_sha"`
|
||||
Event string `json:"event"`
|
||||
}
|
||||
|
||||
type workflowRunsPage struct {
|
||||
TotalCount int `json:"total_count"`
|
||||
WorkflowRuns []WorkflowRun `json:"workflow_runs"`
|
||||
}
|
||||
|
||||
type Job struct {
|
||||
ID int64 `json:"id"`
|
||||
RunID int64 `json:"run_id"`
|
||||
RunAttempt int `json:"run_attempt"`
|
||||
Name string `json:"name"`
|
||||
Status string `json:"status"`
|
||||
Conclusion string `json:"conclusion"`
|
||||
StartedAt string `json:"started_at"`
|
||||
CompletedAt string `json:"completed_at"`
|
||||
}
|
||||
|
||||
type jobsPage struct {
|
||||
TotalCount int `json:"total_count"`
|
||||
Jobs []Job `json:"jobs"`
|
||||
}
|
||||
|
||||
// GetWorkflowRuns returns all runs for a workflow file created at or after `since` (RFC3339).
|
||||
func (e *Extractor) GetWorkflowRuns(repo, workflow, since string) ([]WorkflowRun, error) {
|
||||
var all []WorkflowRun
|
||||
page := 1
|
||||
for {
|
||||
url := fmt.Sprintf("%s/repos/%s/actions/workflows/%s/runs?per_page=100&page=%d&status=completed", apiBase, repo, workflow, page)
|
||||
if since != "" {
|
||||
url += "&created=>=" + since
|
||||
}
|
||||
|
||||
var result workflowRunsPage
|
||||
if err := e.get(url, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
all = append(all, result.WorkflowRuns...)
|
||||
if len(result.WorkflowRuns) < 100 {
|
||||
break
|
||||
}
|
||||
page++
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
|
||||
// GetRunJobs returns all jobs for a given run across all attempts (filter=all).
|
||||
func (e *Extractor) GetRunJobs(repo string, runID int64) ([]Job, error) {
|
||||
var all []Job
|
||||
page := 1
|
||||
for {
|
||||
url := fmt.Sprintf("%s/repos/%s/actions/runs/%d/jobs?filter=all&per_page=100&page=%d", apiBase, repo, runID, page)
|
||||
|
||||
var result jobsPage
|
||||
if err := e.get(url, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
all = append(all, result.Jobs...)
|
||||
if len(result.Jobs) < 100 {
|
||||
break
|
||||
}
|
||||
page++
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
|
||||
func (e *Extractor) get(url string, result any) error {
|
||||
resp, err := e.request(url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("API error %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(result); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
time.Sleep(requestDelay)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Extractor) request(url string) (*http.Response, error) {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxRetries; attempt++ {
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/vnd.github.v3+json")
|
||||
if e.token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+e.token)
|
||||
}
|
||||
resp, err := e.client.Do(req)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
if attempt < maxRetries-1 {
|
||||
time.Sleep(retryDelay)
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
19
vendor/github.com/RoaringBitmap/roaring/v2/.drone.yml
generated
vendored
Normal file
19
vendor/github.com/RoaringBitmap/roaring/v2/.drone.yml
generated
vendored
Normal file
@@ -0,0 +1,19 @@
|
||||
kind: pipeline
|
||||
name: default
|
||||
|
||||
workspace:
|
||||
base: /go
|
||||
path: src/github.com/RoaringBitmap/roaring
|
||||
|
||||
steps:
|
||||
- name: test
|
||||
image: golang
|
||||
commands:
|
||||
- go get -t
|
||||
- go test
|
||||
- go build -tags appengine
|
||||
- go test -tags appengine
|
||||
- GOARCH=386 go build
|
||||
- GOARCH=386 go test
|
||||
- GOARCH=arm go build
|
||||
- GOARCH=arm64 go build
|
||||
Reference in New Issue
Block a user