#!/usr/bin/env python3 """ Run ocis acceptance tests locally and in GitHub Actions CI. Usage: BEHAT_SUITES=apiGraph python3 tests/acceptance/run-github.py """ import json import os import sys import subprocess import signal import time import tempfile import shutil from pathlib import Path # --------------------------------------------------------------------------- # NOTE: EMAIL_SMTP_HOST is "email" (container name) in drone, "localhost" here # --------------------------------------------------------------------------- EMAIL_SMTP_HOST = "localhost" EMAIL_SMTP_PORT = "1025" EMAIL_PORT = "8025" EMAIL_SMTP_SENDER = "ownCloud " KEYCLOAK_IMAGE = "quay.io/keycloak/keycloak:26.5.6" POSTGRES_ALPINE_IMAGE = "postgres:alpine3.18" LOCAL_API_TESTS = { "contractAndLock": { "suites": ["apiContract", "apiLocks"], }, "settingsAndNotification": { "suites": ["apiSettings", "apiNotification", "apiCors"], "emailNeeded": True, "extraEnvironment": { "EMAIL_HOST": EMAIL_SMTP_HOST, "EMAIL_PORT": EMAIL_PORT, }, "extraServerEnvironment": { "OCIS_ADD_RUN_SERVICES": "notifications", "NOTIFICATIONS_SMTP_HOST": EMAIL_SMTP_HOST, "NOTIFICATIONS_SMTP_PORT": EMAIL_SMTP_PORT, "NOTIFICATIONS_SMTP_INSECURE": "true", "NOTIFICATIONS_SMTP_SENDER": EMAIL_SMTP_SENDER, "NOTIFICATIONS_DEBUG_ADDR": "0.0.0.0:9174", }, }, "graphUser": { "suites": ["apiGraphUser"], }, "spaces": { "suites": ["apiSpaces"], }, "spacesShares": { "suites": ["apiSpacesShares"], }, "davOperations": { "suites": [ "apiSpacesDavOperation", "apiDownloads", "apiAsyncUpload", "apiDepthInfinity", "apiArchiver", "apiActivities", ], }, "groupAndSearch1": { "suites": ["apiSearch1", "apiGraph", "apiGraphGroup"], }, "search2": { "suites": ["apiSearch2", "apiSearchContent"], "tikaNeeded": True, "extraServerEnvironment": { "FRONTEND_FULL_TEXT_SEARCH_ENABLED": "true", "SEARCH_EXTRACTOR_TYPE": "tika", "SEARCH_EXTRACTOR_TIKA_TIKA_URL": "http://localhost:9998", "SEARCH_EXTRACTOR_CS3SOURCE_INSECURE": "true", }, }, "sharingNg1": { "suites": ["apiSharingNgShares", "apiReshare", "apiSharingNgPermissions"], }, "sharingNgAdditionalShareRole": { "suites": ["apiSharingNgAdditionalShareRole"], }, "sharingNgShareInvitation": { "suites": ["apiSharingNgDriveInvitation", "apiSharingNgItemInvitation"], }, "sharingNgLinkShare": { "suites": [ "apiSharingNgDriveLinkShare", "apiSharingNgItemLinkShare", "apiSharingNgLinkShareManagement", ], }, "antivirus": { "suites": ["apiAntivirus"], "antivirusNeeded": True, "extraServerEnvironment": { "ANTIVIRUS_SCANNER_TYPE": "clamav", "ANTIVIRUS_CLAMAV_SOCKET": "tcp://clamav:3310", "POSTPROCESSING_STEPS": "virusscan", "OCIS_ADD_RUN_SERVICES": "antivirus", "ANTIVIRUS_DEBUG_ADDR": "0.0.0.0:9277", }, }, "ocm": { "suites": ["apiOcm"], "emailNeeded": True, "federationServer": True, "extraEnvironment": { "EMAIL_HOST": EMAIL_SMTP_HOST, "EMAIL_PORT": EMAIL_PORT, }, "extraServerEnvironment": { "OCIS_ADD_RUN_SERVICES": "ocm,notifications", "OCIS_ENABLE_OCM": "true", "OCM_OCM_INVITE_MANAGER_INSECURE": "true", "OCM_OCM_SHARE_PROVIDER_INSECURE": "true", "OCM_OCM_STORAGE_PROVIDER_INSECURE": "true", "OCM_OCM_PROVIDER_AUTHORIZER_PROVIDERS_FILE": "", # set at runtime "NOTIFICATIONS_SMTP_HOST": EMAIL_SMTP_HOST, "NOTIFICATIONS_SMTP_PORT": EMAIL_SMTP_PORT, "NOTIFICATIONS_SMTP_INSECURE": "true", "NOTIFICATIONS_SMTP_SENDER": EMAIL_SMTP_SENDER, "NOTIFICATIONS_DEBUG_ADDR": "0.0.0.0:9174", }, }, "authApp": { "suites": ["apiAuthApp"], "extraServerEnvironment": { "OCIS_ADD_RUN_SERVICES": "auth-app", "PROXY_ENABLE_APP_AUTH": "true", }, }, "wopi": { "suites": ["apiCollaboration"], "collaborationServiceNeeded": True, "extraServerEnvironment": { "GATEWAY_GRPC_ADDR": "0.0.0.0:9142", }, }, "vault": { "suites": ["apiVault"], "keycloakNeeded": True, "tikaNeeded": True, "vaultStorage": True, "extraEnvironment": { "KEYCLOAK": "true", "KC_URL": "https://localhost:8443" }, "extraServerEnvironment": { "OCIS_ENABLE_VAULT_MODE": "true", "FRONTEND_ENABLE_VAULT_MODE": "true", "OCIS_MFA_ENABLED": "true", "SETTINGS_GRPC_ADDR": "localhost:9191", # Keycloak as external OIDC provider — disable built-in IDP "OCIS_EXCLUDE_RUN_SERVICES": "idp", "OCIS_OIDC_ISSUER": "https://localhost:8443/realms/oCIS", "PROXY_OIDC_REWRITE_WELLKNOWN": "true", "PROXY_AUTOPROVISION_ACCOUNTS": "true", "PROXY_ROLE_ASSIGNMENT_DRIVER": "oidc", "PROXY_USER_OIDC_CLAIM": "preferred_username", "PROXY_USER_CS3_CLAIM": "username", "WEB_OIDC_CLIENT_ID": "web", "KEYCLOAK_DOMAIN": "localhost:8443", "OCIS_ADMIN_USER_ID": "", "GRAPH_ASSIGN_DEFAULT_USER_ROLE": "false", "GRAPH_USERNAME_MATCH": "none", "IDM_CREATE_DEMO_USERS": "false", "MICRO_REGISTRY_ADDRESS": "127.0.0.1:9233", "WEB_OIDC_SCOPE": "openid profile email acr", # tika "SEARCH_EXTRACTOR_TYPE": "tika", "SEARCH_EXTRACTOR_TIKA_TIKA_URL": "http://localhost:9998", "SEARCH_EXTRACTOR_CS3SOURCE_INSECURE": "true", } }, "cliCommands": { "suites": ["cliCommands", "apiServiceAvailability"], "antivirusNeeded": True, "emailNeeded": True, "extraEnvironment": { "EMAIL_HOST": EMAIL_SMTP_HOST, "EMAIL_PORT": EMAIL_PORT, }, "extraServerEnvironment": { "NOTIFICATIONS_SMTP_HOST": EMAIL_SMTP_HOST, "NOTIFICATIONS_SMTP_PORT": EMAIL_SMTP_PORT, "NOTIFICATIONS_SMTP_INSECURE": "true", "NOTIFICATIONS_SMTP_SENDER": EMAIL_SMTP_SENDER, "NOTIFICATIONS_DEBUG_ADDR": "0.0.0.0:9174", "ANTIVIRUS_SCANNER_TYPE": "clamav", "ANTIVIRUS_CLAMAV_SOCKET": "tcp://clamav:3310", "ANTIVIRUS_DEBUG_ADDR": "0.0.0.0:9277", "OCIS_ADD_RUN_SERVICES": "antivirus,notifications", }, }, } # reverse lookup: suite → group config _SUITE_TO_CONFIG: dict = {} for _cfg in LOCAL_API_TESTS.values(): for _s in _cfg.get("suites", []): _SUITE_TO_CONFIG[_s] = _cfg # GitHub Actions uses --network host: all wopi services share one network namespace. # Drone gives each service its own container → all can use 9300/9301/9304. # Assign distinct ports here to avoid collisions. _WOPI_PORTS = { "collabora": {"grpc": 9301, "http": 9300, "debug": 9304}, "onlyoffice": {"grpc": 9311, "http": 9310, "debug": 9314}, "fakeoffice": {"grpc": 9321, "http": 9320, "debug": 9324}, } def merged_config(suites: list) -> dict: """Union config for all requested suites.""" merged = { "emailNeeded": False, "antivirusNeeded": False, "tikaNeeded": False, "federationServer": False, "collaborationServiceNeeded": False, "keycloakNeeded": False, "vaultStorage": False, "extraServerEnvironment": {}, "extraEnvironment": {}, } for suite in suites: cfg = _SUITE_TO_CONFIG.get(suite, {}) for flag in ("emailNeeded", "antivirusNeeded", "tikaNeeded", "federationServer", "collaborationServiceNeeded", "keycloakNeeded", "vaultStorage"): if cfg.get(flag): merged[flag] = True merged["extraServerEnvironment"].update(cfg.get("extraServerEnvironment", {})) merged["extraEnvironment"].update(cfg.get("extraEnvironment", {})) return merged def base_server_env(repo_root: Path, ocis_url: str, ocis_config_dir: str) -> dict: """Base ocis server environment matching drone ocisServer() function.""" return { "OCIS_URL": ocis_url, "OCIS_CONFIG_DIR": ocis_config_dir, "STORAGE_USERS_DRIVER": "ocis", "PROXY_ENABLE_BASIC_AUTH": "true", "OCIS_LOG_LEVEL": "error", "IDM_CREATE_DEMO_USERS": "true", "IDM_ADMIN_PASSWORD": "admin", "FRONTEND_SEARCH_MIN_LENGTH": "2", "OCIS_ASYNC_UPLOADS": "true", "OCIS_EVENTS_ENABLE_TLS": "false", "NATS_NATS_HOST": "0.0.0.0", "NATS_NATS_PORT": "9233", "MICRO_REGISTRY_ADDRESS": "127.0.0.1:9233", "OCIS_JWT_SECRET": "some-ocis-jwt-secret", "EVENTHISTORY_STORE": "memory", "OCIS_TRANSLATION_PATH": str(repo_root / "tests/config/translations"), "WEB_UI_CONFIG_FILE": str(repo_root / "tests/config/ci/ocis-config.json"), "THUMBNAILS_TXT_FONTMAP_FILE": str(repo_root / "tests/config/ci/fontsMap.json"), # default tika off (overridden by search2 extraServerEnvironment) "SEARCH_EXTRACTOR_TYPE": "basic", "FRONTEND_FULL_TEXT_SEARCH_ENABLED": "false", # debug addresses "ACTIVITYLOG_DEBUG_ADDR": "0.0.0.0:9197", "APP_PROVIDER_DEBUG_ADDR": "0.0.0.0:9165", "APP_REGISTRY_DEBUG_ADDR": "0.0.0.0:9243", "AUTH_BASIC_DEBUG_ADDR": "0.0.0.0:9147", "AUTH_MACHINE_DEBUG_ADDR": "0.0.0.0:9167", "AUTH_SERVICE_DEBUG_ADDR": "0.0.0.0:9198", "CLIENTLOG_DEBUG_ADDR": "0.0.0.0:9260", "EVENTHISTORY_DEBUG_ADDR": "0.0.0.0:9270", "FRONTEND_DEBUG_ADDR": "0.0.0.0:9141", "GATEWAY_DEBUG_ADDR": "0.0.0.0:9143", "GRAPH_DEBUG_ADDR": "0.0.0.0:9124", "GROUPS_DEBUG_ADDR": "0.0.0.0:9161", "IDM_DEBUG_ADDR": "0.0.0.0:9239", "IDP_DEBUG_ADDR": "0.0.0.0:9134", "INVITATIONS_DEBUG_ADDR": "0.0.0.0:9269", "NATS_DEBUG_ADDR": "0.0.0.0:9234", "OCDAV_DEBUG_ADDR": "0.0.0.0:9163", "OCM_DEBUG_ADDR": "0.0.0.0:9281", "OCS_DEBUG_ADDR": "0.0.0.0:9114", "POSTPROCESSING_DEBUG_ADDR": "0.0.0.0:9255", "PROXY_DEBUG_ADDR": "0.0.0.0:9205", "SEARCH_DEBUG_ADDR": "0.0.0.0:9224", "SETTINGS_DEBUG_ADDR": "0.0.0.0:9194", "SHARING_DEBUG_ADDR": "0.0.0.0:9151", "SSE_DEBUG_ADDR": "0.0.0.0:9139", "STORAGE_PUBLICLINK_DEBUG_ADDR": "0.0.0.0:9179", "STORAGE_SHARES_DEBUG_ADDR": "0.0.0.0:9156", "STORAGE_SYSTEM_DEBUG_ADDR": "0.0.0.0:9217", "STORAGE_USERS_DEBUG_ADDR": "0.0.0.0:9159", "THUMBNAILS_DEBUG_ADDR": "0.0.0.0:9189", "USERLOG_DEBUG_ADDR": "0.0.0.0:9214", "USERS_DEBUG_ADDR": "0.0.0.0:9145", "WEB_DEBUG_ADDR": "0.0.0.0:9104", "WEBDAV_DEBUG_ADDR": "0.0.0.0:9119", "WEBFINGER_DEBUG_ADDR": "0.0.0.0:9279", } def wait_for(condition_fn, timeout: int, label: str, container: str = None) -> None: start = time.time() deadline = start + timeout last_log = start while not condition_fn(): now = time.time() if now > deadline: elapsed = int(now - start) print(f"Timeout waiting for {label} after {elapsed}s", file=sys.stderr) # dump docker diagnostics — use explicit container name if provided cname = container or label for cmd in ( ["docker", "ps", "-a", "--filter", f"name={cname}", "--no-trunc"], ["docker", "logs", "--tail", "50", cname], ): r = subprocess.run(cmd, capture_output=True, text=True) if r.stdout.strip(): print(f"--- {' '.join(cmd)} ---", file=sys.stderr) print(r.stdout, file=sys.stderr) if r.stderr.strip(): print(r.stderr, file=sys.stderr) sys.exit(1) if now - last_log >= 30: print(f" Waiting for {label}... {int(now - start)}s") last_log = now time.sleep(1) def ocis_healthy(ocis_url: str) -> bool: r = subprocess.run( ["curl", "-sk", "-uadmin:admin", f"{ocis_url}/graph/v1.0/users/admin", "-w", "%{http_code}", "-o", "/dev/null"], capture_output=True, text=True, ) return r.stdout.strip() == "200" def ocis_proxy_ready() -> bool: """Check ocis readiness via the proxy debug port (no auth required). In vault mode PROXY_ROLE_ASSIGNMENT_DRIVER=oidc means basic-auth users never receive the admin role, so /graph/v1.0/users/admin returns 403. The proxy debug readyz endpoint (PROXY_DEBUG_ADDR 0.0.0.0:9205) is unauthenticated and becomes available once the proxy is fully started. """ return subprocess.run( ["curl", "-sf", "http://localhost:9205/readyz"], capture_output=True, ).returncode == 0 def mailpit_healthy() -> bool: return subprocess.run( ["curl", "-sf", "http://localhost:8025/api/v1/messages"], capture_output=True, ).returncode == 0 def tika_healthy() -> bool: return subprocess.run( ["curl", "-sf", "http://localhost:9998"], capture_output=True, ).returncode == 0 def _tcp_ready(host: str, port: int) -> bool: """Check if a TCP port is accepting connections.""" import socket try: with socket.create_connection((host, port), timeout=2): return True except (ConnectionRefusedError, OSError): return False def clamav_healthy() -> bool: return _tcp_ready("localhost", 3310) def keycloak_healthy() -> bool: return subprocess.run( ["curl", "-skf", "https://localhost:9000/health/ready"], capture_output=True, ).returncode == 0 def postgres_healthy() -> bool: return subprocess.run( ["docker", "exec", "postgres", "pg_isready", "-U", "keycloak"], capture_output=True, ).returncode == 0 def load_env_file(path: Path) -> dict: """Parse a bash-style env file (export KEY=value) into a dict.""" env = {} for line in path.read_text().splitlines(): line = line.strip() if not line or line.startswith("#") or line.startswith("!"): continue line = line.removeprefix("export ").strip() if "=" in line: k, v = line.split("=", 1) env[k.strip()] = v.strip() return env def run(cmd: list, env: dict = None, check: bool = True): e = {**os.environ, **(env or {})} return subprocess.run(cmd, env=e, check=check) def main() -> int: behat_suites_raw = os.environ.get("BEHAT_SUITES", "").strip() if not behat_suites_raw: print("BEHAT_SUITES is required", file=sys.stderr) return 1 suites = [s.strip() for s in behat_suites_raw.split(",") if s.strip()] acceptance_test_type = os.environ.get("ACCEPTANCE_TEST_TYPE", "api") repo_root = Path(__file__).resolve().parents[2] ocis_bin = repo_root / "ocis/bin/ocis" wrapper_bin = repo_root / "tests/ociswrapper/bin/ociswrapper" ocis_url = "https://localhost:9200" ocis_config_dir = Path.home() / ".ocis/config" ocis_fed_url = "https://localhost:10200" cfg = merged_config(suites) print(f"Suites: {suites}") print(f"Services: email={cfg['emailNeeded']} tika={cfg['tikaNeeded']} " f"antivirus={cfg['antivirusNeeded']} federation={cfg['federationServer']} " f"wopi={cfg['collaborationServiceNeeded']}") if not ocis_bin.exists(): # generate IDP/web assets and build (skipped when pre-built binary is provided) run(["make", "-C", str(repo_root / "services/idp"), "ci-node-generate"]) run(["make", "-C", str(repo_root / "services/web"), "ci-node-generate"]) build_env = {} if subprocess.run(["pkg-config", "--exists", "vips"], capture_output=True).returncode == 0: build_env["ENABLE_VIPS"] = "true" # build debug for code coverage make_target = "build-debug" if os.environ.get("GOCOVERDIR") else "build" run(["make", "-C", str(repo_root / "ocis"), make_target], env=build_env) if make_target == "build-debug": (repo_root / "ocis" / "bin" / "ocis-debug").replace(repo_root / "ocis" / "bin" / "ocis") if not wrapper_bin.exists(): run(["make", "-C", str(repo_root / "tests/ociswrapper"), "build"], env={"GOWORK": "off"}) # php deps run(["composer", "install", "--no-progress"], env={"COMPOSER_NO_INTERACTION": "1", "COMPOSER_NO_AUDIT": "1"}) run(["composer", "bin", "behat", "install", "--no-progress"], env={"COMPOSER_NO_INTERACTION": "1", "COMPOSER_NO_AUDIT": "1"}) # optional services procs = [] if cfg["emailNeeded"]: print("Starting mailpit...") run(["docker", "run", "-d", "--name", "mailpit", "--network", "host", "axllent/mailpit:v1.22.3"]) wait_for(mailpit_healthy, 60, "mailpit") print("mailpit ready.") if cfg["antivirusNeeded"]: print("Starting clamav...") run(["docker", "run", "-d", "--name", "clamav", "--network", "host", "owncloudci/clamavd"]) wait_for(clamav_healthy, 300, "clamav") print("clamav ready.") # override socket: drone uses container DNS "clamav", we use localhost cfg["extraServerEnvironment"]["ANTIVIRUS_CLAMAV_SOCKET"] = "tcp://localhost:3310" if cfg["tikaNeeded"]: print("Starting tika...") run(["docker", "run", "-d", "--name", "tika", "--network", "host", "apache/tika:3.2.2.0-full"]) wait_for(tika_healthy, 120, "tika") print("tika ready.") # OCM federation: rewrite providers.json with localhost URLs if cfg["federationServer"]: providers_src = repo_root / "tests/config/ci/providers.json" providers = json.loads(providers_src.read_text()) for p in providers: # replace container DNS names with localhost p["domain"] = p["domain"].replace("ocis-server:9200", "localhost:9200") p["domain"] = p["domain"].replace("federation-ocis-server:10200", "localhost:10200") for svc in p.get("services", []): ep = svc.get("endpoint", {}) ep["path"] = ep.get("path", "").replace("ocis-server:9200", "localhost:9200") ep["path"] = ep.get("path", "").replace("federation-ocis-server:10200", "localhost:10200") svc["host"] = svc.get("host", "").replace("ocis-server:9200", "localhost:9200") svc["host"] = svc.get("host", "").replace("federation-ocis-server:10200", "localhost:10200") providers_tmp = tempfile.NamedTemporaryFile( mode="w", suffix=".json", prefix="ocm-providers-", delete=False) json.dump(providers, providers_tmp) providers_tmp.close() cfg["extraServerEnvironment"]["OCM_OCM_PROVIDER_AUTHORIZER_PROVIDERS_FILE"] = providers_tmp.name if cfg["keycloakNeeded"]: # Generate TLS certs for keycloak print("Generating keycloak certs...") keycloak_certs_dir = repo_root / "keycloak-certs" keycloak_certs_dir.mkdir(parents=True, exist_ok=True) run(["openssl", "req", "-x509", "-newkey", "rsa:2048", "-keyout", str(keycloak_certs_dir / "keycloakkey.pem"), "-out", str(keycloak_certs_dir / "keycloakcrt.pem"), "-nodes", "-days", "365", "-subj", "/CN=keycloak"]) for f in keycloak_certs_dir.iterdir(): f.chmod(0o777) # Set up postgres (keycloak backend) print("Setting up postgres...") # GitHub runners ship PostgreSQL pre-started on 5432; stop it to avoid conflicts. subprocess.run(["sudo", "systemctl", "stop", "postgresql"], capture_output=True) run(["docker", "run", "-d", "--name", "postgres", "--network", "host", "-e", "POSTGRES_DB=keycloak", "-e", "POSTGRES_USER=keycloak", "-e", "POSTGRES_PASSWORD=keycloak", POSTGRES_ALPINE_IMAGE]) wait_for(postgres_healthy, 30, "postgres", container="postgres") print("postgres ready.") # Patch realm and start keycloak realm_src = repo_root / "tests/config/ci/ocis-mfa-ci-realm.dist.json" realm_patched = Path("/tmp/ocis-realm.json") realm_patched.write_text( realm_src.read_text() .replace("https://ocis:9200", "https://localhost:9200") ) print("Starting keycloak...") run(["docker", "run", "-d", "--name", "keycloak", "--network", "host", "-e", "OCIS_DOMAIN=https://localhost:9200", "-e", "KC_HOSTNAME=localhost", "-e", "KC_PORT=8443", "-e", "KC_DB=postgres", "-e", "KC_DB_URL=jdbc:postgresql://localhost:5432/keycloak", "-e", "KC_DB_USERNAME=keycloak", "-e", "KC_DB_PASSWORD=keycloak", "-e", "KC_FEATURES=impersonation", "-e", "KC_BOOTSTRAP_ADMIN_USERNAME=admin", "-e", "KC_BOOTSTRAP_ADMIN_PASSWORD=admin", "-e", "KC_HTTPS_CERTIFICATE_FILE=/keycloak-certs/keycloakcrt.pem", "-e", "KC_HTTPS_CERTIFICATE_KEY_FILE=/keycloak-certs/keycloakkey.pem", "-v", f"{repo_root}/keycloak-certs:/keycloak-certs:ro", "-v", "/tmp/ocis-realm.json:/opt/keycloak/data/import/ocis-mfa-ci-realm.dist.json:ro", KEYCLOAK_IMAGE, "start-dev", "--proxy-headers", "xforwarded", "--spi-connections-http-client-default-disable-trust-manager=true", "--import-realm", "--health-enabled=true"]) wait_for(keycloak_healthy, 300, "keycloak", container="keycloak") print("keycloak ready.") # init ocis run([str(ocis_bin), "init", "--insecure", "true"]) shutil.copy( repo_root / "tests/config/ci/app-registry.yaml", ocis_config_dir / "app-registry.yaml", ) # generate fontsMap.json with the correct absolute font path for this runner font_path = str(repo_root / "tests/config/ci/NotoSans.ttf") fontmap_tmp = tempfile.NamedTemporaryFile( mode="w", suffix=".json", prefix="fontsMap-", delete=False) json.dump({"defaultFont": font_path}, fontmap_tmp) fontmap_tmp.close() # assemble ocis server env server_env = {**os.environ} server_env.update(base_server_env(repo_root, ocis_url, str(ocis_config_dir))) server_env["THUMBNAILS_TXT_FONTMAP_FILE"] = fontmap_tmp.name server_env.update(cfg["extraServerEnvironment"]) # Patch web UI config: replace container DNS name with localhost so the # browser's 'self' CSP origin (localhost:9200) is not violated. web_cfg_src = Path(server_env["WEB_UI_CONFIG_FILE"]) web_cfg_data = json.loads( web_cfg_src.read_text().replace("ocis-server:9200", "localhost:9200") ) if cfg["vaultStorage"]: web_cfg_data.setdefault("openIdConnect", {})["scope"] = "openid profile email acr" server_env["PROXY_CSP_CONFIG_FILE_LOCATION"] = str(repo_root / "tests/config/ci/csp.yaml") web_cfg_tmp = tempfile.NamedTemporaryFile( mode="w", suffix=".json", prefix="ocis-web-config-", delete=False) json.dump(web_cfg_data, web_cfg_tmp, indent=2) web_cfg_tmp.close() server_env["WEB_UI_CONFIG_FILE"] = web_cfg_tmp.name # start ociswrapper (primary ocis) print("Starting ocis...") wrapper_proc = subprocess.Popen( [str(wrapper_bin), "serve", "--bin", str(ocis_bin), "--url", ocis_url, "--admin-username", "admin", "--admin-password", "admin"], env=server_env, ) procs.append(wrapper_proc) # start federation ocis server (second instance on port 10200) if cfg["federationServer"]: fed_config_dir = Path.home() / ".ocis-federation/config" fed_config_dir.mkdir(parents=True, exist_ok=True) fed_data_dir = Path.home() / ".ocis-federation" fed_env = {**os.environ} fed_env.update(base_server_env(repo_root, ocis_fed_url, str(fed_config_dir))) fed_env.update(cfg["extraServerEnvironment"]) # load federation port mappings from canonical env file (single source of truth) fed_env.update(load_env_file(repo_root / "tests/config/local/.env-federation")) # CI-specific overrides fed_env.update({ "OCIS_URL": ocis_fed_url, "OCIS_BASE_DATA_PATH": str(fed_data_dir), "OCIS_CONFIG_DIR": str(fed_config_dir), "OCIS_RUNTIME_PORT": "10250", "MICRO_REGISTRY_ADDRESS": "127.0.0.1:10233", }) # init federation ocis with separate config run([str(ocis_bin), "init", "--insecure", "true", "--config-path", str(fed_config_dir)]) shutil.copy( repo_root / "tests/config/ci/app-registry.yaml", fed_config_dir / "app-registry.yaml", ) print("Starting federation ocis...") fed_proc = subprocess.Popen( [str(ocis_bin), "server"], env=fed_env, ) procs.append(fed_proc) # --------------------------------------------------------------------------- # Collaboration service helpers — same names/call pattern as drone.star # Only deviation: container hostnames → localhost # --------------------------------------------------------------------------- def fakeOffice(): # BusyBox nc -k has a race between connections: after each response the # while-loop restarts nc, leaving a window where connections are refused. # The collaboration service hits that window at startup → readLoopPeekFailLocked # → healthz never binds → 300s timeout. Use Python's built-in HTTP server # instead; it handles concurrent connections without gaps. run(["docker", "run", "-d", "--name", "fakeoffice", "--network", "host", "-v", f"{repo_root}:/ocis:ro", "python:3-alpine", "python3", "-c", "import http.server, pathlib\n" "body = pathlib.Path('/ocis/tests/config/ci/hosting-discovery.xml').read_bytes()\n" "class H(http.server.BaseHTTPRequestHandler):\n" " def do_GET(self):\n" " self.send_response(200)\n" " self.send_header('Content-Type', 'text/xml')\n" " self.end_headers()\n" " self.wfile.write(body)\n" " def log_message(self, *a): pass\n" "http.server.HTTPServer(('', 8080), H).serve_forever()\n" ]) return [] def collaboraService(): # drone commands copy-pasted verbatim run(["docker", "run", "-d", "--name", "collabora", "--network", "host", "-e", "DONT_GEN_SSL_CERT=set", "-e", f"extra_params=--o:ssl.enable=true --o:ssl.termination=true " f"--o:welcome.enable=false --o:net.frame_ancestors=https://localhost:9200", "--entrypoint", "/bin/sh", "collabora/code:24.04.5.1.1", "-c", "\n".join([ "set -e", "coolconfig generate-proof-key", "bash /start-collabora-online.sh", ])]) return [] def onlyofficeService(): # GitHub runner ships PostgreSQL pre-started on 5432. # OnlyOffice supervisord starts its own PostgreSQL on 5432 internally. # With --network host both compete for the same port → OnlyOffice DB never # starts → docservice stays down → nginx returns 502 forever. # Drone avoids this because each service has its own network namespace. subprocess.run(["sudo", "systemctl", "stop", "postgresql"], capture_output=True) only_office_json = repo_root / "tests/config/ci/only-office.json" run(["docker", "run", "-d", "--name", "onlyoffice", "--network", "host", "-e", "WOPI_ENABLED=true", "-e", "USE_UNAUTHORIZED_STORAGE=true", "-v", f"{only_office_json}:/tmp/only-office.json:ro", "--entrypoint", "/bin/sh", "onlyoffice/documentserver:9.0.0", "-c", "\n".join([ "set -e", "cp /tmp/only-office.json /etc/onlyoffice/documentserver/local.json", "openssl req -x509 -newkey rsa:4096 -keyout onlyoffice.key -out onlyoffice.crt -sha256 -days 365 -batch -nodes", "mkdir -p /var/www/onlyoffice/Data/certs", "cp onlyoffice.key /var/www/onlyoffice/Data/certs/", "cp onlyoffice.crt /var/www/onlyoffice/Data/certs/", "chmod 400 /var/www/onlyoffice/Data/certs/onlyoffice.key", "/app/ds/run-document-server.sh", ])]) return [] def wopiCollaborationService(name, ocis_url=ocis_url): # drone: startOcisService("collaboration", "wopi-{name}", environment) # runs: ocis/bin/ocis-debug collaboration server service_name = "wopi-%s" % name ports = _WOPI_PORTS[name] environment = { **os.environ, "OCIS_URL": ocis_url, "MICRO_REGISTRY": "nats-js-kv", "MICRO_REGISTRY_ADDRESS": "localhost:9233", "COLLABORATION_LOG_LEVEL": "debug", "COLLABORATION_GRPC_ADDR": f"0.0.0.0:{ports['grpc']}", "COLLABORATION_HTTP_ADDR": f"0.0.0.0:{ports['http']}", "COLLABORATION_DEBUG_ADDR": f"0.0.0.0:{ports['debug']}", "COLLABORATION_APP_PROOF_DISABLE": "true", "COLLABORATION_APP_INSECURE": "true", "COLLABORATION_CS3API_DATAGATEWAY_INSECURE": "true", "OCIS_JWT_SECRET": "some-ocis-jwt-secret", "COLLABORATION_WOPI_SECRET": "some-wopi-secret", } if name == "collabora": environment["COLLABORATION_APP_NAME"] = "Collabora" environment["COLLABORATION_APP_PRODUCT"] = "Collabora" environment["COLLABORATION_APP_ADDR"] = "https://localhost:9980" environment["COLLABORATION_APP_ICON"] = "https://localhost:9980/favicon.ico" elif name == "onlyoffice": environment["COLLABORATION_APP_NAME"] = "OnlyOffice" environment["COLLABORATION_APP_PRODUCT"] = "OnlyOffice" environment["COLLABORATION_APP_ADDR"] = "https://localhost:443" environment["COLLABORATION_APP_ICON"] = "https://localhost:443/web-apps/apps/documenteditor/main/resources/img/favicon.ico" elif name == "fakeoffice": environment["COLLABORATION_APP_NAME"] = "FakeOffice" environment["COLLABORATION_APP_PRODUCT"] = "Microsoft" environment["COLLABORATION_APP_ADDR"] = "http://localhost:8080" environment["COLLABORATION_WOPI_SRC"] = f"http://localhost:{ports['http']}" print(f"Starting {service_name}...") return [subprocess.Popen([str(ocis_bin), "collaboration", "server"], env=environment)] def ocisHealthCheck(name, services=[]): # drone: curl healthz + readyz on each service (timeout 300s) for service in services: host, port = service.rsplit(":", 1) for endpoint in ("healthz", "readyz"): wait_for( lambda h="localhost", p=int(port), ep=endpoint: subprocess.run( ["curl", "-sf", f"http://{h}:{p}/{ep}"], capture_output=True ).returncode == 0, 300, f"{service}/{endpoint}", ) print(f"health-check-{name}: all services healthy.") def wopi_discovery_ready(app_url: str) -> bool: """Return True once the WOPI app's /hosting/discovery returns HTTP 200.""" url = app_url.rstrip("/") + "/hosting/discovery" r = subprocess.run( ["curl", "-sfk", url], capture_output=True ) return r.returncode == 0 # drone.star non-k8s collaborationServiceNeeded path (lines 1195-1196, 1140-1141, 1179-1183) if cfg["collaborationServiceNeeded"]: procs += fakeOffice() + collaboraService() + onlyofficeService() # Wait for each app's /hosting/discovery to return 200 before starting its # collaboration service. GetAppURLs in server.go calls discovery synchronously # at startup — non-200 exits the process immediately, healthz never binds. wait_for(lambda: wopi_discovery_ready("http://localhost:8080"), 300, "fakeoffice discovery", container="fakeoffice") wait_for(lambda: wopi_discovery_ready("https://localhost:9980"), 300, "collabora discovery", container="collabora") wait_for(lambda: wopi_discovery_ready("https://localhost:443"), 300, "onlyoffice discovery", container="onlyoffice") procs += wopiCollaborationService("fakeoffice") + \ wopiCollaborationService("collabora") + \ wopiCollaborationService("onlyoffice") ocisHealthCheck("wopi", [ f"localhost:{_WOPI_PORTS['collabora']['debug']}", f"localhost:{_WOPI_PORTS['onlyoffice']['debug']}", f"localhost:{_WOPI_PORTS['fakeoffice']['debug']}", ]) def cleanup(*_): for p in procs: try: p.terminate() except Exception: pass signal.signal(signal.SIGTERM, cleanup) signal.signal(signal.SIGINT, cleanup) try: if cfg["vaultStorage"]: # Basic auth is blocked in vault mode (OIDC role assignment), # use the unauthenticated proxy debug readyz endpoint instead. wait_for(ocis_proxy_ready, 300, "ocis") else: wait_for(lambda: ocis_healthy(ocis_url), 300, "ocis") print("ocis ready.") if cfg["federationServer"]: wait_for(lambda: ocis_healthy(ocis_fed_url), 300, "federation ocis") print("federation ocis ready.") if cfg["vaultStorage"]: vault_data_dir = Path.home() / "ocis/storage/users-vault" vault_data_dir.mkdir(parents=True, exist_ok=True) # Inherit the full server env so the process has PATH, OCIS_JWT_SECRET, # OCIS_CONFIG_DIR, SSL settings, etc. — then override with vault-specific values. vault_env = { **server_env, "STORAGE_USERS_ENABLE_VAULT_MODE": "true", "STORAGE_USERS_SERVICE_NAME": "storage-users-vault", "STORAGE_USERS_GRPC_ADDR": "localhost:19285", "STORAGE_USERS_HTTP_ADDR": "localhost:19286", "STORAGE_USERS_DATA_SERVER_URL": "http://localhost:19286/data", "STORAGE_USERS_DEBUG_ADDR": "localhost:19287", "STORAGE_USERS_OCIS_ROOT": str(vault_data_dir), "STORAGE_USERS_EVENTS_CONSUMER_GROUP": "vault-dcfs", } print("Starting storage-users-vault...") vault_proc = subprocess.Popen( [str(ocis_bin), "storage-users", "server"], env=vault_env, ) procs.append(vault_proc) wait_for( lambda: subprocess.run( ["curl", "-sf", "http://localhost:19287/healthz"], capture_output=True, ).returncode == 0, 120, "storage-users-vault", ) print("vault ready.") # expected failures file if acceptance_test_type == "core-api": filter_tags = "~@skipOnGraph&&~@skipOnOcis-OCIS-Storage" base_failures = repo_root / "tests/acceptance/expected-failures-API-on-OCIS-storage.md" else: filter_tags = "~@skip&&~@skipOnGraph&&~@skipOnOcis-OCIS-Storage" base_failures = repo_root / "tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md" ef_override = os.environ.get("EXPECTED_FAILURES_FILE") if ef_override: p = Path(ef_override) base_failures = p if p.is_absolute() else repo_root / p # merge expected-failures-without-remotephp.md only when not using remote.php # (mirrors drone.star: "" if run_with_remote_php else "cat ...without-remotephp.md >> ...") tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".md", delete=False) tmp.write(base_failures.read_text()) if os.environ.get("WITH_REMOTE_PHP", "false").lower() != "true": without_rphp = repo_root / "tests/acceptance/expected-failures-without-remotephp.md" if without_rphp.exists(): tmp.write("\n") tmp.write(without_rphp.read_text()) tmp.close() # run tests behat_env = { **os.environ, "TEST_SERVER_URL": ocis_url, "TEST_SERVER_FED_URL": ocis_fed_url, "OCIS_WRAPPER_URL": "http://localhost:5200", "BEHAT_SUITES": behat_suites_raw, "ACCEPTANCE_TEST_TYPE": acceptance_test_type, "BEHAT_FILTER_TAGS": filter_tags, "EXPECTED_FAILURES_FILE": tmp.name, "STORAGE_DRIVER": "ocis", "UPLOAD_DELETE_WAIT_TIME": "0", "EMAIL_HOST": "localhost", "EMAIL_PORT": EMAIL_PORT, "COLLABORATION_SERVICE_URL": f"http://localhost:{_WOPI_PORTS['fakeoffice']['http']}", } behat_env.update(cfg["extraEnvironment"]) print(f"Running suites: {behat_suites_raw} (type: {acceptance_test_type})") result = subprocess.run( ["make", "-C", str(repo_root), "test-acceptance-api"], env=behat_env, ) return result.returncode finally: cleanup() if __name__ == "__main__": sys.exit(main())