From 834214bcac6a13ffca3267ce95faccf0a255bb7a Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Fri, 18 Nov 2022 19:28:16 +0100 Subject: [PATCH] Do not embed the WASM-compiled policies in the binary --- .github/workflows/ci.yaml | 32 ++++++--------- Dockerfile | 9 ++-- README.md | 11 ++++- crates/cli/src/commands/debug.rs | 17 +++----- crates/cli/src/commands/server.rs | 17 +++----- crates/config/src/sections/http.rs | 36 +++++++++++----- crates/config/src/sections/policy.rs | 18 ++++++-- crates/handlers/src/lib.rs | 20 ++++++++- crates/policy/Cargo.toml | 5 ++- crates/policy/src/lib.rs | 41 +++++++++---------- docs/config.schema.json | 10 ++--- .../policy/policies => policies}/.gitignore | 0 {crates/policy/policies => policies}/Makefile | 0 .../authorization_grant.rego | 0 .../authorization_grant_test.rego | 0 .../client_registration.rego | 0 .../client_registration_test.rego | 0 .../policies => policies}/register.rego | 0 .../policies => policies}/register_test.rego | 0 .../policies => policies}/util/coveralls.rego | 0 20 files changed, 124 insertions(+), 92 deletions(-) rename {crates/policy/policies => policies}/.gitignore (100%) rename {crates/policy/policies => policies}/Makefile (100%) rename {crates/policy/policies => policies}/authorization_grant.rego (100%) rename {crates/policy/policies => policies}/authorization_grant_test.rego (100%) rename {crates/policy/policies => policies}/client_registration.rego (100%) rename {crates/policy/policies => policies}/client_registration_test.rego (100%) rename {crates/policy/policies => policies}/register.rego (100%) rename {crates/policy/policies => policies}/register_test.rego (100%) rename {crates/policy/policies => policies}/util/coveralls.rego (100%) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b23709d8..f637114b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -32,24 +32,21 @@ jobs: version: 0.45.0 - name: Lint policies - run: | - cd crates/policy/policies - make lint + working-directory: ./policies + run: make lint - name: Run OPA tests - run: | - cd crates/policy/policies - make test + working-directory: ./policies + run: make test - name: Run OPA tests with coverage - run: | - cd crates/policy/policies - make coverage + working-directory: ./policies + run: make coverage - name: Upload to codecov.io uses: codecov/codecov-action@v3 with: - files: crates/policy/policies/coverage.json + files: policies/coverage.json flags: policies frontend-lint: @@ -198,9 +195,8 @@ jobs: version: 0.45.0 - name: Compile OPA policies - run: | - cd crates/policy/policies - make + working-directory: ./policies + run: make - name: Setup Rust cache uses: Swatinem/rust-cache@v2 @@ -259,9 +255,8 @@ jobs: version: 0.45.0 - name: Compile OPA policies - run: | - cd crates/policy/policies - make + working-directory: ./policies + run: make - name: Setup Rust cache uses: Swatinem/rust-cache@v2 @@ -325,9 +320,8 @@ jobs: version: 0.45.0 - name: Compile OPA policies - run: | - cd crates/policy/policies - make + working-directory: ./policies + run: make - name: Setup Rust cache uses: Swatinem/rust-cache@v2 diff --git a/Dockerfile b/Dockerfile index 544215b1..6f378684 100644 --- a/Dockerfile +++ b/Dockerfile @@ -74,8 +74,8 @@ ARG OPA_VERSION # Download Open Policy Agent ADD --chmod=755 https://github.com/open-policy-agent/opa/releases/download/v${OPA_VERSION}/opa_${BUILDOS}_${BUILDARCH}_static /usr/local/bin/opa -WORKDIR /app/crates/policy/policies -COPY ./crates/policy/policies/ /app/crates/policy/policies +WORKDIR /app/policies +COPY ./policies /app/policies RUN make -B # Change the timestamp of built files for better caching @@ -148,7 +148,6 @@ RUN cargo chef cook \ COPY ./Cargo.toml ./Cargo.lock /app/ COPY ./crates /app/crates COPY --from=static-files /app/crates/static-files/public /app/crates/static-files/public -COPY --from=policy /app/crates/policy/policies/policy.wasm /app/crates/policy/policies/policy.wasm ENV SQLX_OFFLINE=true RUN cargo auditable zigbuild \ --locked \ @@ -168,6 +167,8 @@ FROM --platform=${TARGETPLATFORM} gcr.io/distroless/cc-debian${DEBIAN_VERSION}:d COPY --from=builder /usr/local/bin/mas-cli /usr/local/bin/mas-cli COPY --from=frontend /usr/local/share/mas-cli /usr/local/share/mas-cli +COPY --from=policy /app/policies/policy.wasm /usr/local/share/mas-cli/policy.wasm + WORKDIR / ENTRYPOINT ["/usr/local/bin/mas-cli"] @@ -178,5 +179,7 @@ FROM --platform=${TARGETPLATFORM} gcr.io/distroless/cc-debian${DEBIAN_VERSION}:n COPY --from=builder /usr/local/bin/mas-cli /usr/local/bin/mas-cli COPY --from=frontend /usr/local/share/mas-cli /usr/local/share/mas-cli +COPY --from=policy /app/policies/policy.wasm /usr/local/share/mas-cli/policy.wasm + WORKDIR / ENTRYPOINT ["/usr/local/bin/mas-cli"] diff --git a/README.md b/README.md index e26b52da..6ecf5de4 100644 --- a/README.md +++ b/README.md @@ -11,16 +11,23 @@ See the [Documentation](https://matrix-org.github.io/matrix-authentication-servi - [Install Node.js and npm](https://nodejs.org/) - [Install Open Policy Agent](https://www.openpolicyagent.org/docs/latest/#1-download-opa) - Clone this repository -- Generate the frontend: +- Generate the static-files: ```sh cd crates/static-files npm ci npm run build cd ../.. ``` +- Build the frontend + ```sh + cd frontend + npm ci + npm run build + cd ../.. + ``` - Build the Open Policy Agent policies ```sh - cd crates/policy/policies + cd policies make # OR, if you don't have `opa` installed and want to build through the OPA docker image make DOCKER=1 diff --git a/crates/cli/src/commands/debug.rs b/crates/cli/src/commands/debug.rs index 5ba65631..00007d86 100644 --- a/crates/cli/src/commands/debug.rs +++ b/crates/cli/src/commands/debug.rs @@ -18,7 +18,7 @@ use hyper::{Response, Uri}; use mas_config::PolicyConfig; use mas_http::HttpServiceExt; use mas_policy::PolicyFactory; -use tokio::io::{AsyncRead, AsyncWriteExt}; +use tokio::io::AsyncWriteExt; use tower::{Service, ServiceExt}; use tracing::info; @@ -121,19 +121,12 @@ impl Options { SC::Policy => { let config: PolicyConfig = root.load_config()?; info!("Loading and compiling the policy module"); - let mut policy: Box = - if let Some(path) = &config.wasm_module { - Box::new( - tokio::fs::File::open(path) - .await - .context("failed to open OPA WASM policy file")?, - ) - } else { - Box::new(mas_policy::default_wasm_policy()) - }; + let policy_file = tokio::fs::File::open(&config.wasm_module) + .await + .context("failed to open OPA WASM policy file")?; let policy_factory = PolicyFactory::load( - &mut policy, + policy_file, config.data.clone().unwrap_or_default(), config.register_entrypoint.clone(), config.client_registration_entrypoint.clone(), diff --git a/crates/cli/src/commands/server.rs b/crates/cli/src/commands/server.rs index 6be058c1..7725071c 100644 --- a/crates/cli/src/commands/server.rs +++ b/crates/cli/src/commands/server.rs @@ -28,7 +28,7 @@ use mas_router::UrlBuilder; use mas_storage::MIGRATOR; use mas_tasks::TaskQueue; use mas_templates::Templates; -use tokio::{io::AsyncRead, signal::unix::SignalKind}; +use tokio::signal::unix::SignalKind; use tracing::{error, info, log::warn}; #[derive(Parser, Debug, Default)] @@ -144,19 +144,12 @@ impl Options { // Load and compile the WASM policies (and fallback to the default embedded one) info!("Loading and compiling the policy module"); - let mut policy: Box = - if let Some(path) = &config.policy.wasm_module { - Box::new( - tokio::fs::File::open(path) - .await - .context("failed to open OPA WASM policy file")?, - ) - } else { - Box::new(mas_policy::default_wasm_policy()) - }; + let policy_file = tokio::fs::File::open(&config.policy.wasm_module) + .await + .context("failed to open OPA WASM policy file")?; let policy_factory = PolicyFactory::load( - &mut policy, + policy_file, config.policy.data.clone().unwrap_or_default(), config.policy.register_entrypoint.clone(), config.policy.client_registration_entrypoint.clone(), diff --git a/crates/config/src/sections/http.rs b/crates/config/src/sections/http.rs index 3988401c..12aa25ab 100644 --- a/crates/config/src/sections/http.rs +++ b/crates/config/src/sections/http.rs @@ -43,6 +43,26 @@ fn http_address_example_4() -> &'static str { "0.0.0.0:8080" } +#[cfg(not(feature = "docker"))] +fn http_listener_spa_manifest_default() -> Utf8PathBuf { + "./frontend/dist/manifest.json".into() +} + +#[cfg(not(feature = "docker"))] +fn http_listener_spa_assets_default() -> Utf8PathBuf { + "./frontend/dist/".into() +} + +#[cfg(feature = "docker")] +fn http_listener_spa_manifest_default() -> Utf8PathBuf { + "/usr/local/share/mas-cli/frontend-manifest.json".into() +} + +#[cfg(feature = "docker")] +fn http_listener_spa_assets_default() -> Utf8PathBuf { + "/usr/local/share/mas-cli/frontend-assets/".into() +} + /// Kind of socket #[derive(Debug, Serialize, Deserialize, JsonSchema, Clone, Copy)] #[serde(rename_all = "lowercase")] @@ -267,11 +287,13 @@ pub enum Resource { /// Mount the single page app Spa { - /// Path to the vite manifest + /// Path to the vite mamanifest.jsonnifest + #[serde(default = "http_listener_spa_manifest_default")] #[schemars(with = "String")] manifest: Utf8PathBuf, /// Path to the assets to server + #[serde(default = "http_listener_spa_assets_default")] #[schemars(with = "String")] assets: Utf8PathBuf, }, @@ -325,17 +347,9 @@ impl Default for HttpConfig { Resource::Compat, Resource::GraphQL { playground: true }, Resource::Static { web_root: None }, - #[cfg(not(feature = "docker"))] Resource::Spa { - manifest: "./frontend/dist/manifest.json".into(), - assets: "./frontend/dist/".into(), - }, - #[cfg(feature = "docker")] - Resource::Spa { - // This is where the frontend files are mounted in the docker image by - // default - manifest: "/usr/local/share/mas-cli/frontend-manifest.json".into(), - assets: "/usr/local/share/mas-cli/frontend-assets/".into(), + manifest: http_listener_spa_manifest_default(), + assets: http_listener_spa_assets_default(), }, ], tls: None, diff --git a/crates/config/src/sections/policy.rs b/crates/config/src/sections/policy.rs index 446c5b80..4eccf059 100644 --- a/crates/config/src/sections/policy.rs +++ b/crates/config/src/sections/policy.rs @@ -21,6 +21,16 @@ use serde_with::serde_as; use super::ConfigurationSection; +#[cfg(not(feature = "docker"))] +fn default_policy_path() -> Utf8PathBuf { + "./policies/policy.wasm".into() +} + +#[cfg(feature = "docker")] +fn default_policy_path() -> Utf8PathBuf { + "/usr/local/share/mas-cli/policy.wasm".into() +} + fn default_client_registration_endpoint() -> String { "client_registration/violation".to_owned() } @@ -38,9 +48,9 @@ fn default_authorization_grant_endpoint() -> String { #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct PolicyConfig { /// Path to the WASM module - #[serde(default)] - #[schemars(with = "Option")] - pub wasm_module: Option, + #[serde(default = "default_policy_path")] + #[schemars(with = "String")] + pub wasm_module: Utf8PathBuf, /// Entrypoint to use when evaluating client registrations #[serde(default = "default_client_registration_endpoint")] @@ -62,7 +72,7 @@ pub struct PolicyConfig { impl Default for PolicyConfig { fn default() -> Self { Self { - wasm_module: None, + wasm_module: default_policy_path(), client_registration_entrypoint: default_client_registration_endpoint(), register_entrypoint: default_register_endpoint(), authorization_grant_entrypoint: default_authorization_grant_endpoint(), diff --git a/crates/handlers/src/lib.rs b/crates/handlers/src/lib.rs index 770cc0a4..4e7c0b60 100644 --- a/crates/handlers/src/lib.rs +++ b/crates/handlers/src/lib.rs @@ -376,7 +376,25 @@ async fn test_state(pool: PgPool) -> Result { let mailer = Mailer::new(&templates, &transport, &mailbox, &mailbox); let homeserver = MatrixHomeserver::new("example.com".to_owned()); - let policy_factory = PolicyFactory::load_default(serde_json::json!({})).await?; + + #[allow(clippy::disallowed_types)] + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("..") + .join("policies") + .join("policy.wasm"); + + let file = tokio::fs::File::open(path).await?; + + let policy_factory = PolicyFactory::load( + file, + serde_json::json!({}), + "register/violation".to_owned(), + "client_registration/violation".to_owned(), + "authorization_grant/violation".to_owned(), + ) + .await?; + let policy_factory = Arc::new(policy_factory); let graphql_schema = graphql_schema(&pool); diff --git a/crates/policy/Cargo.toml b/crates/policy/Cargo.toml index 73a5c1a2..db492262 100644 --- a/crates/policy/Cargo.toml +++ b/crates/policy/Cargo.toml @@ -11,12 +11,15 @@ opa-wasm = { git = "https://github.com/matrix-org/rust-opa-wasm.git" } serde = { version = "1.0.147", features = ["derive"] } serde_json = "1.0.88" thiserror = "1.0.37" -tokio = { version = "1.21.2", features = ["io-util", "rt"] } +tokio = { version = "1.21.2", features = ["io-util"] } tracing = "0.1.37" wasmtime = { version = "2.0.2", default-features = false, features = ["async", "cranelift"] } mas-data-model = { path = "../data-model" } oauth2-types = { path = "../oauth2-types" } +[dev-dependencies] +tokio = { version = "1.21.2", features = ["fs", "rt", "macros"] } + [features] cache = ["wasmtime/cache"] diff --git a/crates/policy/src/lib.rs b/crates/policy/src/lib.rs index 7e6e7a97..2569ad83 100644 --- a/crates/policy/src/lib.rs +++ b/crates/policy/src/lib.rs @@ -17,8 +17,6 @@ #![warn(clippy::pedantic)] #![allow(clippy::missing_errors_doc)] -use std::io::Cursor; - use anyhow::bail; use mas_data_model::{AuthorizationGrant, StorageBackend, User}; use oauth2_types::registration::VerifiedClientMetadata; @@ -28,13 +26,6 @@ use thiserror::Error; use tokio::io::{AsyncRead, AsyncReadExt}; use wasmtime::{Config, Engine, Module, Store}; -const DEFAULT_POLICY: &[u8] = include_bytes!("../policies/policy.wasm"); - -#[must_use] -pub fn default_wasm_policy() -> impl AsyncRead + std::marker::Unpin { - Cursor::new(DEFAULT_POLICY) -} - #[derive(Debug, Error)] pub enum LoadError { #[error("failed to read module")] @@ -115,17 +106,6 @@ impl PolicyFactory { Ok(factory) } - pub async fn load_default(data: serde_json::Value) -> Result { - Self::load( - default_wasm_policy(), - data, - "register/violation".to_owned(), - "client_registration/violation".to_owned(), - "authorization_grant/violation".to_owned(), - ) - .await - } - #[tracing::instrument(skip(self), err)] pub async fn instantiate(&self) -> Result { let mut store = Store::new(&self.engine, ()); @@ -257,10 +237,27 @@ mod tests { #[tokio::test] async fn test_register() { - let factory = PolicyFactory::load_default(serde_json::json!({ + let data = serde_json::json!({ "allowed_domains": ["element.io", "*.element.io"], "banned_domains": ["staging.element.io"], - })) + }); + + #[allow(clippy::disallowed_types)] + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("..") + .join("policies") + .join("policy.wasm"); + + let file = tokio::fs::File::open(path).await.unwrap(); + + let factory = PolicyFactory::load( + file, + data, + "register/violation".to_owned(), + "client_registration/violation".to_owned(), + "authorization_grant/violation".to_owned(), + ) .await .unwrap(); diff --git a/docs/config.schema.json b/docs/config.schema.json index 91e4120c..0772f8b4 100644 --- a/docs/config.schema.json +++ b/docs/config.schema.json @@ -135,7 +135,7 @@ "client_registration_entrypoint": "client_registration/violation", "data": null, "register_entrypoint": "register/violation", - "wasm_module": null + "wasm_module": "./policies/policy.wasm" }, "allOf": [ { @@ -1234,7 +1234,7 @@ }, "wasm_module": { "description": "Path to the WASM module", - "default": null, + "default": "./policies/policy.wasm", "type": "string" } } @@ -1430,17 +1430,17 @@ "description": "Mount the single page app", "type": "object", "required": [ - "assets", - "manifest", "name" ], "properties": { "assets": { "description": "Path to the assets to server", + "default": "./frontend/dist/", "type": "string" }, "manifest": { - "description": "Path to the vite manifest", + "description": "Path to the vite mamanifest.jsonnifest", + "default": "./frontend/dist/manifest.json", "type": "string" }, "name": { diff --git a/crates/policy/policies/.gitignore b/policies/.gitignore similarity index 100% rename from crates/policy/policies/.gitignore rename to policies/.gitignore diff --git a/crates/policy/policies/Makefile b/policies/Makefile similarity index 100% rename from crates/policy/policies/Makefile rename to policies/Makefile diff --git a/crates/policy/policies/authorization_grant.rego b/policies/authorization_grant.rego similarity index 100% rename from crates/policy/policies/authorization_grant.rego rename to policies/authorization_grant.rego diff --git a/crates/policy/policies/authorization_grant_test.rego b/policies/authorization_grant_test.rego similarity index 100% rename from crates/policy/policies/authorization_grant_test.rego rename to policies/authorization_grant_test.rego diff --git a/crates/policy/policies/client_registration.rego b/policies/client_registration.rego similarity index 100% rename from crates/policy/policies/client_registration.rego rename to policies/client_registration.rego diff --git a/crates/policy/policies/client_registration_test.rego b/policies/client_registration_test.rego similarity index 100% rename from crates/policy/policies/client_registration_test.rego rename to policies/client_registration_test.rego diff --git a/crates/policy/policies/register.rego b/policies/register.rego similarity index 100% rename from crates/policy/policies/register.rego rename to policies/register.rego diff --git a/crates/policy/policies/register_test.rego b/policies/register_test.rego similarity index 100% rename from crates/policy/policies/register_test.rego rename to policies/register_test.rego diff --git a/crates/policy/policies/util/coveralls.rego b/policies/util/coveralls.rego similarity index 100% rename from crates/policy/policies/util/coveralls.rego rename to policies/util/coveralls.rego