Release enterprise ci (#613)

* fix aarch64 ci Signed-off-by: xieydd <xieydd@gmail.com> * add arch to pg-slim image Signed-off-by: xieydd <xieydd@gmail.com> * trunk not support pg17 yet Signed-off-by: xieydd <xieydd@gmail.com> * trunk not support pg17 yet Signed-off-by: xieydd <xieydd@gmail.com> * set lib dir Signed-off-by: xieydd <xieydd@gmail.com> * fix env error Signed-off-by: xieydd <xieydd@gmail.com> * fix ARG error Signed-off-by: xieydd <xieydd@gmail.com> * delete arm64 Signed-off-by: xieydd <xieydd@gmail.com> * give postgres tmp permission Signed-off-by: xieydd <xieydd@gmail.com> * delete aarch64 binary release Signed-off-by: xieydd <xieydd@gmail.com> * delete tmp dir chmod Signed-off-by: xieydd <xieydd@gmail.com> * change postgres default datadir to var/lib/postgresql/data/tensorchord Signed-off-by: xieydd <xieydd@gmail.com> * delete slim arm support Signed-off-by: xieydd <xieydd@gmail.com> * fix Signed-off-by: xieydd <xieydd@gmail.com> * fix Signed-off-by: xieydd <xieydd@gmail.com> * debug Signed-off-by: xieydd <xieydd@gmail.com> * debug Signed-off-by: xieydd <xieydd@gmail.com> * debug Signed-off-by: xieydd <xieydd@gmail.com> * debug Signed-off-by: xieydd <xieydd@gmail.com> * debug Signed-off-by: xieydd <xieydd@gmail.com> * debug Signed-off-by: xieydd <xieydd@gmail.com> * debug Signed-off-by: xieydd <xieydd@gmail.com> * add exts suffix avoid cover old image Signed-off-by: xieydd <xieydd@gmail.com> --------- Signed-off-by: xieydd <xieydd@gmail.com>
2025-04-18 21:44:00 +03:00 · 2024-11-20 11:23:23 +08:00 · 2024-11-20 11:23:23 +08:00 · 93e5cf4eb1
commit 93e5cf4eb1
parent cc4776fec0
5 changed files with 55 additions and 35 deletions
--- a/.github/workflows/release_enterprise.yml
+++ b/.github/workflows/release_enterprise.yml
@ -41,8 +41,8 @@ jobs:
  binary:
    strategy:
      matrix:
-        version: [14, 15, 16, 17]
-        arch: ["x86_64", "aarch64"]
+        version: [14, 15, 16]
+        arch: ["x86_64"]
        schema: ["vectors", "extensions", "public"]
    runs-on: ubuntu-22.04
    env:
@ -61,7 +61,11 @@ jobs:
          sudo apt-get update
          sudo apt-get install -y build-essential crossbuild-essential-arm64
          sudo apt-get install -y qemu-user-static
-          echo 'target.aarch64-unknown-linux-gnu.linker = "aarch64-linux-gnu-gcc"' | tee ~/.cargo/config.toml
+          touch ~/.cargo/config.toml
+          echo 'target.aarch64-unknown-linux-gnu.linker = "aarch64-linux-gnu-gcc"' >> ~/.cargo/config.toml
+          echo 'target.aarch64-unknown-linux-gnu.runner = ["qemu-aarch64-static", "-L", "/usr/aarch64-linux-gnu"]' >> ~/.cargo/config.toml
+          rustup target add x86_64-unknown-linux-gnu
+          rustup target add aarch64-unknown-linux-gnu
      - name: Set up Sccache
        uses: mozilla-actions/sccache-action@v0.0.4
      - name: Set up Cache
@ -110,8 +114,8 @@ jobs:
    needs: ["binary", "semver"]
    strategy:
      matrix:
-        version: [14, 15, 16, 17]
-        platform: ["amd64", "arm64"]
+        version: [14, 15, 16]
+        platform: ["amd64"]
        schema: ["vectors", "extensions", "public"]
    runs-on: ubuntu-latest
    steps:
@ -144,9 +148,9 @@ jobs:
    runs-on: ubuntu-latest
    strategy:
      matrix:
-        version: [14, 15, 16, 17]
+        version: [14, 15, 16] # https://github.com/tembo-io/trunk/issues/797, 17 not support yet
        schema: ["vectors", "extensions", "public"]
-        platform: ["amd64", "arm64"]
+        platform: ["amd64"] # Only support x86-64 architecture, aarch64 not support yet
    steps:
      - name: Checkout
        uses: actions/checkout@v4
@ -155,7 +159,7 @@ jobs:
        uses: actions/github-script@v7
        with:
          script: |
-            core.setOutput('tags', "modelzai/pgvecto-rs:${{ matrix.version }}-v${{ github.event.inputs.version }}-${{ matrix.schema }}");
+            core.setOutput('tags', "modelzai/pgvecto-rs:${{ matrix.version }}-v${{ github.event.inputs.version }}-${{ matrix.schema }}-exts");
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
@ -165,6 +169,15 @@ jobs:
        with:
          username: ${{ secrets.DOCKERIO_MODELZ_USERNAME }}
          password: ${{ secrets.DOCKERIO_MODELZ_TOKEN }}
+      - name: Set Lib Dir
+        run: |
+          if [ "${{ matrix.platform }}" == "amd64" ]; then
+            echo "LIB_DIR=/usr/lib/x86_64-linux-gnu" >> $GITHUB_ENV
+          elif [ "${{ matrix.platform }}" == "arm64" ]; then
+            echo "LIB_DIR=/usr/lib/aarch64-linux-gnu" >> $GITHUB_ENV
+          else
+            echo "LIB_DIR=/usr/lib/unknown" >> $GITHUB_ENV
+          fi
      - name: Push postgres with pgvecto.rs enterprise to Docker Registry
        uses: docker/build-push-action@v4
        with:
@ -177,4 +190,5 @@ jobs:
            FROM_TAG=pg${{ matrix.version }}-v${{ github.event.inputs.version }}
            SCHEMA=${{ matrix.schema }}
            TARGETARCH=${{ matrix.platform }}
+            LIB_DIR=${{ env.LIB_DIR }}
          tags: ${{ steps.variables.outputs.tags }}
--- a/.github/workflows/release_pg_slim.yml
+++ b/.github/workflows/release_pg_slim.yml
@ -8,7 +8,7 @@ jobs:
    strategy:
      matrix:
        version: [14, 15, 16, 17]
-        platform: ["amd64", "arm64"]
+        platform: ["amd64"]
    runs-on: ubuntu-latest
    env:
      PG_MAJOR: ${{ matrix.version }}
@ -33,4 +33,4 @@ jobs:
          file: ./docker/pg-slim/Dockerfile
          build-args: |
            PG_MAJOR=${{ matrix.version }}
-          tags: modelzai/pg-slim:${{ matrix.version }}
+          tags: modelzai/pg-slim:${{ matrix.version }}-${{ matrix.platform }}
--- a/.github/workflows/trunk-install-test.yml
+++ b/.github/workflows/trunk-install-test.yml
@ -15,9 +15,9 @@ jobs:
      - ubuntu-latest
    strategy:
      matrix:
-        version: [14, 15, 16, 17]
+        version: [14, 15, 16]
        schema: ["vectors", "extensions", "public"]
-        platform: ["amd64", "arm64"]
+        platform: ["amd64"]
    container:
      image: modelzai/pgvecto-rs:${{ matrix.version }}-v${{ github.event.inputs.version }}-${{ matrix.schema }}
      options: --user root
@ -28,6 +28,7 @@ jobs:
      PGUSER: "postgres"
      PGPASSWORD: "postgres"
      POSTGRES_PASSWORD: "password"
+      PGDATA: "/var/lib/postgresql/data2"

    steps:
      - name: Install all extensions in registry
--- a/docker/pg-cnpg/Dockerfile
+++ b/docker/pg-cnpg/Dockerfile
@ -10,8 +10,10 @@ ARG TRUNK_VER=0.12.25
 ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL sparse 
 RUN cargo install --version $TRUNK_VER pg-trunk

-FROM modelzai/pg-slim:${PG_MAJOR}
+FROM modelzai/pg-slim:${PG_MAJOR}-${TARGETARCH}
 ARG PG_MAJOR
+ARG LIB_DIR
+ARG ALTDIR=/var/lib/postgresql/data/tensorchord

 USER root

@ -40,8 +42,8 @@ RUN set -xe; \
  apt-get clean; \
 	rm -rf /var/lib/apt/lists/*;

-RUN chown -R postgres:postgres /usr/lib/postgresql/${PG_MAJOR} && \
-    chmod -R 0700 /usr/lib/postgresql/${PG_MAJOR}
+RUN chown -R postgres:postgres ${ALTDIR}/${PG_MAJOR} && \
+    chmod -R 0700 ${ALTDIR}/${PG_MAJOR}
 RUN chown postgres /usr/share/postgresql/${PG_MAJOR}/extension

 RUN apt-get update && apt-get install -y \
@ -91,12 +93,12 @@ RUN apt-get update && apt-get install -y \
    libbson-dev \
    libgsl-dev \
    && rm -rf /var/lib/apt/lists/*
-RUN ln -s /usr/lib/jvm/java-11-openjdk-amd64/lib/server/libjvm.so /usr/lib/x86_64-linux-gnu/libjvm.so
+RUN ln -s /usr/lib/jvm/java-11-openjdk-amd64/lib/server/libjvm.so ${LIB_DIR}/libjvm.so
 RUN wget https://download.oracle.com/otn_software/linux/instantclient/1920000/instantclient-basiclite-linux.x64-19.20.0.0.0dbru.zip && \
    unzip instantclient-basiclite-linux.x64-19.20.0.0.0dbru.zip && \
-    cp instantclient_19_20/libclntsh.so.19.1 /usr/lib/x86_64-linux-gnu/ && \
-    cp instantclient_19_20/libnnz19.so /usr/lib/x86_64-linux-gnu/ && \
-    cp instantclient_19_20/libclntshcore.so.19.1 /usr/lib/x86_64-linux-gnu/ && \
+    cp instantclient_19_20/libclntsh.so.19.1 ${LIB_DIR}/ && \
+    cp instantclient_19_20/libnnz19.so ${LIB_DIR}/ && \
+    cp instantclient_19_20/libclntshcore.so.19.1 ${LIB_DIR}/ && \
    rm -rf instantclient_19_20 && \
    rm instantclient-basiclite-linux.x64-19.20.0.0.0dbru.zip

@ -108,12 +110,12 @@ RUN  wget http://www.xunsearch.com/scws/down/scws-1.2.3.tar.bz2 && \
     make install && \
     cd .. && \
     rm -rf scws-1.2.3.tar.bz2 scws-1.2.3 && \
-     ln -s /usr/local/lib/libscws.so /usr/lib/x86_64-linux-gnu/libscws.so
+     ln -s /usr/local/lib/libscws.so ${LIB_DIR}/libscws.so

 # Install duckdb libs
 RUN wget https://github.com/duckdb/duckdb/releases/download/v0.8.1/libduckdb-linux-amd64.zip && \
    unzip libduckdb-linux-amd64.zip && \
-    cp libduckdb.so /usr/lib/x86_64-linux-gnu/ && \
+    cp libduckdb.so ${LIB_DIR}/ && \
    rm -rf libduckdb-linux-amd64.zip libduckdb.so

 # Install pg_stat_statements
@ -125,13 +127,6 @@ RUN trunk install auto_explain
 # Install plpython3u
 RUN trunk install plpython3u

-# cache pg_stat_statements and auto_explain and pg_stat_kcache to temp directory
-RUN set -eux; \
-	  mkdir /tmp/pg_pkglibdir; \
-	  mkdir /tmp/pg_sharedir; \
-      cp -r $(pg_config --pkglibdir)/* /tmp/pg_pkglibdir; \
-      cp -r $(pg_config --sharedir)/* /tmp/pg_sharedir
-
 # Clone and build AWS SDK for C++
 RUN git clone https://github.com/aws/aws-sdk-cpp.git && \
    cd aws-sdk-cpp && \
@ -175,7 +170,16 @@ COPY trunk-install.sh /usr/local/bin/

 # Change the uid of postgres to 26
 RUN usermod -u 26 postgres
-RUN chown -R postgres:postgres /usr/lib/postgresql/${PG_MAJOR}
-RUN cp /usr/share/postgresql/${PG_MAJOR}/extension/* /usr/lib/postgresql/${PG_MAJOR}/share/extension/
+RUN chown -R postgres:postgres ${ALTDIR}
+RUN cp /usr/share/postgresql/${PG_MAJOR}/extension/* ${ALTDIR}/extension/
+RUN cp /usr/lib/postgresql/${PG_MAJOR}/lib/* ${ALTDIR}/${PG_MAJOR}/lib/
+
+RUN set -eux; \
+    mkdir /tmp/pg_pkglibdir; \
+    mkdir /tmp/pg_sharedir; \
+    cp -r $(pg_config --pkglibdir)/* /tmp/pg_pkglibdir; \
+    cp -r $(pg_config --sharedir)/* /tmp/pg_sharedir
+
+RUN chown -R postgres:postgres /tmp
 USER 26
 ENV PATH $PATH:/usr/lib/postgresql/${PG_MAJOR}/bin
--- a/docker/pg-slim/Dockerfile
+++ b/docker/pg-slim/Dockerfile
@ -5,9 +5,9 @@ ARG PG_MAJOR
 # 15.8-1.pgdg120+1 16.4-1.pgdg120+2 17.0-1.pgdg120+1
 # ARG PG_VERSION 
 ARG DEBIAN_FRONTEND=noninteractive
+ARG ALTDIR=/var/lib/postgresql/data/tensorchord
 ENV TZ=Etc/UTC
 ENV PGDATA /var/lib/postgresql/data
-ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin

 # Get latest package updates
 RUN set -eux; \
@ -79,7 +79,8 @@ WORKDIR postgres
 ENV CFLAGS "-g -O2 -fstack-protector-strong -Wformat -Werror=format-security -fno-omit-frame-pointer"
 ENV LDFLAGS "-Wl,-z,relro -Wl,-z,now"
 RUN ./configure --prefix=/usr/lib/postgresql/${PG_MAJOR} \
-        --libdir=/usr/lib/postgresql/$PG_MAJOR/lib \
+        --datarootdir=${ALTDIR} \
+        --libdir=${ALTDIR}/${PG_MAJOR}/lib \
        --with-perl \
        --with-python \
        --with-tcl \
@ -116,9 +117,9 @@ COPY --from=tianon/gosu /gosu /usr/local/bin/

 # make the sample config easier to munge (and "correct by default")
 RUN set -eux; \
-    echo "unix_socket_directories = '/var/run/postgresql'" >> /usr/lib/postgresql/${PG_MAJOR}/share/postgresql.conf.sample; \
-	sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/lib/postgresql/${PG_MAJOR}/share/postgresql.conf.sample; \
-	grep -F "listen_addresses = '*'" /usr/lib/postgresql/${PG_MAJOR}/share/postgresql.conf.sample
+    echo "unix_socket_directories = '/var/run/postgresql'" >> ${ALTDIR}/postgresql.conf.sample; \
+	sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" ${ALTDIR}/postgresql.conf.sample; \
+	grep -F "listen_addresses = '*'" ${ALTDIR}/postgresql.conf.sample

 RUN install --verbose --directory --owner postgres --group postgres --mode 3777 /var/run/postgresql