diff --git a/.github/workflows/build_and_push.yml b/.github/workflows/build_and_push.yml index 7423e720d..880134505 100644 --- a/.github/workflows/build_and_push.yml +++ b/.github/workflows/build_and_push.yml @@ -14,6 +14,14 @@ on: default: false required: true type: boolean + runtimeBaseImage: + description: 'SQLBot runtime base image (optional tag or digest override)' + required: false + type: string + builderBaseImage: + description: 'SQLBot build base image paired with the runtime base (optional tag or digest)' + required: false + type: string architecture: description: 'Architecture' required: true @@ -70,18 +78,23 @@ jobs: else DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME}" fi - echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} --memory-swap -1 \ - --build-arg DOCKER_IMAGE_TAG=${{ github.event.inputs.dockerImageTag }} --build-arg BUILD_AT=$(TZ=Asia/Shanghai date +'%Y-%m-%dT%H:%M') --build-arg GITHUB_COMMIT=`git rev-parse --short HEAD` --no-cache \ - ${DOCKER_IMAGE_TAGS} . + printf '%s\n' "buildx_args=--platform ${DOCKER_PLATFORMS} --memory-swap -1 \ + --build-arg DOCKER_IMAGE_TAG=${{ github.event.inputs.dockerImageTag }} \ + --build-arg BUILD_AT=$(TZ=Asia/Shanghai date +'%Y-%m-%dT%H:%M') \ + --build-arg GITHUB_COMMIT=$(git rev-parse --short HEAD) --no-cache \ + ${DOCKER_IMAGE_TAGS} ." >> "$GITHUB_OUTPUT" - name: Login to Aliyun Registry - uses: docker/login-action@v2 + uses: docker/login-action@v4 with: registry: ${{ secrets.ALIYUN_REGISTRY_HOST }} username: ${{ secrets.ALIYUN_REGISTRY_USERNAME }} password: ${{ secrets.ALIYUN_REGISTRY_PASSWORD }} - name: Docker Buildx (build-and-push) + env: + SQLBOT_RUNTIME_IMAGE: ${{ inputs.runtimeBaseImage || format('{0}/dataease/sqlbot-python-pg:latest', secrets.ALIYUN_REGISTRY_HOST) }} + SQLBOT_BUILD_IMAGE: ${{ inputs.builderBaseImage || format('{0}/dataease/sqlbot-build-base:latest', secrets.ALIYUN_REGISTRY_HOST) }} run: | - docker buildx build --provenance=false --sbom=false --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} + docker buildx build --build-arg SQLBOT_RUNTIME_IMAGE --build-arg SQLBOT_BUILD_IMAGE --provenance=false --sbom=false --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} build-and-push-to-dockerhub: if: ${{ contains(github.event.inputs.registry, 'dockerhub') }} @@ -119,14 +132,19 @@ jobs: else DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME}" fi - echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} --memory-swap -1 \ - --build-arg DOCKER_IMAGE_TAG=${{ github.event.inputs.dockerImageTag }} --build-arg BUILD_AT=$(TZ=Asia/Shanghai date +'%Y-%m-%dT%H:%M') --build-arg GITHUB_COMMIT=`git rev-parse --short HEAD` --no-cache \ - ${DOCKER_IMAGE_TAGS} . + printf '%s\n' "buildx_args=--platform ${DOCKER_PLATFORMS} --memory-swap -1 \ + --build-arg DOCKER_IMAGE_TAG=${{ github.event.inputs.dockerImageTag }} \ + --build-arg BUILD_AT=$(TZ=Asia/Shanghai date +'%Y-%m-%dT%H:%M') \ + --build-arg GITHUB_COMMIT=$(git rev-parse --short HEAD) --no-cache \ + ${DOCKER_IMAGE_TAGS} ." >> "$GITHUB_OUTPUT" - name: Login to Docker Hub - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Docker Buildx (build-and-push) + env: + SQLBOT_RUNTIME_IMAGE: ${{ inputs.runtimeBaseImage || 'docker.io/dataease/sqlbot-python-pg:latest' }} + SQLBOT_BUILD_IMAGE: ${{ inputs.builderBaseImage || 'docker.io/dataease/sqlbot-build-base:latest' }} run: | - docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} + docker buildx build --build-arg SQLBOT_RUNTIME_IMAGE --build-arg SQLBOT_BUILD_IMAGE --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} diff --git a/.github/workflows/build_base_and_push.yml b/.github/workflows/build_base_and_push.yml index fac179384..273ff69da 100644 --- a/.github/workflows/build_base_and_push.yml +++ b/.github/workflows/build_base_and_push.yml @@ -6,11 +6,11 @@ on: workflow_dispatch: inputs: dockerImageTag: - description: 'Image Tag' + description: 'Base image version tag' default: 'v0.9.0' required: true dockerImageTagWithLatest: - description: '是否发布latest tag(正式发版时选择,测试版本切勿选择)' + description: 'Also publish the latest tag (official releases only)' default: false required: true type: boolean @@ -44,32 +44,41 @@ jobs: ref: ${{ github.ref_name }} - name: Prepare id: prepare + env: + DOCKER_IMAGE: ${{ secrets.ALIYUN_REGISTRY_HOST }}/dataease/sqlbot-python-pg + TAG_NAME: ${{ inputs.dockerImageTag }} + TAG_NAME_WITH_LATEST: ${{ inputs.dockerImageTagWithLatest }} run: | - DOCKER_IMAGE=${{ secrets.ALIYUN_REGISTRY_HOST }}/dataease/sqlbot-python-pg - DOCKER_PLATFORMS=${{ github.event.inputs.architecture }} - TAG_NAME=${{ github.event.inputs.dockerImageTag }} - TAG_NAME_WITH_LATEST=${{ github.event.inputs.dockerImageTagWithLatest }} - if [[ ${TAG_NAME_WITH_LATEST} == 'true' ]]; then - DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:${TAG_NAME%%.*} --tag ${DOCKER_IMAGE}:latest" - else - DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME}" - fi - echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} --memory-swap -1 \ - --build-arg DOCKER_IMAGE_TAG=${{ github.event.inputs.dockerImageTag }} --build-arg BUILD_AT=$(TZ=Asia/Shanghai date +'%Y-%m-%dT%H:%M') --build-arg GITHUB_COMMIT=`git rev-parse --short HEAD` --no-cache \ - ${DOCKER_IMAGE_TAGS} . + image_tags() { + local image="$1" + local tags="$image:$TAG_NAME" + if [[ "$TAG_NAME_WITH_LATEST" == 'true' ]]; then + tags="$tags,$image:latest" + fi + printf '%s' "$tags" + } + printf '%s=%s\n' \ + runtime_tags "$(image_tags "$DOCKER_IMAGE")" \ + build_tags "$(image_tags "${DOCKER_IMAGE%/*}/sqlbot-build-base")" >> "$GITHUB_OUTPUT" - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Login to Aliyun Registry - uses: docker/login-action@v2 + uses: docker/login-action@v4 with: registry: ${{ secrets.ALIYUN_REGISTRY_HOST }} username: ${{ secrets.ALIYUN_REGISTRY_USERNAME }} password: ${{ secrets.ALIYUN_REGISTRY_PASSWORD }} - name: Docker Buildx (build-and-push) + env: + SQLBOT_BASE_TAGS: ${{ steps.prepare.outputs.runtime_tags }} + SQLBOT_BUILD_TAGS: ${{ steps.prepare.outputs.build_tags }} + SQLBOT_BASE_PLATFORMS: ${{ inputs.architecture }} run: | - docker buildx build -f Dockerfile-base --output "type=image,push=true,oci-mediatypes=false" --provenance=false --sbom=false ${{ steps.prepare.outputs.buildx_args }} + docker buildx bake -f docker-bake.hcl --pull --no-cache --provenance=false --sbom=false \ + --set 'base.output=type=image,push=true,oci-mediatypes=false' \ + --set 'build-base.output=type=image,push=true,oci-mediatypes=false' base build-base build-and-push-to-dockerhub: if: ${{ contains(github.event.inputs.registry, 'dockerhub') }} @@ -81,28 +90,37 @@ jobs: ref: ${{ github.ref_name }} - name: Prepare id: prepare + env: + DOCKER_IMAGE: dataease/sqlbot-python-pg + TAG_NAME: ${{ inputs.dockerImageTag }} + TAG_NAME_WITH_LATEST: ${{ inputs.dockerImageTagWithLatest }} run: | - DOCKER_IMAGE=dataease/sqlbot-python-pg - DOCKER_PLATFORMS=${{ github.event.inputs.architecture }} - TAG_NAME=${{ github.event.inputs.dockerImageTag }} - TAG_NAME_WITH_LATEST=${{ github.event.inputs.dockerImageTagWithLatest }} - if [[ ${TAG_NAME_WITH_LATEST} == 'true' ]]; then - DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:${TAG_NAME%%.*} --tag ${DOCKER_IMAGE}:latest" - else - DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME}" - fi - echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} --memory-swap -1 \ - --build-arg DOCKER_IMAGE_TAG=${{ github.event.inputs.dockerImageTag }} --build-arg BUILD_AT=$(TZ=Asia/Shanghai date +'%Y-%m-%dT%H:%M') --build-arg GITHUB_COMMIT=`git rev-parse --short HEAD` --no-cache \ - ${DOCKER_IMAGE_TAGS} . + image_tags() { + local image="$1" + local tags="$image:$TAG_NAME" + if [[ "$TAG_NAME_WITH_LATEST" == 'true' ]]; then + tags="$tags,$image:latest" + fi + printf '%s' "$tags" + } + printf '%s=%s\n' \ + runtime_tags "$(image_tags "$DOCKER_IMAGE")" \ + build_tags "$(image_tags "${DOCKER_IMAGE%/*}/sqlbot-build-base")" >> "$GITHUB_OUTPUT" - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Login to Docker Hub - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Docker Buildx (build-and-push) + env: + SQLBOT_BASE_TAGS: ${{ steps.prepare.outputs.runtime_tags }} + SQLBOT_BUILD_TAGS: ${{ steps.prepare.outputs.build_tags }} + SQLBOT_BASE_PLATFORMS: ${{ inputs.architecture }} run: | - docker buildx build -f Dockerfile-base --output "type=image,push=true,oci-mediatypes=false" --provenance=false --sbom=false ${{ steps.prepare.outputs.buildx_args }} + docker buildx bake -f docker-bake.hcl --pull --no-cache --provenance=false --sbom=false \ + --set 'base.output=type=image,push=true,oci-mediatypes=false' \ + --set 'build-base.output=type=image,push=true,oci-mediatypes=false' base build-base diff --git a/Dockerfile b/Dockerfile index 52f84cc5b..edc16c112 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,72 +1,82 @@ -# Build sqlbot -FROM ghcr.io/1panel-dev/maxkb-vector-model:v1.0.1 AS vector-model -FROM --platform=${BUILDPLATFORM} registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-base:latest AS sqlbot-ui-builder -ENV SQLBOT_HOME=/opt/sqlbot -ENV APP_HOME=${SQLBOT_HOME}/app -ENV UI_HOME=${SQLBOT_HOME}/frontend -ENV DEBIAN_FRONTEND=noninteractive - -RUN mkdir -p ${APP_HOME} ${UI_HOME} - -COPY frontend /tmp/frontend -RUN cd /tmp/frontend && npm install && npm run build && mv dist ${UI_HOME}/dist - - -FROM registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-base:latest AS sqlbot-builder -# Set build environment variables -ENV PYTHONUNBUFFERED=1 -ENV SQLBOT_HOME=/opt/sqlbot -ENV APP_HOME=${SQLBOT_HOME}/app -ENV UI_HOME=${SQLBOT_HOME}/frontend -ENV PYTHONPATH=${SQLBOT_HOME}/app -ENV PATH="${APP_HOME}/.venv/bin:$PATH" -ENV UV_COMPILE_BYTECODE=1 -ENV UV_LINK_MODE=copy -ENV DEBIAN_FRONTEND=noninteractive +# syntax=docker/dockerfile:1 +ARG SQLBOT_RUNTIME_IMAGE=registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-python-pg:latest +ARG SQLBOT_BUILD_IMAGE=registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-build-base:latest +ARG NODE_IMAGE=node:22-bookworm-slim -# Create necessary directories -RUN mkdir -p ${APP_HOME} ${UI_HOME} - -WORKDIR ${APP_HOME} - -COPY --from=sqlbot-ui-builder ${UI_HOME} ${UI_HOME} -# Install dependencies -RUN test -f "./uv.lock" && \ - --mount=type=cache,target=/root/.cache/uv \ - --mount=type=bind,source=backend/uv.lock,target=uv.lock \ - --mount=type=bind,source=backend/pyproject.toml,target=pyproject.toml \ - uv sync --frozen --no-install-project || echo "uv.lock file not found, skipping intermediate-layers" +FROM ghcr.io/1panel-dev/maxkb-vector-model:v1.0.1 AS vector-model -COPY ./backend ${APP_HOME} +FROM --platform=$BUILDPLATFORM ${NODE_IMAGE} AS sqlbot-ui-builder +WORKDIR /tmp/frontend +COPY frontend/ ./ +RUN npm install && npm run build + +FROM ${SQLBOT_RUNTIME_IMAGE} AS runtime-base +# Rebuild Dockerfile-base first, then point this build at the rebuilt image. +RUN if ! node -e 'if (process.versions.node.split(".")[0] !== "22") process.exit(1)' \ + || [ -e /opt/dmdbms/bin/libssl.so ] \ + || ! dpkg --compare-versions "$(dpkg-query -W -f='${Version}' libharfbuzz0b)" ge '6.0.0+dfsg-3+sqlbot1' \ + || dpkg-query -W libtiff6 >/dev/null 2>&1; then \ + echo 'Rebuild Dockerfile-base and set SQLBOT_RUNTIME_IMAGE to the new image.' >&2; \ + exit 1; \ + fi + +FROM ${SQLBOT_BUILD_IMAGE} AS build-base + +FROM build-base AS sqlbot-builder +ENV PYTHONUNBUFFERED=1 \ + PYTHONPATH=/opt/sqlbot/app \ + UV_COMPILE_BYTECODE=1 \ + UV_LINK_MODE=copy \ + UV_PYTHON_DOWNLOADS=never +WORKDIR /opt/sqlbot/app +COPY backend/ ./ + +# Resolve the security constraints together with the application dependencies. +# Elasticsearch 7.17.13 permits urllib3 2.x on Python 3.11. The source-built +# PostgreSQL and SQL Server drivers use the same patched OS libraries as runtime. +RUN python - <<'PY' +from pathlib import Path + +project = Path('pyproject.toml') +contents = project.read_text() +for old, new in ( + ('psycopg[binary]<4.0.0,>=3.1.13', 'psycopg[c]<4.0.0,>=3.1.13'), + ('psycopg2-binary (>=2.9.12,<3.0.0)', 'psycopg2 (>=2.9.12,<3.0.0)'), + ('elasticsearch[requests] (>=7.10,<8.0)', 'elasticsearch[requests] (>=7.17.13,<8.0)'), + ('dmpython==2.5.22', 'dmpython==2.6.1'), +): + assert contents.count(old) == 1, old + contents = contents.replace(old, new) +contents = contents.replace( + '[tool.uv]\n', + '[tool.uv]\nconstraint-dependencies = ["idna>=3.7", "setuptools>=78.1.1", "urllib3>=2.2.2,<3", "pymssql==2.4.6"]\n', + 1, +) +project.write_text(contents) +PY -# Final sync to ensure all dependencies are installed RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --extra cpu --default-index https://pypi.org/simple - -# Build g2-ssr -FROM registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-base:latest AS ssr-builder + uv sync --no-dev --extra cpu --python /usr/local/bin/python3.11 \ + --default-index https://pypi.org/simple \ + --no-binary-package psycopg2 --no-binary-package pymssql \ + && uv pip check --python .venv/bin/python \ + && .venv/bin/python -c 'import psycopg, psycopg2, pymssql; assert psycopg.pq.__impl__ == "c"' +FROM build-base AS ssr-builder WORKDIR /app - -# Install build dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential python3 pkg-config \ - libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev \ - libpixman-1-dev libfreetype6-dev \ - && rm -rf /var/lib/apt/lists/* - -# configure npm -RUN npm config set fund false \ - && npm config set audit false \ - && npm config set progress false - -COPY g2-ssr/app.js g2-ssr/package.json /app/ -COPY g2-ssr/charts/* /app/charts/ -RUN npm install - -# Runtime stage -FROM registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-python-pg:latest - +COPY g2-ssr/app.js g2-ssr/package.json ./ +COPY g2-ssr/charts/ ./charts/ +# g2-ssr uses canvas. Drop the unused node-canvas fork, which targets old V8. +# Build the Node 22-compatible canvas release against distro libraries. +RUN npm pkg delete dependencies.node-canvas dependencies.nan \ + && npm pkg set 'overrides.canvas=^3.2.3' \ + && npm_config_build_from_source=true npm install --no-fund --no-audit --no-progress + +FROM runtime-base +COPY --from=build-base /usr/local/lib/libsybdb.so.5.* /usr/local/lib/ +COPY --from=build-base /etc/freetds/ /etc/freetds/ +COPY --from=build-base /usr/local/share/doc/freetds/COPYING_LIB.txt /usr/local/share/doc/freetds/COPYING_LIB.txt +RUN ldconfig RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ echo "Asia/Shanghai" > /etc/timezone @@ -75,6 +85,8 @@ ENV PYTHONUNBUFFERED=1 ENV SQLBOT_HOME=/opt/sqlbot ENV PYTHONPATH=${SQLBOT_HOME}/app ENV PATH="${SQLBOT_HOME}/app/.venv/bin:$PATH" +# The DM SSL symlinks need the wheel's hashed crypto library at load time. +ENV LD_LIBRARY_PATH="${SQLBOT_HOME}/app/.venv/lib/python3.11/site-packages/dmpython.libs:$LD_LIBRARY_PATH" ENV POSTGRES_DB=sqlbot ENV POSTGRES_USER=root @@ -83,14 +95,26 @@ ENV POSTGRES_PASSWORD=Password123@pg # Copy necessary files from builder COPY start.sh /opt/sqlbot/app/start.sh COPY g2-ssr/*.ttf /usr/share/fonts/truetype/liberation/ -COPY --from=sqlbot-builder ${SQLBOT_HOME} ${SQLBOT_HOME} -COPY --from=ssr-builder /app /opt/sqlbot/g2-ssr +COPY --from=sqlbot-builder \ + --exclude=.venv/lib/python3.11/site-packages/torch/test \ + /opt/sqlbot/app/ /opt/sqlbot/app/ +COPY --from=sqlbot-ui-builder /tmp/frontend/dist/ /opt/sqlbot/frontend/dist/ +COPY --from=ssr-builder /app/ /opt/sqlbot/g2-ssr/ COPY g2-ssr/supervisord.conf /etc/supervisor/conf.d/g2-ssr.conf -COPY --from=vector-model /opt/maxkb/app/model /opt/sqlbot/models +# SQLBot runs this embedding model with PyTorch; omit its duplicate ONNX export. +COPY --from=vector-model --exclude=embedding/shibing624_text2vec-base-chinese/onnx /opt/maxkb/app/model /opt/sqlbot/models WORKDIR ${SQLBOT_HOME}/app -RUN mkdir -p /opt/sqlbot/images /opt/sqlbot/g2-ssr/logs +RUN set -eu; \ + mkdir -p /opt/sqlbot/images /opt/sqlbot/g2-ssr/logs; \ + dmssl="$SQLBOT_HOME/app/.venv/lib/python3.11/site-packages/dmssl"; \ + for library in libssl.so libcrypto.so; do \ + test -f "$dmssl/$library"; \ + ln -s "$dmssl/$library" "$DM_HOME/bin/$library"; \ + done; \ + python -c 'import ctypes, os, ssl; import dmPython, psycopg, psycopg2, pymssql; assert psycopg.pq.__impl__ == "c"; ctypes.CDLL(os.environ["DM_HOME"] + "/bin/libssl.so")'; \ + node -e "require('/opt/sqlbot/g2-ssr/node_modules/@antv/g2-ssr')" EXPOSE 3000 8000 8001 5432 diff --git a/Dockerfile-base b/Dockerfile-base index 7a778bd61..de083a077 100644 --- a/Dockerfile-base +++ b/Dockerfile-base @@ -1,59 +1,116 @@ -FROM python:3.11-slim-bookworm AS python-builder -FROM registry.cn-qingdao.aliyuncs.com/dataease/postgres:17.6 - -# python environment -COPY --from=python-builder /usr/local /usr/local - -RUN python --version && pip --version - -# Install uv tool -COPY --from=ghcr.io/astral-sh/uv:0.7.8 /uv /uvx /bin/ - -RUN apt-get update && apt-get install -y --no-install-recommends \ - wait-for-it \ - build-essential \ - curl \ - gnupg \ - gcc \ - g++ \ - libcairo2-dev \ - libpango1.0-dev \ - libjpeg-dev \ - libgif-dev \ - librsvg2-dev \ - supervisor \ - && curl -fsSL https://deb.nodesource.com/setup_18.x | bash - \ - && curl -qL https://www.npmjs.com/install.sh | sh - \ - && apt-get install -y nodejs \ - && curl -L --connect-timeout 60 -m 1800 https://fit2cloud-support.oss-cn-beijing.aliyuncs.com/xpack-license/get-validator-linux | sh \ +# syntax=docker/dockerfile:1 +ARG PYTHON_IMAGE=python:3.11-slim-bookworm +ARG NODE_IMAGE=node:22-bookworm-slim + +ARG PGVECTOR_IMAGE=pgvector/pgvector:pg17-bookworm@sha256:7d0a41464797aeb9862dfdc0bbf1526b82cfd324cdd8036df19e3545f4a9ccb8 + +FROM ${PGVECTOR_IMAGE} AS pg-base +# Refresh inherited Debian packages and keep the bundled pgvector extension. +RUN apt-get -o APT::Update::Error-Mode=any update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y --no-remove dist-upgrade \ && rm -rf /var/lib/apt/lists/* \ - && chmod g+xr /usr/bin/ld.so \ - && chmod g+x /usr/local/bin/python* + && test "$PG_MAJOR" = 17 \ + && test "$PGDATA" = /var/lib/postgresql/data \ + && test -f "$(pg_config --sharedir)/extension/vector.control" \ + && test -f "$(pg_config --pkglibdir)/vector.so" + +FROM ${PYTHON_IMAGE} AS python-builder +ARG PIP_VERSION=26.3 +# Refresh the installed pip and the wheel used by ensurepip to seed new venvs. +RUN set -eu; \ + wheels="$(mktemp -d)"; \ + python -m pip --isolated download --no-cache-dir --only-binary=:all: --no-deps \ + --index-url https://pypi.org/simple --dest "$wheels" "pip==$PIP_VERSION"; \ + python -m pip --isolated install --no-index --no-cache-dir "$wheels"/pip-*.whl; \ + ensurepip_dir="$(python -c 'from pathlib import Path; import ensurepip; print(Path(ensurepip.__file__).parent)')"; \ + rm "$ensurepip_dir"/_bundled/pip-*.whl; \ + mv "$wheels"/pip-*.whl "$ensurepip_dir/_bundled/"; \ + sed -i "s/^_PIP_VERSION = .*/_PIP_VERSION = \"$PIP_VERSION\"/" "$ensurepip_dir/__init__.py"; \ + test "$(python -m ensurepip --version)" = "pip $PIP_VERSION"; \ + rmdir "$wheels" + +FROM ${NODE_IMAGE} AS node-runtime + +FROM pg-base AS graphics-builder +# Install the trust store before enabling HTTPS in this disposable stage. +RUN apt-get -o Acquire::Retries=3 -o APT::Update::Error-Mode=any update \ + && DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::Retries=3 install -y --no-install-recommends \ + build-essential ca-certificates curl dpkg-dev + +# Build against the signed Bookworm source index. +RUN sed -i 's|http://deb.debian.org/|https://deb.debian.org/|g' /etc/apt/sources.list.d/debian.sources \ + && sed 's/^Types: deb$/Types: deb-src/' /etc/apt/sources.list.d/debian.sources \ + > /etc/apt/sources.list.d/sqlbot-src.sources \ + && apt-get -o Acquire::Retries=3 -o APT::Update::Error-Mode=any update \ + && DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::Retries=3 build-dep -y --no-install-recommends harfbuzz gdk-pixbuf + +COPY docker/build-graphics.sh /tmp/build-graphics.sh +RUN sh /tmp/build-graphics.sh + +FROM pg-base AS runtime-tools + +COPY --from=python-builder /usr/local/ /usr/local/ +COPY --from=node-runtime /usr/local/bin/node /usr/local/bin/node +COPY --from=ghcr.io/astral-sh/uv:0.13.7@sha256:96f40d3904aad8d0bd010539001d36c8e2188b981e1334471a90c4a1d93bb490 /uv /uvx /bin/ + +FROM runtime-tools AS runtime-deps + +# Add the libraries needed by Python, chart rendering and database clients. +RUN --mount=type=bind,from=graphics-builder,source=/out,target=/tmp/graphics \ + --mount=type=bind,from=graphics-builder,source=/etc/ssl/certs/ca-certificates.crt,target=/tmp/debian-ca.crt \ + sed -i 's|http://deb.debian.org/|https://deb.debian.org/|g' /etc/apt/sources.list.d/debian.sources \ + && apt-get -o Acquire::https::CaInfo=/tmp/debian-ca.crt -o Acquire::Retries=3 -o APT::Update::Error-Mode=any update \ + && DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::https::CaInfo=/tmp/debian-ca.crt -o Acquire::Retries=3 install -y --no-install-recommends \ + ca-certificates curl wait-for-it supervisor tzdata unzip \ + libaio1 libbz2-1.0 libcairo2 libexpat1 libffi8 libfreetype6 \ + libgdbm6 libgdbm-compat4 libgif7 libjpeg62-turbo liblzma5 \ + libncursesw6 libpango-1.0-0 libpangocairo-1.0-0 libpixman-1-0 \ + libpq5 libreadline8 librsvg2-2 libsqlite3-0 libssl3 libuuid1 zlib1g \ + /tmp/graphics/*.deb \ + && rm -rf /var/lib/apt/lists/* \ + && python -c 'import bz2, ctypes, lzma, readline, sqlite3, ssl' \ + && node --version \ + && test "$PG_MAJOR" = 17 \ + && test "$PGDATA" = /var/lib/postgresql/data \ + && ! dpkg-query -W libtiff6 2>/dev/null \ + && test -z "$(find /usr/lib -name 'libpixbufloader-tiff.so' -o -name 'libtiff.so*')" \ + && test -f "$(pg_config --sharedir)/extension/vector.control" + +FROM runtime-deps AS runtime-base -# Download Oracle instant client -ENV DB_CLIENT=/opt/sqlbot/db_client -ENV LD_LIBRARY_PATH="${DB_CLIENT}/oracle_instant_client:${LD_LIBRARY_PATH}" ARG TARGETARCH -RUN apt-get update && \ - apt-get install -y unzip libaio1 && \ - mkdir -p ${DB_CLIENT} && \ - if [ "$TARGETARCH" = "amd64" ]; then \ - echo "Building for x86_64"; \ - curl -L -o app.zip https://download.oracle.com/otn_software/linux/instantclient/2326000/instantclient-basic-linux.x64-23.26.0.0.0.zip; \ - elif [ "$TARGETARCH" = "arm64" ]; then \ - echo "Building for ARM64"; \ - curl -L -o app.zip https://download.oracle.com/otn_software/linux/instantclient/2390000/instantclient-basic-linux.arm64-23.9.0.25.07.zip; \ - fi && \ - unzip app.zip -d ${DB_CLIENT} && \ - rm app.zip && \ - rm -rf /var/lib/apt/lists/* && \ - mv ${DB_CLIENT}/instantclient* ${DB_CLIENT}/oracle_instant_client - -# Download DM library +ENV DB_CLIENT=/opt/sqlbot/db_client ENV DM_HOME=/opt/dmdbms -ENV LD_LIBRARY_PATH=$DM_HOME/bin:$LD_LIBRARY_PATH -RUN curl -L -o dm.zip https://resource-fit2cloud-com.oss-cn-hangzhou.aliyuncs.com/sqlbot/dm/$TARGETARCH.zip && \ - mkdir -p $DM_HOME/bin && \ - unzip dm.zip -d $DM_HOME/bin && \ - chmod -R +x $DM_HOME/bin && \ - rm -rf dm.zip \ No newline at end of file +ENV LD_LIBRARY_PATH=/opt/dmdbms/bin:/opt/sqlbot/db_client/oracle_instant_client + +# Install the existing Oracle client and license validator for the target CPU. +# The application image supplies the DM client through its dmPython wheel. +RUN --mount=type=cache,id=sqlbot-oracle-downloads,target=/var/cache/sqlbot-oracle,sharing=locked set -eu; \ + case "$TARGETARCH" in \ + amd64) oracle_path=2326000/instantclient-basic-linux.x64-23.26.0.0.0.zip ;; \ + arm64) oracle_path=2390000/instantclient-basic-linux.arm64-23.9.0.25.07.zip ;; \ + *) echo "Unsupported architecture: $TARGETARCH" >&2; exit 1 ;; \ + esac; \ + mkdir -p "$DB_CLIENT" "$DM_HOME/bin"; \ + archive="/var/cache/sqlbot-oracle/${oracle_path##*/}"; \ + oracle_url="https://download.oracle.com/otn_software/linux/instantclient/$oracle_path"; \ + if ! unzip -tq "$archive" >/dev/null 2>&1; then \ + for attempt in 1 2 3 4; do \ + if curl -fsSL --connect-timeout 15 --max-time 600 -C - -o "$archive" "$oracle_url"; then \ + if unzip -tq "$archive" >/dev/null 2>&1; then break; fi; \ + rm -f "$archive"; \ + fi; \ + if [ "$attempt" = 4 ]; then echo 'Oracle Instant Client download did not complete.' >&2; exit 1; fi; \ + done; \ + fi; \ + unzip -q "$archive" -d "$DB_CLIENT"; \ + mv "$DB_CLIENT"/instantclient_* "$DB_CLIENT/oracle_instant_client"; \ + validator="/usr/local/bin/validator_linux_$TARGETARCH"; \ + curl -fsSL --retry 3 --connect-timeout 60 --max-time 1800 \ + -o "$validator" "https://fit2cloud-support.oss-cn-beijing.aliyuncs.com/xpack-license/validator_linux_$TARGETARCH"; \ + chmod 0755 "$validator"; \ + ln -s "$validator" /usr/local/bin/validator_linux; \ + ln -s "$validator" /usr/local/bin/validator; \ + ln -s "$validator" /usr/bin/validator; \ + chmod g+xr /usr/bin/ld.so; \ + chmod g+x /usr/local/bin/python* diff --git a/Dockerfile-build-base b/Dockerfile-build-base new file mode 100644 index 000000000..97cf92ffe --- /dev/null +++ b/Dockerfile-build-base @@ -0,0 +1,47 @@ +# syntax=docker/dockerfile:1 +ARG NODE_IMAGE=node:22-bookworm-slim +ARG SQLBOT_RUNTIME_TOOLS_IMAGE=sqlbot-runtime-tools + +FROM ${NODE_IMAGE} AS node-tools +FROM ${SQLBOT_RUNTIME_TOOLS_IMAGE} AS runtime-tools +FROM pg-base + +# Use the same interpreters and PGDG packages as runtime. Bookworm development +# packages require the stock graphics libraries, which stay in this builder. +COPY --from=runtime-tools /usr/local/ /usr/local/ +COPY --from=runtime-tools /bin/uv /bin/uvx /bin/ +COPY --from=node-tools /usr/local/lib/node_modules/ /usr/local/lib/node_modules/ +RUN ln -s ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm \ + && ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx \ + && npm --version + +# Install the toolchain for the Python drivers and native chart modules once. +RUN apt-get -o APT::Update::Error-Mode=any update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential pkg-config libpq-dev libssl-dev libkrb5-dev \ + ca-certificates curl libbz2-1.0 libffi8 libgdbm6 libgdbm-compat4 \ + liblzma5 libncursesw6 libreadline8 libsqlite3-0 \ + libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev \ + libpixman-1-dev libfreetype6-dev \ + && rm -rf /var/lib/apt/lists/* \ + && python -c 'import bz2, ctypes, lzma, readline, sqlite3, ssl' + +# Match the FreeTDS release used by pymssql 2.4.6 wheels and link it to +# the runtime's maintained OpenSSL and Kerberos libraries. +RUN set -eu; \ + curl -fsSL --retry 3 https://www.freetds.org/files/stable/freetds-1.5.19.tar.gz -o /tmp/freetds.tar.gz; \ + echo 'ff70d0c8ae8dd23e7b1314274d6e9d30b479998415760b3fa656754b737c2bd6 /tmp/freetds.tar.gz' | sha256sum -c -; \ + mkdir /tmp/freetds; \ + tar -xzf /tmp/freetds.tar.gz --strip-components=1 -C /tmp/freetds; \ + cd /tmp/freetds; \ + ./configure --prefix=/usr/local --sysconfdir=/etc/freetds --enable-msdblib \ + --with-tdsver=auto --with-openssl=yes --with-gnutls=no --enable-krb5 \ + --disable-static --disable-odbc --disable-apps --disable-server --disable-pool; \ + make -j "$(nproc)"; \ + make install; \ + install -D -m 0644 COPYING_LIB.txt /usr/local/share/doc/freetds/COPYING_LIB.txt; \ + ldconfig; \ + cd /; \ + rm -rf /tmp/freetds /tmp/freetds.tar.gz +ENV LINK_FREETDS_STATICALLY=NO \ + PYMSSQL_FREETDS=/usr/local diff --git a/docker-bake.hcl b/docker-bake.hcl new file mode 100644 index 000000000..886c97c9f --- /dev/null +++ b/docker-bake.hcl @@ -0,0 +1,60 @@ +variable "SQLBOT_BASE_TAGS" { + default = "sqlbot-python-pg:local" +} + +variable "SQLBOT_BUILD_TAGS" { + default = "sqlbot-build-base:local" +} + +variable "SQLBOT_APP_TAGS" { + default = "sqlbot:local" +} + +variable "SQLBOT_BASE_PLATFORMS" { + default = "" +} + +group "default" { + targets = ["base", "build-base"] +} + +target "pg-base" { + dockerfile = "Dockerfile-base" + target = "pg-base" + platforms = compact(split(",", SQLBOT_BASE_PLATFORMS)) +} + +target "runtime-tools" { + inherits = ["pg-base"] + target = "runtime-tools" +} + +target "base" { + inherits = ["runtime-tools"] + target = "runtime-base" + tags = split(",", SQLBOT_BASE_TAGS) +} + +target "build-base" { + dockerfile = "Dockerfile-build-base" + contexts = { + pg-base = "target:pg-base" + sqlbot-runtime-tools = "target:runtime-tools" + } + platforms = compact(split(",", SQLBOT_BASE_PLATFORMS)) + tags = split(",", SQLBOT_BUILD_TAGS) +} + +target "app" { + dockerfile = "Dockerfile" + contexts = { + sqlbot-runtime = "target:base" + sqlbot-build-base = "target:build-base" + } + args = { + SQLBOT_RUNTIME_IMAGE = "sqlbot-runtime" + SQLBOT_BUILD_IMAGE = "sqlbot-build-base" + } + platforms = compact(split(",", SQLBOT_BASE_PLATFORMS)) + tags = split(",", SQLBOT_APP_TAGS) +} diff --git a/docker/build-graphics.sh b/docker/build-graphics.sh new file mode 100644 index 000000000..24ffd6bb4 --- /dev/null +++ b/docker/build-graphics.sh @@ -0,0 +1,34 @@ +#!/bin/sh +set -eu + +export DEB_BUILD_OPTIONS='nocheck nodoc parallel=3' +mkdir -p /out /tmp/graphics/harfbuzz /tmp/graphics/pixbuf + +add_harfbuzz_patch() { + patch_path="debian/patches/$1.patch" + curl -fsSL --retry 3 -o "$patch_path" \ + "https://github.com/harfbuzz/harfbuzz/commit/$2.patch" + echo "$3 $patch_path" | sha256sum -c - + printf '\n%s.patch\n' "$1" >> debian/patches/series +} + +cd /tmp/graphics/harfbuzz +apt-get source --only-source 'harfbuzz=6.0.0+dfsg-3' +cd harfbuzz-*/ +# The fix depends on the matching API introduced by the first commit. +add_harfbuzz_patch sqlbot-skipping-iterator b29fbd16fa82b82bdf0dcb2f13a63f7dc23cf324 \ + 480d72df65f946a38f952a85821ded3b312056e3d5c1c51ce84ecd1bf248b59e +add_harfbuzz_patch sqlbot-CVE-2023-25193 8708b9e081192786c027bb7f5f23d76dbe5c19e8 \ + 06eb34c2b0c7d546f1ff62fe09e5df1a6d45a28428d05fe532a8f1f28a4590dd +sed -i '1s/) /+sqlbot1) /' debian/changelog +dpkg-buildpackage -b -us -uc +cp ../libharfbuzz0b_*_"$(dpkg --print-architecture)".deb /out/ + +cd /tmp/graphics/pixbuf +apt-get source --only-source gdk-pixbuf +cd gdk-pixbuf-*/ +grep -q -- '-Dauto_features=enabled$' debian/rules +sed -i 's/-Dauto_features=enabled$/-Dauto_features=enabled -Dtiff=disabled \\/' debian/rules +sed -i '1s/) /+sqlbot1) /' debian/changelog +dpkg-buildpackage -b -us -uc +cp ../libgdk-pixbuf-2.0-0_*_"$(dpkg --print-architecture)".deb ../libgdk-pixbuf2.0-common_*_all.deb /out/ diff --git a/docs/agents/packaging.md b/docs/agents/packaging.md index de150ee65..a5edda514 100644 --- a/docs/agents/packaging.md +++ b/docs/agents/packaging.md @@ -4,12 +4,14 @@ 运行镜像由多个阶段组成: -1. 前端构建:`frontend/` 执行 `npm install` 和 `npm run build`,产物进入 `/opt/sqlbot/frontend/dist`。 -2. 后端构建:复制 `backend/`,使用 base 镜像中的 uv 安装依赖。以根 `Dockerfile` 为准:中间层包含 `uv sync --frozen` 尝试,最终层执行 `uv sync --extra cpu --default-index https://pypi.org/simple`,使用官方 PyPI 作为默认源,不是全程冻结安装。`uv.lock` 不入库,不能据此承诺全新 checkout 的依赖可复现;`|| echo` 的提示也不能证明失败只因缺少 lock,需检查实际构建日志。 -3. 图表服务构建:复制 `g2-ssr/app.js`、`package.json` 和 `charts/`,安装 Node 依赖和 canvas 相关库。 -4. 运行层:基于含 PostgreSQL / Python 的 base 镜像,复制前端、后端、g2-ssr、字体、向量模型和启动脚本。 +1. 前端构建:在 Node 22 Bookworm 镜像中对 `frontend/` 执行 `npm install` 和 `npm run build`,产物进入 `/opt/sqlbot/frontend/dist`。 +2. 后端构建:使用 `Dockerfile-build-base` 提前构建 FreeTDS 并准备编译器及开发库,应用构建使用 uv 同时解析应用依赖和镜像专用安全约束,构建 PostgreSQL 和 SQL Server 驱动,使其链接运行层系统库。`uv.lock` 不入库,构建会重新解析依赖。 +3. 图表服务构建:复制 `g2-ssr/app.js`、`package.json` 和 `charts/`,使用同一构建基础镜像从源码构建 canvas,使其使用运行层的 Debian 图形库。 +4. 运行层:`Dockerfile-base` 的 `pg-base` 阶段使用固定摘要的 PostgreSQL 17 Bookworm pgvector 镜像,并统一更新继承的 Debian 系统包。后续阶段加入 Python、Node 和共享运行库。`Dockerfile-build-base` 使用同一个 PostgreSQL 底层,复用 Python、Node 和 uv,并单独安装与标准 Bookworm 图形库匹配的开发包。最终运行基础镜像 `sqlbot-python-pg` 还包含 Oracle 客户端及许可证校验器。图形库从 Bookworm 源码包构建,HarfBuzz 应用 CVE-2023-25193 上游补丁,GDK Pixbuf 禁用未使用的 TIFF 加载器。应用镜像复制前端、后端、g2-ssr、字体、向量模型和启动脚本,并使用 dmPython wheel 提供的达梦客户端库。 5. 启动脚本依次准备 PostgreSQL、supervisor/g2-ssr、MCP 服务和主 FastAPI 服务。 +应用镜像保留默认中文向量模型的 PyTorch 权重和 tokenizer,复制模型时排除同一模型未使用的 ONNX 导出。修改模型打包内容时,应在断网容器中验证默认模型可以加载并生成向量。 + 修改路径、端口、依赖或构建脚本时,必须核对最终运行目录仍是 `/opt/sqlbot` 布局,且数据、图片、日志和字体路径没有断开。 ## 本地验证 @@ -56,7 +58,10 @@ node --check g2-ssr/charts/ 规则: -- 根 `Dockerfile` 的构建阶段还依赖 `registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-base:latest`,该镜像不在本仓库工作流中构建(外部维护);`build-base-and-push` 只产出 `sqlbot-python-pg` base 镜像。 +- `build-base-and-push` 使用 `docker buildx bake -f docker-bake.hcl base build-base` 生成并发布同一批次的 `sqlbot-python-pg` 和 `sqlbot-build-base`。两个镜像共享 PostgreSQL 底层和语言工具。图形运行库的补丁只影响运行基础镜像的构建。 +- 两个基础镜像在所选仓库中使用相同的版本标签(如 `v1.10.4`)。启用 `dockerImageTagWithLatest` 时额外发布 `latest`。重跑同一版本会更新该版本标签。 +- 构建应用镜像时,`SQLBOT_RUNTIME_IMAGE` 和 `SQLBOT_BUILD_IMAGE` 可传入同一批次的基础镜像标签或 digest。工作流对应的输入为 `runtimeBaseImage` 和 `builderBaseImage`,各发布任务默认使用目标仓库中对应的 `latest` 标签。 +- 本地执行 `docker buildx bake -f docker-bake.hcl --load base build-base` 构建两个基础镜像,或执行 `docker buildx bake -f docker-bake.hcl --load app` 构建完整应用镜像。单独验证 PostgreSQL 镜像时使用 `docker buildx build -f Dockerfile-base --target pg-base --load -t sqlbot-pg-base:local .`。漏洞复扫以最终应用镜像为准。 - 这些工作流都是发布/分发流程,不要因为普通代码修改建议运行。 - `latest` 只应在正式发布时启用。 - arm64 和 amd64 的依赖、驱动和包名必须同时考虑。 @@ -71,7 +76,7 @@ node --check g2-ssr/charts/ 2. 发布目标 wheel; 3. 主仓库更新 `backend/pyproject.toml` 版本范围,`uv sync` 重锁本机 lock 并安装新 wheel 后重新验证。 3. 确认迁移、默认配置和 installer 模板兼容。 -4. 构建 base 镜像(仅在 base 变更时)。 +4. 构建运行和构建基础镜像(仅在基础镜像变更时)。 5. 构建主镜像。 6. 构建在线 / 离线安装包。 diff --git a/installer/install.sh b/installer/install.sh index 707f27931..e7f30d634 100644 --- a/installer/install.sh +++ b/installer/install.sh @@ -248,7 +248,7 @@ function install_docker_compose() { chmod +x /usr/bin/docker-compose else log_content "在线安装 docker-compose" - curl -L https://resource.fit2cloud.com/docker/compose/releases/download/v2.16.0/docker-compose-$(uname -s | tr A-Z a-z)-$(uname -m) -o /usr/local/bin/docker-compose 2>&1 | tee -a ${CURRENT_DIR}/install.log + curl -L https://resource.fit2cloud.com/docker/compose/releases/download/v5.7.0/docker-compose-$(uname -s | tr A-Z a-z)-$(uname -m) -o /usr/local/bin/docker-compose 2>&1 | tee -a ${CURRENT_DIR}/install.log if [[ ! -f /usr/local/bin/docker-compose ]];then log_content "docker-compose 下载失败,请稍候重试" exit 1