Skip to content

build/bake: support configurable image layer compression #339

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
crazy-max wants to merge 1 commit into
base: main
Choose a base branch
from
Open

build/bake: support configurable image layer compression #339

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions .github/workflows/.test-bake.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -750,3 +750,41 @@ jobs:
username: ${{ vars.DOCKERHUB_OIDC_USERNAME }}
connection_id: ${{ vars.DOCKERHUB_OIDC_CONNECTIONID }}
scope: 'dhi.io@pull'

bake-compression:
strategy:
matrix:
include:
- variant: default
compression: ''
level: ''
force: false
- variant: zstd
compression: zstd
level: '0'
force: false
- variant: zstd-force
compression: zstd
level: '9'
force: true
uses: ./.github/workflows/bake.yml
permissions:
contents: read
id-token: write
packages: write
with:
compression: ${{ matrix.compression }}
compression-level: ${{ matrix.level }}
force-compression: ${{ matrix.force }}
output: image
push: ${{ github.event_name != 'pull_request' }}
context: test
target: compression
meta-images: ghcr.io/docker/github-builder-test
meta-tags: |
type=raw,value=bake-compression-${{ matrix.variant }}-${{ github.run_id }}-${{ github.run_attempt }}
secrets:
registry-auths: |
- registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
38 changes: 38 additions & 0 deletions .github/workflows/.test-build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -759,3 +759,41 @@ jobs:
username: ${{ vars.DOCKERHUB_OIDC_USERNAME }}
connection_id: ${{ vars.DOCKERHUB_OIDC_CONNECTIONID }}
scope: 'dhi.io@pull'

build-compression:
strategy:
matrix:
include:
- variant: default
compression: ''
level: ''
force: false
- variant: zstd
compression: zstd
level: '0'
force: false
- variant: zstd-force
compression: zstd
level: '9'
force: true
uses: ./.github/workflows/build.yml
permissions:
contents: read
id-token: write
packages: write
with:
compression: ${{ matrix.compression }}
compression-level: ${{ matrix.level }}
force-compression: ${{ matrix.force }}
output: image
push: ${{ github.event_name != 'pull_request' }}
file: test/compression.Dockerfile
platforms: linux/amd64,linux/arm64
meta-images: ghcr.io/docker/github-builder-test
meta-tags: |
type=raw,value=build-compression-${{ matrix.variant }}-${{ github.run_id }}-${{ github.run_attempt }}
secrets:
registry-auths: |
- registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
77 changes: 51 additions & 26 deletions .github/workflows/bake.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,19 @@ on:
description: "Cache layers to export if cache enabled (min or max)"
required: false
default: 'min'
compression:
type: string
description: "Image layer compression (uncompressed, gzip, estargz or zstd)"
required: false
compression-level:
type: string
description: "Image layer compression level (0-9 for gzip and estargz, 0-22 for zstd)"
required: false
force-compression:
type: boolean
description: "Recompress existing image layers using the selected compression type"
required: false
default: false
context:
type: string
description: "Context to build from in the Git working tree"
Expand Down Expand Up @@ -219,6 +232,7 @@ jobs:
include:
- name: ${{ inputs.job-name-prefix }}
outputs:
buildOutput: ${{ steps.set.outputs.buildOutput }}
includes: ${{ steps.set.outputs.includes }}
metaImages: ${{ steps.set.outputs.metaImages }}
sign: ${{ steps.set.outputs.sign }}
Expand Down Expand Up @@ -315,6 +329,10 @@ jobs:
id: set
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
INPUT_LOCAL-EXPORT-DIR: ${{ env.LOCAL_EXPORT_DIR }}
INPUT_COMPRESSION: ${{ inputs.compression }}
INPUT_COMPRESSION-LEVEL: ${{ inputs.compression-level }}
INPUT_FORCE-COMPRESSION: ${{ inputs.force-compression }}
INPUT_SBOM-IMAGE: ${{ env.SBOM_IMAGE }}
INPUT_MATRIX-SIZE-LIMIT: ${{ env.MATRIX_SIZE_LIMIT }}
INPUT_META-IMAGES: ${{ inputs.meta-images }}
Expand All @@ -341,6 +359,7 @@ jobs:
const { Build } = require('@docker/github-builder-runtime/lib/buildx/build');
const { GitHub } = require('@docker/github-builder-runtime/lib/github/github');
const { MetaImages } = require('@docker/github-builder-runtime/lib/github-builder/meta-images');
const { BuildOutput } = require('@docker/github-builder-runtime/lib/github-builder/build-output');
const { RunnerMapping } = require('@docker/github-builder-runtime/lib/github-builder/runner-mapping');
const { BakeTargets } = require('@docker/github-builder-runtime/lib/github-builder/bake-targets');
const { BakeVars } = require('@docker/github-builder-runtime/lib/github-builder/bake-vars');
Expand All @@ -364,6 +383,10 @@ jobs:
const inpSbom = core.getBooleanInput('sbom');
const inpSet = Util.getInputList('set', {ignoreComma: true, quote: false});
const inpSign = core.getInput('sign');
const inpCompression = core.getInput('compression');
const inpCompressionLevel = core.getInput('compression-level');
const inpForceCompression = core.getBooleanInput('force-compression');
const inpLocalExportDir = core.getInput('local-export-dir');
const inpTarget = core.getInput('target');
const inpGitHubToken = core.getInput('github-token');
const inpProxyNetwork = core.getBooleanInput('buildkit-proxy-network');
Expand Down Expand Up @@ -414,7 +437,33 @@ jobs:
}).sort(), null, 2));
});

if (inpCompression && !['uncompressed', 'gzip', 'estargz', 'zstd'].includes(inpCompression)) {
core.setFailed(`Invalid compression: ${inpCompression}`);
return;
}
if (inpCompressionLevel !== '' && (!/^\d+$/.test(inpCompressionLevel) || !Number.isSafeInteger(Number(inpCompressionLevel)))) {
core.setFailed(`Invalid compression-level: ${inpCompressionLevel}`);
return;
}
if (inpOutput === 'local' && (inpCompression || inpCompressionLevel !== '' || inpForceCompression)) {
core.warning('Compression settings are ignored for local output');
}

const metaImages = MetaImages.resolve(inpMetaImages);
const buildOutput = BuildOutput.resolve({
output: inpOutput,
images: metaImages,
push: inpPush,
localExportDir: inpLocalExportDir,
compression: inpCompression || undefined,
compressionLevel: inpCompressionLevel !== '' ? Number(inpCompressionLevel) : undefined,
forceCompression: inpForceCompression || undefined
});
await core.group(`Set buildOutput output`, async () => {
core.info(buildOutput);
core.setOutput('buildOutput', buildOutput);
});

await core.group(`Set metaImages output`, async () => {
core.info(JSON.stringify(metaImages, null, 2));
core.setOutput('metaImages', metaImages.join('\n'));
Expand Down Expand Up @@ -822,23 +871,20 @@ jobs:
id: prepare
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
INPUT_BUILD-OUTPUT: ${{ needs.prepare.outputs.buildOutput }}
INPUT_PLATFORM: ${{ matrix.platform }}
INPUT_SBOM-IMAGE: ${{ env.SBOM_IMAGE }}
INPUT_LOCAL-EXPORT-DIR: ${{ env.LOCAL_EXPORT_DIR }}
INPUT_CACHE: ${{ inputs.cache }}
INPUT_CACHE-SCOPE: ${{ inputs.cache-scope }}
INPUT_CACHE-MODE: ${{ inputs.cache-mode }}
INPUT_BUILD-SECRETS: ${{ secrets.build-secrets }}
INPUT_TARGETS: ${{ needs.prepare.outputs.targets }}
INPUT_CONTEXT: ${{ inputs.context }}
INPUT_FILES: ${{ inputs.files }}
INPUT_OUTPUT: ${{ inputs.output }}
INPUT_PUSH: ${{ inputs.push }}
INPUT_SBOM: ${{ inputs.sbom }}
INPUT_SET: ${{ inputs.set }}
INPUT_TARGET: ${{ inputs.target }}
INPUT_VARS: ${{ inputs.vars }}
INPUT_META-IMAGES: ${{ needs.prepare.outputs.metaImages }}
INPUT_META-VERSION: ${{ steps.meta.outputs.version }}
INPUT_META-TAGS: ${{ steps.meta.outputs.tags }}
INPUT_SET-META-ANNOTATIONS: ${{ inputs.set-meta-annotations }}
Expand All @@ -861,7 +907,6 @@ jobs:
core.setOutput('platform-pair-suffix', platformPairSuffix);

const inpSbomImage = core.getInput('sbom-image');
const inpLocalExportDir = core.getInput('local-export-dir');

const inpCache = core.getBooleanInput('cache');
const inpCacheScope = core.getInput('cache-scope');
Expand All @@ -870,15 +915,12 @@ jobs:
const inpTargets = core.getInput('targets');
const inpContext = core.getInput('context');
const inpFiles = Util.getInputList('files');
const inpOutput = core.getInput('output');
const inpPush = core.getBooleanInput('push');
const inpSbom = core.getBooleanInput('sbom');
const inpSet = Util.getInputList('set', {ignoreComma: true, quote: false});
const inpTarget = core.getInput('target');
const inpVars = Util.getInputList('vars', {ignoreComma: true, quote: false});
const inpBuildkitProxyNetwork = core.getBooleanInput('buildkit-proxy-network');

const inpMetaImages = core.getMultilineInput('meta-images');
const inpMetaVersion = core.getInput('meta-version');
const inpMetaTags = core.getMultilineInput('meta-tags');
const inpSetMetaAnnotations = core.getBooleanInput('set-meta-annotations');
Expand Down Expand Up @@ -947,23 +989,6 @@ jobs:
core.setOutput('files', bakeFiles.join(os.EOL));
});

let outputOverride = '';
switch (inpOutput) {
case 'image':
if (inpMetaImages.length == 0) {
core.setFailed('meta-images is required when output is image');
return;
}
outputOverride = `*.output=type=image,"name=${inpMetaImages.join(',')}",oci-artifact=true,push-by-digest=true,name-canonical=true,push=${inpPush}`;
break;
case 'local':
outputOverride = `*.output=type=local,platform-split=true,dest=${inpLocalExportDir}`;
break;
default:
core.setFailed(`Invalid output: ${inpOutput}`);
return;
}

let bakeOverrides = [];
try {
bakeOverrides = inpSet.map(override => renderTemplate(override));
Expand All @@ -972,7 +997,7 @@ jobs:
return;
}
await core.group(`Set bake overrides`, async () => {
bakeOverrides.push(outputOverride);
bakeOverrides.push(`*.output=${core.getInput('build-output')}`);
bakeOverrides.push('*.tags=');
if (GitHub.context.payload.repository?.private ?? false) {
// if this is a private repository, we set min provenance mode
Expand Down
Loading
Loading