Skip to content

Commit d33b693

Browse files
evertlammertsevertlammerts
committed
enable caching
1 parent 342e4ba commit d33b693

2 files changed

Lines changed: 141 additions & 7 deletions

File tree

‎.github/actions/ccache-action/action.yml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
name: 'Setup Ccache'
2+
inputs:
3+
key:
4+
description: 'Cache key (defaults to github.job)'
5+
required: false
6+
default: ''
7+
runs:
8+
using: "composite"
9+
steps:
10+
- name: Setup Ccache
11+
uses: hendrikmuhs/ccache-action@main
12+
with:
13+
key: ${{ inputs.key || github.job }}
14+
save: ${{ github.repository != 'duckdb/duckdb-python' || contains('["refs/heads/main", "refs/heads/v1.4-andium", "refs/heads/v1.5-variegata"]', github.ref) }}
15+
# Dump verbose ccache statistics report at end of CI job.
16+
verbose: 1
17+
# Increase per-directory limit: 5*1024 MB / 16 = 320 MB.
18+
# Note: `layout=subdirs` computes the size limit divided by 16 dirs.
19+
# See also: https://ccache.dev/manual/4.9.html#_cache_size_management
20+
max-size: 1500MB
21+
# Evicts all cache files that were not touched during the job run.
22+
# Removing cache files from previous runs avoids creating huge caches.
23+
evict-old-files: 'job'

‎.github/workflows/packaging_wheels.yml‎

Lines changed: 118 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,15 +25,112 @@ on:
2525
type: string
2626

2727
jobs:
28+
seed_wheels:
29+
name: 'Seed: cp314-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}'
30+
strategy:
31+
fail-fast: false
32+
matrix:
33+
python: [ cp314 ]
34+
platform:
35+
- { os: windows-2025, arch: amd64, cibw_system: win }
36+
- { os: windows-11-arm, arch: ARM64, cibw_system: win }
37+
- { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux }
38+
- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux }
39+
- { os: macos-15, arch: arm64, cibw_system: macosx }
40+
- { os: macos-15, arch: universal2, cibw_system: macosx }
41+
- { os: macos-15-intel, arch: x86_64, cibw_system: macosx }
42+
minimal:
43+
- ${{ inputs.minimal }}
44+
exclude:
45+
- { minimal: true, platform: { arch: universal2 } }
46+
runs-on: ${{ matrix.platform.os }}
47+
env:
48+
CCACHE_DIR: ${{ github.workspace }}/.ccache
49+
### cibuildwheel configuration
50+
#
51+
# This is somewhat brittle, so be careful with changes. Some notes for our future selves (and others):
52+
# - cibw will change its cwd to a temp dir and create a separate venv for testing. It then installs the wheel it
53+
# built into that venv, and run the CIBW_TEST_COMMAND. We have to install all dependencies ourselves, and make
54+
# sure that the pytest config in pyproject.toml is available.
55+
# - CIBW_BEFORE_TEST installs the test dependencies by exporting them into a pylock.toml. At the time of writing,
56+
# `uv sync --no-install-project` had problems correctly resolving dependencies using resolution environments
57+
# across all platforms we build for. This might be solved in newer uv versions.
58+
# - CIBW_TEST_COMMAND specifies pytest conf from pyproject.toml. --confcutdir is needed to prevent pytest from
59+
# traversing the full filesystem, which produces an error on Windows.
60+
# - CIBW_TEST_SKIP we always skip tests for *-macosx_universal2 builds, because we run tests for arm64 and x86_64.
61+
CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }}
62+
CIBW_TEST_SOURCES: tests
63+
CIBW_BEFORE_TEST: >
64+
uv export --only-group test --no-emit-project --quiet --output-file pylock.toml --directory {project} &&
65+
uv pip install -r pylock.toml
66+
CIBW_TEST_COMMAND: >
67+
uv run -v pytest --confcutdir=. --rootdir . -c {project}/pyproject.toml ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }}
68+
69+
steps:
70+
- name: Checkout DuckDB Python
71+
uses: actions/checkout@v4
72+
with:
73+
ref: ${{ inputs.duckdb-python-sha }}
74+
fetch-depth: 0
75+
submodules: true
76+
77+
- name: Checkout DuckDB
78+
shell: bash
79+
if: ${{ inputs.duckdb-sha }}
80+
run: |
81+
cd external/duckdb
82+
git fetch origin
83+
git checkout ${{ inputs.duckdb-sha }}
84+
85+
- name: Set CIBW_ENVIRONMENT
86+
shell: bash
87+
run: |
88+
cibw_env=""
89+
if [[ "${{ matrix.platform.cibw_system }}" == "manylinux" ]]; then
90+
cibw_env="CCACHE_DIR=/host${{ github.workspace }}/.ccache"
91+
fi
92+
if [[ -n "${{ inputs.set-version }}" ]]; then
93+
cibw_env="${cibw_env:+$cibw_env }OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}"
94+
fi
95+
if [[ -n "$cibw_env" ]]; then
96+
echo "CIBW_ENVIRONMENT=${cibw_env}" >> $GITHUB_ENV
97+
fi
98+
99+
- name: Setup Ccache
100+
uses: ./.github/actions/ccache-action
101+
with:
102+
key: ${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
103+
104+
# Install Astral UV, which will be used as build-frontend for cibuildwheel
105+
- uses: astral-sh/setup-uv@v7
106+
with:
107+
version: "0.9.0"
108+
enable-cache: false
109+
cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
110+
111+
- name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels
112+
uses: pypa/cibuildwheel@v3.2
113+
env:
114+
CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }}
115+
CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
116+
117+
- name: Upload wheel
118+
uses: actions/upload-artifact@v4
119+
with:
120+
name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
121+
path: wheelhouse/*.whl
122+
compression-level: 0
123+
28124
build_wheels:
29125
name: 'Wheel: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}'
126+
needs: seed_wheels
30127
strategy:
31128
fail-fast: false
32129
matrix:
33-
python: [ cp310, cp311, cp312, cp313, cp314 ]
130+
python: [ cp310, cp311, cp312, cp313 ]
34131
platform:
35132
- { os: windows-2025, arch: amd64, cibw_system: win }
36-
- { os: windows-11-arm, arch: ARM64, cibw_system: win } # cibw requires ARM64 to be uppercase
133+
- { os: windows-11-arm, arch: ARM64, cibw_system: win }
37134
- { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux }
38135
- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux }
39136
- { os: macos-15, arch: arm64, cibw_system: macosx }
@@ -46,9 +143,10 @@ jobs:
46143
- { minimal: true, python: cp312 }
47144
- { minimal: true, python: cp313 }
48145
- { minimal: true, platform: { arch: universal2 } }
49-
- { python: cp310, platform: { os: windows-11-arm, arch: ARM64 } } # too many dependency problems for win arm64
146+
- { python: cp310, platform: { os: windows-11-arm, arch: ARM64 } }
50147
runs-on: ${{ matrix.platform.os }}
51148
env:
149+
CCACHE_DIR: ${{ github.workspace }}/.ccache
52150
### cibuildwheel configuration
53151
#
54152
# This is somewhat brittle, so be careful with changes. Some notes for our future selves (and others):
@@ -85,11 +183,24 @@ jobs:
85183
git fetch origin
86184
git checkout ${{ inputs.duckdb-sha }}
87185
88-
# Make sure that OVERRIDE_GIT_DESCRIBE is propagated to cibuildwhel's env, also when it's running linux builds
89-
- name: Set OVERRIDE_GIT_DESCRIBE
186+
- name: Set CIBW_ENVIRONMENT
90187
shell: bash
91-
if: ${{ inputs.set-version != '' }}
92-
run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV
188+
run: |
189+
cibw_env=""
190+
if [[ "${{ matrix.platform.cibw_system }}" == "manylinux" ]]; then
191+
cibw_env="CCACHE_DIR=/host${{ github.workspace }}/.ccache"
192+
fi
193+
if [[ -n "${{ inputs.set-version }}" ]]; then
194+
cibw_env="${cibw_env:+$cibw_env }OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}"
195+
fi
196+
if [[ -n "$cibw_env" ]]; then
197+
echo "CIBW_ENVIRONMENT=${cibw_env}" >> $GITHUB_ENV
198+
fi
199+
200+
- name: Setup Ccache
201+
uses: ./.github/actions/ccache-action
202+
with:
203+
key: ${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
93204

94205
# Install Astral UV, which will be used as build-frontend for cibuildwheel
95206
- uses: astral-sh/setup-uv@v7

0 commit comments

Comments
 (0)