Merge branch 'main' into msvc_static

Author: evertlammerts

.github/workflows/packaging_wheels.yml

@@ -33,11 +33,12 @@ jobs:
         python: [ cp39, cp310, cp311, cp312, cp313, cp314 ]
         platform:
           - { os: windows-2025,     arch: amd64,      cibw_system: win }
+          - { os: windows-11-arm,   arch: ARM64,      cibw_system: win } # cibw requires ARM64 to be uppercase
           - { os: ubuntu-24.04,     arch: x86_64,     cibw_system: manylinux }
           - { os: ubuntu-24.04-arm, arch: aarch64,    cibw_system: manylinux }
           - { os: macos-15,         arch: arm64,      cibw_system: macosx }
           - { os: macos-15,         arch: universal2, cibw_system: macosx }
-          - { os: macos-13,         arch: x86_64,     cibw_system: macosx }
+          - { os: macos-15-intel,   arch: x86_64,     cibw_system: macosx }
         minimal:
           - ${{ inputs.minimal }}
         exclude:
@@ -46,6 +47,8 @@ jobs:
           - { minimal: true, python: cp312 }
           - { minimal: true, python: cp313 }
           - { minimal: true, platform: { arch: universal2 } }
+          - { python: cp39,  platform: { os: windows-11-arm, arch: ARM64 } }  # too many dependency problems for win arm64
+          - { python: cp310,  platform: { os: windows-11-arm, arch: ARM64 } }  # too many dependency problems for win arm64
     runs-on: ${{ matrix.platform.os }}
     env:
       ### cibuildwheel configuration

.github/workflows/release.yml

@@ -22,6 +22,11 @@ on:
         options:
           - test
           - prod
+      nightly-stale-after-days:
+        type: string
+        description: After how many days should nightlies be considered stale
+        required: true
+        default: 3
       store-s3:
         type: boolean
         description: Also store test packages in S3 (always true for prod)
@@ -41,6 +46,17 @@ jobs:
       duckdb-sha: ${{ inputs.duckdb-sha }}
       set-version: ${{ inputs.stable-version }}
 
+  submodule_pr:
+    name: Create or update PR to bump submodule to given SHA
+    needs: build_sdist
+    uses: ./.github/workflows/submodule_auto_pr.yml
+    with:
+      duckdb-python-sha: ${{ inputs.duckdb-python-sha }}
+      duckdb-sha: ${{ inputs.duckdb-sha }}
+    secrets:
+      # reusable workflows and secrets are not great: https://github.com/actions/runner/issues/3206
+      DUCKDBLABS_BOT_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
+
   workflow_state:
     name: Set state for the release workflow
     needs: build_sdist
@@ -51,23 +67,36 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - id: index_check
-        name: Check ${{ needs.build_sdist.outputs.package-version }} on PyPI
+        name: Check version on PyPI
         run: |
-          set -eu
-          # Check PyPI whether the release we're building is already present
+          set -ex
           pypi_hostname=${{ inputs.pypi-index == 'test' && 'test.' || '' }}pypi.org
-          pkg_version=${{ needs.build_sdist.outputs.package-version }}
-          url=https://${pypi_hostname}/pypi/duckdb/${pkg_version}/json
-          http_status=$( curl -s -o /dev/null -w "%{http_code}" $url || echo $? )
-          if [[ $http_status == "200" ]]; then
-            echo "::warning::Package version ${pkg_version} is already present on ${pypi_hostname}"
-            pypi_state=VERSION_FOUND
-          elif [[ $http_status == 000* ]]; then
-            echo "::error::Error checking PyPI at ${url}: curl exit code ${http_status#'000'}"
-            pypi_state=UNKNOWN
-          else
-            echo "::notice::Package version ${pkg_version} not found on ${pypi_hostname} (http status: ${http_status})"
+          # install duckdb
+          curl https://install.duckdb.org | sh
+          # query pypi
+          result=$(cat <<EOF | ${HOME}/.duckdb/cli/latest/duckdb | xargs
+          ---- Output lines
+          .mode line
+          ---- Query that fetches the given version's age, if the version already exists
+          SELECT
+            today() - (file.value->>'upload_time_iso_8601')::DATE     AS age,
+          FROM read_json('https://${pypi_hostname}/pypi/duckdb/json') AS jd
+          CROSS JOIN json_each(jd.releases)                           AS rel(key, value)
+          CROSS JOIN unnest(FROM_JSON(rel.value, '["JSON"]'))         AS file(value)
+          WHERE rel.key='${{ needs.build_sdist.outputs.package-version }}'
+          LIMIT 1;
+          EOF
+          )
+          if [ -z "$result" ]; then
             pypi_state=VERSION_NOT_FOUND
+          else
+            pypi_state=VERSION_FOUND
+          fi
+          if [[ -z "${{ inputs.stable-version }}" ]]; then
+            age=${result#age = }
+            if [ "${age}" -ge "${{ inputs.nightly-stale-after-days }}" ]; then
+              echo "::warning title=Stale nightly for ${{ github.ref_name }}::Nightly is ${age} days old (max=${{ inputs.nightly-stale-after-days }})"
+            fi
           fi
           echo "pypi_state=${pypi_state}" >> $GITHUB_OUTPUT
 
@@ -96,7 +125,7 @@ jobs:
             echo "::notice::S3 upload disabled in inputs, not generating S3 URL"
             exit 0
           fi
-          if [[ VERSION_FOUND == "${{ steps.index_check.outputs.pypi_state }}" ]]; then
+          if [[ VERSION_NOT_FOUND != "${{ steps.index_check.outputs.pypi_state }}" ]]; then
             echo "::warning::S3 upload disabled because package version already uploaded to PyPI"
             exit 0
           fi
@@ -110,7 +139,7 @@ jobs:
   build_wheels:
     name: Build and test releases
     needs: workflow_state
-    if: ${{ needs.workflow_state.outputs.pypi_state != 'VERSION_FOUND' }}
+    if: ${{ needs.workflow_state.outputs.pypi_state == 'VERSION_NOT_FOUND' }}
     uses: ./.github/workflows/packaging_wheels.yml
     with:
       minimal: false
@@ -132,14 +161,12 @@ jobs:
           path: artifacts/
           merge-multiple: true
 
-      - name: Authenticate with AWS
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-region: 'us-east-2'
-          aws-access-key-id: ${{ secrets.S3_DUCKDB_STAGING_ID }}
-          aws-secret-access-key: ${{ secrets.S3_DUCKDB_STAGING_KEY }}
-
       - name: Upload Artifacts
+        env:
+          AWS_ENDPOINT_URL: ${{ secrets.S3_DUCKDB_STAGING_ENDPOINT }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }}
+
         run: |
           aws s3 cp artifacts ${{ needs.workflow_state.outputs.s3_url }} --recursive
 

.github/workflows/submodule_auto_pr.yml

@@ -0,0 +1,130 @@
+name: Submodule Auto PR
+on:
+  workflow_call:
+    inputs:
+      duckdb-python-sha:
+        type: string
+        description: The commit to build against (defaults to latest commit of current ref)
+        required: false
+      duckdb-sha:
+        type: string
+        description: The DuckDB submodule commit or ref to build against
+        required: true
+      auto-land:
+        type: boolean
+        description: Immediately merge the PR (placeholder - doesn't work)
+        default: false
+    secrets:
+      DUCKDBLABS_BOT_TOKEN:
+        description: Github token of the DuckDBLabs bot
+        required: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  create_pr:
+    name: Create PR to bump duckdb submodule to given SHA
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout DuckDB Python
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.duckdb-python-sha }}
+          fetch-depth: 0
+          submodules: true
+
+      - name: Checkout or Create Needed Branch
+        run: |
+          git fetch --all
+          head_sha=${{ inputs.duckdb-python-sha }}
+          branch_name="vendoring-${{ github.ref_name }}"
+          if [[ `git rev-parse --verify ${branch_name} 2>/dev/null` ]]; then
+            # branch exists
+            git checkout ${branch_name}
+          else
+            # new branch
+            git checkout -b ${branch_name}
+          fi
+          [[ ${head_sha} ]] && git reset --hard ${head_sha} || true
+
+      - name: Checkout DuckDB at Given SHA
+        run: |
+          cd external/duckdb
+          git fetch origin
+          git checkout ${{ inputs.duckdb-sha }}
+
+      - name: Determine GH PR Command
+        id: gh_pr_command
+        env:
+          GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
+        run: |
+          pr_url=$( gh pr list --head vendoring-${{ github.ref_name }} --state open --json url --jq '.[].url' )
+          if [[ $pr_url ]]; then
+            echo "::notice::Found existing pr, will edit (${pr_url})"
+            gh_command="edit ${pr_url}"
+          else
+            echo "::notice::No existing PR, will create new"
+            gh_command="create --head vendoring-${{ github.ref_name }} --base ${{ github.ref_name }}"
+          fi
+          echo "subcommand=${gh_command}" >> $GITHUB_OUTPUT
+
+      - name: Set Git User
+        run: |
+          git config --global user.email "github_bot@duckdblabs.com"
+          git config --global user.name "DuckDB Labs GitHub Bot"
+
+      - name: Create PR to Bump DuckDB Submodule
+        env:
+          GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
+        run: |
+          # No need to do anything if the submodule is already at the given sha
+          [[ `git status --porcelain -- external/duckdb` == "" ]] && exit 0
+          # We have changes. Commit and push
+          git add external/duckdb
+          git commit -m "Bump submodule"
+          git push --force origin vendoring-${{ github.ref_name }}
+          # create PR msg
+          echo "Bump duckdb submodule:" > body.txt
+          echo "- Target branch: ${{ github.ref_name }}" >> body.txt
+          echo "- Date: $( date +"%Y-%m-%d %H:%M:%S" )" >> body.txt
+          echo "- DuckDB SHA: ${{ inputs.duckdb-sha }}" >> body.txt
+          echo "- Trigger: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> body.txt
+          subcommand="${{ steps.gh_pr_command.outputs.subcommand }}"
+          gh pr ${subcommand} \
+            --title "[duckdb-labs bot] Bump DuckDB submodule" \
+            --body-file body.txt > output.txt 2>&1
+          success=$?
+          # Show summary
+          url=$( [[ $success ]] && gh pr view vendoring-${{ github.ref_name }} --json url --jq .url || true )
+          echo "## Submodule PR Summary" >> $GITHUB_STEP_SUMMARY
+          if [[ $success ]]; then
+            prefix=$( [[ $subcommand == edit* ]] && echo "Created" || echo "Updated" )
+            echo "### ${prefix} PR: [${url}](${url})" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "### Failed to create PR" >> $GITHUB_STEP_SUMMARY
+          fi
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          cat output.txt >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          [[ $success ]] || exit 1
+
+      - name: Automerge PR
+        if: ${{ inputs.auto-land }}
+        env:
+          GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
+        run: |
+          # PLACEHOLDER: DUCKDBLABS_BOT_TOKEN DOES NOT HAVE PERMISSIONS TO MERGE PRS
+          set -ex
+          gh pr merge vendoring-${{ github.ref_name }} --rebase > output.txt
+          success=$?
+          # Show summary
+          if [[ $success ]]; then
+            echo "### PR merged" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "### Failed to auto-merge PR" >> $GITHUB_STEP_SUMMARY
+          fi
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          cat output.txt >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY

.github/workflows/targeted_test.yml

@@ -0,0 +1,92 @@
+name: Targeted Platform Testing
+
+on:
+  workflow_dispatch:
+    inputs:
+      platform:
+        description: 'Platform to test on'
+        required: true
+        type: choice
+        options:
+          - 'windows-2025'
+          - 'windows-11-arm'
+          - 'ubuntu-24.04'
+          - 'ubuntu-24.04-arm'
+          - 'macos-15'
+          - 'macos-15-intel'
+      python_version:
+        description: 'Python version to test'
+        required: true
+        type: choice
+        options:
+          - '3.9'
+          - '3.10'
+          - '3.11'
+          - '3.12'
+          - '3.13'
+          - '3.14'
+      testsuite:
+        description: 'Test suite to run (ignored if custom_test_path is provided)'
+        required: false
+        type: choice
+        options:
+          - 'fast'
+          - 'all'
+        default: 'fast'
+      custom_test_path:
+        description: 'Custom test path (must be in tests/ directory, overrides testsuite)'
+        required: false
+        type: string
+      verbose-uv:
+        description: 'Let uv generate verbose output (pytest verbosity is always on)'
+        required: false
+        type: boolean
+        default: true
+
+jobs:
+  test:
+    name: 'Test with Python ${{ inputs.python_version }} on ${{ inputs.platform }}'
+    runs-on: ${{ inputs.platform }}
+
+    steps:
+      - name: Checkout DuckDB Python
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          version: "0.9.0"
+          enable-cache: false
+          python-version: ${{ inputs.python_version }}
+
+      - name: Set and validate test path
+        id: test_path
+        shell: bash
+        run: |
+          if [[ -n "${{ inputs.custom_test_path }}" ]]; then
+            # test path was passed in
+            tests_base="$( pwd -P )/tests"
+            test_path="${{ inputs.custom_test_path }}"
+
+            # Ensure the given test path exists
+            [[ -e "$test_path" ]] || { echo "${test_path} does not exist"; exit 1; }
+
+            # Resolve custom test path to absolute path
+            test_path_abs=$(cd "$test_path" 2>/dev/null && pwd -P || ( cd "$(dirname "$test_path")" && printf '%s/%s' "$(pwd -P)" "$(basename "$test_path")" ) )
+
+            # Make sure test_path_abs is inside tests_base
+            [[ "$test_path_abs" == "$tests_base" || "$test_path_abs" == "$tests_base"/* ]] || { echo "${test_path_abs} is not part of ${tests_base}?"; exit 1; }
+
+            echo "test_path=$test_path_abs" >> $GITHUB_OUTPUT
+          else
+            # use a testsuite
+            echo "test_path=$GITHUB_WORKSPACE/${{ inputs.testsuite == 'fast' && 'tests/fast' || 'tests' }}" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Run tests
+        shell: bash
+        run: |
+          uv ${{ inputs.verbose-uv && 'run -v' || 'run' }} pytest -vv ${{ steps.test_path.outputs.test_path }}

CHANGELOG.md

@@ -79,6 +79,7 @@ pybind11_add_module(
   $<TARGET_OBJECTS:python_type>)
 # add _duckdb_dependencies
 target_link_libraries(_duckdb PRIVATE _duckdb_dependencies)
+duckdb_link_extensions(_duckdb)
 
 # ────────────────────────────────────────────
 # Controlling symbol export