Skip to content

Commit 11fa60f

Browse files
evertlammertsevertlammerts
authored
Merge v1.4-andium and fix build (#214)
Merges andium into main and fixes build problems: - replacement scan needs to throw an exception on serialization - adbc driver manager now throws a programming error when a table already exists
2 parents ee14b83 + c5d6dc0 commit 11fa60f

26 files changed

Lines changed: 404 additions & 162 deletions

‎.github/workflows/packaging_wheels.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,12 @@ jobs:
3333
python: [ cp39, cp310, cp311, cp312, cp313, cp314 ]
3434
platform:
3535
- { os: windows-2025, arch: amd64, cibw_system: win }
36+
- { os: windows-11-arm, arch: ARM64, cibw_system: win } # cibw requires ARM64 to be uppercase
3637
- { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux }
3738
- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux }
3839
- { os: macos-15, arch: arm64, cibw_system: macosx }
3940
- { os: macos-15, arch: universal2, cibw_system: macosx }
40-
- { os: macos-13, arch: x86_64, cibw_system: macosx }
41+
- { os: macos-15-intel, arch: x86_64, cibw_system: macosx }
4142
minimal:
4243
- ${{ inputs.minimal }}
4344
exclude:
@@ -46,6 +47,8 @@ jobs:
4647
- { minimal: true, python: cp312 }
4748
- { minimal: true, python: cp313 }
4849
- { minimal: true, platform: { arch: universal2 } }
50+
- { python: cp39, platform: { os: windows-11-arm, arch: ARM64 } } # too many dependency problems for win arm64
51+
- { python: cp310, platform: { os: windows-11-arm, arch: ARM64 } } # too many dependency problems for win arm64
4952
runs-on: ${{ matrix.platform.os }}
5053
env:
5154
### cibuildwheel configuration

‎.github/workflows/release.yml‎

Lines changed: 45 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,11 @@ on:
2222
options:
2323
- test
2424
- prod
25+
nightly-stale-after-days:
26+
type: string
27+
description: After how many days should nightlies be considered stale
28+
required: true
29+
default: 3
2530
store-s3:
2631
type: boolean
2732
description: Also store test packages in S3 (always true for prod)
@@ -41,6 +46,17 @@ jobs:
4146
duckdb-sha: ${{ inputs.duckdb-sha }}
4247
set-version: ${{ inputs.stable-version }}
4348

49+
submodule_pr:
50+
name: Create or update PR to bump submodule to given SHA
51+
needs: build_sdist
52+
uses: ./.github/workflows/submodule_auto_pr.yml
53+
with:
54+
duckdb-python-sha: ${{ inputs.duckdb-python-sha }}
55+
duckdb-sha: ${{ inputs.duckdb-sha }}
56+
secrets:
57+
# reusable workflows and secrets are not great: https://github.com/actions/runner/issues/3206
58+
DUCKDBLABS_BOT_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
59+
4460
workflow_state:
4561
name: Set state for the release workflow
4662
needs: build_sdist
@@ -51,23 +67,36 @@ jobs:
5167
runs-on: ubuntu-latest
5268
steps:
5369
- id: index_check
54-
name: Check ${{ needs.build_sdist.outputs.package-version }} on PyPI
70+
name: Check version on PyPI
5571
run: |
56-
set -eu
57-
# Check PyPI whether the release we're building is already present
72+
set -ex
5873
pypi_hostname=${{ inputs.pypi-index == 'test' && 'test.' || '' }}pypi.org
59-
pkg_version=${{ needs.build_sdist.outputs.package-version }}
60-
url=https://${pypi_hostname}/pypi/duckdb/${pkg_version}/json
61-
http_status=$( curl -s -o /dev/null -w "%{http_code}" $url || echo $? )
62-
if [[ $http_status == "200" ]]; then
63-
echo "::warning::Package version ${pkg_version} is already present on ${pypi_hostname}"
64-
pypi_state=VERSION_FOUND
65-
elif [[ $http_status == 000* ]]; then
66-
echo "::error::Error checking PyPI at ${url}: curl exit code ${http_status#'000'}"
67-
pypi_state=UNKNOWN
68-
else
69-
echo "::notice::Package version ${pkg_version} not found on ${pypi_hostname} (http status: ${http_status})"
74+
# install duckdb
75+
curl https://install.duckdb.org | sh
76+
# query pypi
77+
result=$(cat <<EOF | ${HOME}/.duckdb/cli/latest/duckdb | xargs
78+
---- Output lines
79+
.mode line
80+
---- Query that fetches the given version's age, if the version already exists
81+
SELECT
82+
today() - (file.value->>'upload_time_iso_8601')::DATE AS age,
83+
FROM read_json('https://${pypi_hostname}/pypi/duckdb/json') AS jd
84+
CROSS JOIN json_each(jd.releases) AS rel(key, value)
85+
CROSS JOIN unnest(FROM_JSON(rel.value, '["JSON"]')) AS file(value)
86+
WHERE rel.key='${{ needs.build_sdist.outputs.package-version }}'
87+
LIMIT 1;
88+
EOF
89+
)
90+
if [ -z "$result" ]; then
7091
pypi_state=VERSION_NOT_FOUND
92+
else
93+
pypi_state=VERSION_FOUND
94+
fi
95+
if [[ -z "${{ inputs.stable-version }}" ]]; then
96+
age=${result#age = }
97+
if [ "${age}" -ge "${{ inputs.nightly-stale-after-days }}" ]; then
98+
echo "::warning title=Stale nightly for ${{ github.ref_name }}::Nightly is ${age} days old (max=${{ inputs.nightly-stale-after-days }})"
99+
fi
71100
fi
72101
echo "pypi_state=${pypi_state}" >> $GITHUB_OUTPUT
73102
@@ -96,7 +125,7 @@ jobs:
96125
echo "::notice::S3 upload disabled in inputs, not generating S3 URL"
97126
exit 0
98127
fi
99-
if [[ VERSION_FOUND == "${{ steps.index_check.outputs.pypi_state }}" ]]; then
128+
if [[ VERSION_NOT_FOUND != "${{ steps.index_check.outputs.pypi_state }}" ]]; then
100129
echo "::warning::S3 upload disabled because package version already uploaded to PyPI"
101130
exit 0
102131
fi
@@ -110,7 +139,7 @@ jobs:
110139
build_wheels:
111140
name: Build and test releases
112141
needs: workflow_state
113-
if: ${{ needs.workflow_state.outputs.pypi_state != 'VERSION_FOUND' }}
142+
if: ${{ needs.workflow_state.outputs.pypi_state == 'VERSION_NOT_FOUND' }}
114143
uses: ./.github/workflows/packaging_wheels.yml
115144
with:
116145
minimal: false

‎.github/workflows/submodule_auto_pr.yml‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
name: Submodule Auto PR
2+
on:
3+
workflow_call:
4+
inputs:
5+
duckdb-python-sha:
6+
type: string
7+
description: The commit to build against (defaults to latest commit of current ref)
8+
required: false
9+
duckdb-sha:
10+
type: string
11+
description: The DuckDB submodule commit or ref to build against
12+
required: true
13+
auto-land:
14+
type: boolean
15+
description: Immediately merge the PR (placeholder - doesn't work)
16+
default: false
17+
secrets:
18+
DUCKDBLABS_BOT_TOKEN:
19+
description: Github token of the DuckDBLabs bot
20+
required: true
21+
22+
defaults:
23+
run:
24+
shell: bash
25+
26+
jobs:
27+
create_pr:
28+
name: Create PR to bump duckdb submodule to given SHA
29+
runs-on: ubuntu-latest
30+
steps:
31+
- name: Checkout DuckDB Python
32+
uses: actions/checkout@v4
33+
with:
34+
ref: ${{ inputs.duckdb-python-sha }}
35+
fetch-depth: 0
36+
submodules: true
37+
38+
- name: Checkout or Create Needed Branch
39+
run: |
40+
git fetch --all
41+
head_sha=${{ inputs.duckdb-python-sha }}
42+
branch_name="vendoring-${{ github.ref_name }}"
43+
if [[ `git rev-parse --verify ${branch_name} 2>/dev/null` ]]; then
44+
# branch exists
45+
git checkout ${branch_name}
46+
else
47+
# new branch
48+
git checkout -b ${branch_name}
49+
fi
50+
[[ ${head_sha} ]] && git reset --hard ${head_sha} || true
51+
52+
- name: Checkout DuckDB at Given SHA
53+
run: |
54+
cd external/duckdb
55+
git fetch origin
56+
git checkout ${{ inputs.duckdb-sha }}
57+
58+
- name: Determine GH PR Command
59+
id: gh_pr_command
60+
env:
61+
GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
62+
run: |
63+
pr_url=$( gh pr list --head vendoring-${{ github.ref_name }} --state open --json url --jq '.[].url' )
64+
if [[ $pr_url ]]; then
65+
echo "::notice::Found existing pr, will edit (${pr_url})"
66+
gh_command="edit ${pr_url}"
67+
else
68+
echo "::notice::No existing PR, will create new"
69+
gh_command="create --head vendoring-${{ github.ref_name }} --base ${{ github.ref_name }}"
70+
fi
71+
echo "subcommand=${gh_command}" >> $GITHUB_OUTPUT
72+
73+
- name: Set Git User
74+
run: |
75+
git config --global user.email "github_bot@duckdblabs.com"
76+
git config --global user.name "DuckDB Labs GitHub Bot"
77+
78+
- name: Create PR to Bump DuckDB Submodule
79+
env:
80+
GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
81+
run: |
82+
# No need to do anything if the submodule is already at the given sha
83+
[[ `git status --porcelain -- external/duckdb` == "" ]] && exit 0
84+
# We have changes. Commit and push
85+
git add external/duckdb
86+
git commit -m "Bump submodule"
87+
git push --force origin vendoring-${{ github.ref_name }}
88+
# create PR msg
89+
echo "Bump duckdb submodule:" > body.txt
90+
echo "- Target branch: ${{ github.ref_name }}" >> body.txt
91+
echo "- Date: $( date +"%Y-%m-%d %H:%M:%S" )" >> body.txt
92+
echo "- DuckDB SHA: ${{ inputs.duckdb-sha }}" >> body.txt
93+
echo "- Trigger: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> body.txt
94+
subcommand="${{ steps.gh_pr_command.outputs.subcommand }}"
95+
gh pr ${subcommand} \
96+
--title "[duckdb-labs bot] Bump DuckDB submodule" \
97+
--body-file body.txt > output.txt 2>&1
98+
success=$?
99+
# Show summary
100+
url=$( [[ $success ]] && gh pr view vendoring-${{ github.ref_name }} --json url --jq .url || true )
101+
echo "## Submodule PR Summary" >> $GITHUB_STEP_SUMMARY
102+
if [[ $success ]]; then
103+
prefix=$( [[ $subcommand == edit* ]] && echo "Created" || echo "Updated" )
104+
echo "### ${prefix} PR: [${url}](${url})" >> $GITHUB_STEP_SUMMARY
105+
else
106+
echo "### Failed to create PR" >> $GITHUB_STEP_SUMMARY
107+
fi
108+
echo '```' >> $GITHUB_STEP_SUMMARY
109+
cat output.txt >> $GITHUB_STEP_SUMMARY
110+
echo '```' >> $GITHUB_STEP_SUMMARY
111+
[[ $success ]] || exit 1
112+
113+
- name: Automerge PR
114+
if: ${{ inputs.auto-land }}
115+
env:
116+
GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
117+
run: |
118+
# PLACEHOLDER: DUCKDBLABS_BOT_TOKEN DOES NOT HAVE PERMISSIONS TO MERGE PRS
119+
set -ex
120+
gh pr merge vendoring-${{ github.ref_name }} --rebase > output.txt
121+
success=$?
122+
# Show summary
123+
if [[ $success ]]; then
124+
echo "### PR merged" >> $GITHUB_STEP_SUMMARY
125+
else
126+
echo "### Failed to auto-merge PR" >> $GITHUB_STEP_SUMMARY
127+
fi
128+
echo '```' >> $GITHUB_STEP_SUMMARY
129+
cat output.txt >> $GITHUB_STEP_SUMMARY
130+
echo '```' >> $GITHUB_STEP_SUMMARY

‎.github/workflows/targeted_test.yml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ on:
99
type: choice
1010
options:
1111
- 'windows-2025'
12+
- 'windows-11-arm'
1213
- 'ubuntu-24.04'
1314
- 'ubuntu-24.04-arm'
1415
- 'macos-15'
@@ -36,6 +37,11 @@ on:
3637
description: 'Custom test path (must be in tests/ directory, overrides testsuite)'
3738
required: false
3839
type: string
40+
verbose-uv:
41+
description: 'Let uv generate verbose output (pytest verbosity is always on)'
42+
required: false
43+
type: boolean
44+
default: true
3945

4046
jobs:
4147
test:
@@ -83,4 +89,4 @@ jobs:
8389
- name: Run tests
8490
shell: bash
8591
run: |
86-
uv run pytest -vv ${{ steps.test_path.outputs.test_path }}
92+
uv ${{ inputs.verbose-uv && 'run -v' || 'run' }} pytest -vv ${{ steps.test_path.outputs.test_path }}

‎CHANGELOG.md‎

Lines changed: 0 additions & 19 deletions
This file was deleted.

‎_duckdb-stubs/__init__.pyi‎

Lines changed: 41 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -318,7 +318,18 @@ class DuckDBPyConnection:
318318
def list_type(self, type: sqltypes.DuckDBPyType) -> sqltypes.DuckDBPyType: ...
319319
def load_extension(self, extension: str) -> None: ...
320320
def map_type(self, key: sqltypes.DuckDBPyType, value: sqltypes.DuckDBPyType) -> sqltypes.DuckDBPyType: ...
321-
def pl(self, rows_per_batch: pytyping.SupportsInt = 1000000, *, lazy: bool = False) -> polars.DataFrame: ...
321+
@pytyping.overload
322+
def pl(
323+
self, rows_per_batch: pytyping.SupportsInt = 1000000, *, lazy: pytyping.Literal[False] = ...
324+
) -> polars.DataFrame: ...
325+
@pytyping.overload
326+
def pl(
327+
self, rows_per_batch: pytyping.SupportsInt = 1000000, *, lazy: pytyping.Literal[True]
328+
) -> polars.LazyFrame: ...
329+
@pytyping.overload
330+
def pl(
331+
self, rows_per_batch: pytyping.SupportsInt = 1000000, *, lazy: bool = False
332+
) -> pytyping.Union[polars.DataFrame, polars.LazyFrame]: ...
322333
def query(self, query: str, *, alias: str = "", params: object = None) -> DuckDBPyRelation: ...
323334
def query_progress(self) -> float: ...
324335
def read_csv(
@@ -596,7 +607,16 @@ class DuckDBPyRelation:
596607
) -> DuckDBPyRelation: ...
597608
def order(self, order_expr: str) -> DuckDBPyRelation: ...
598609
def percent_rank(self, window_spec: str, projected_columns: str = "") -> DuckDBPyRelation: ...
599-
def pl(self, batch_size: pytyping.SupportsInt = 1000000, *, lazy: bool = False) -> polars.DataFrame: ...
610+
@pytyping.overload
611+
def pl(
612+
self, batch_size: pytyping.SupportsInt = 1000000, *, lazy: pytyping.Literal[False] = ...
613+
) -> polars.DataFrame: ...
614+
@pytyping.overload
615+
def pl(self, batch_size: pytyping.SupportsInt = 1000000, *, lazy: pytyping.Literal[True]) -> polars.LazyFrame: ...
616+
@pytyping.overload
617+
def pl(
618+
self, batch_size: pytyping.SupportsInt = 1000000, *, lazy: bool = False
619+
) -> pytyping.Union[polars.DataFrame, polars.LazyFrame]: ...
600620
def product(
601621
self, column: str, groups: str = "", window_spec: str = "", projected_columns: str = ""
602622
) -> DuckDBPyRelation: ...
@@ -700,6 +720,7 @@ class DuckDBPyRelation:
700720
partition_by: pytyping.List[str] | None = None,
701721
write_partition_columns: bool | None = None,
702722
append: bool | None = None,
723+
filename_pattern: str | None = None,
703724
) -> None: ...
704725
def to_table(self, table_name: str) -> None: ...
705726
def to_view(self, view_name: str, replace: bool = True) -> DuckDBPyRelation: ...
@@ -752,6 +773,7 @@ class DuckDBPyRelation:
752773
partition_by: pytyping.List[str] | None = None,
753774
write_partition_columns: bool | None = None,
754775
append: bool | None = None,
776+
filename_pattern: str | None = None,
755777
) -> None: ...
756778
@property
757779
def alias(self) -> str: ...
@@ -1048,7 +1070,7 @@ def commit(*, connection: DuckDBPyConnection | None = None) -> DuckDBPyConnectio
10481070
def connect(
10491071
database: str | pathlib.Path = ":memory:",
10501072
read_only: bool = False,
1051-
config: dict[str, str] | None = None,
1073+
config: dict[str, str | bool | int | float | list[str]] | None = None,
10521074
) -> DuckDBPyConnection: ...
10531075
def create_function(
10541076
name: str,
@@ -1241,12 +1263,27 @@ def map_type(
12411263
def order(
12421264
df: pandas.DataFrame, order_expr: str, *, connection: DuckDBPyConnection | None = None
12431265
) -> DuckDBPyRelation: ...
1266+
@pytyping.overload
12441267
def pl(
12451268
rows_per_batch: pytyping.SupportsInt = 1000000,
12461269
*,
1247-
lazy: bool = False,
1270+
lazy: pytyping.Literal[False] = ...,
12481271
connection: DuckDBPyConnection | None = None,
12491272
) -> polars.DataFrame: ...
1273+
@pytyping.overload
1274+
def pl(
1275+
rows_per_batch: pytyping.SupportsInt = 1000000,
1276+
*,
1277+
lazy: pytyping.Literal[True],
1278+
connection: DuckDBPyConnection | None = None,
1279+
) -> polars.LazyFrame: ...
1280+
@pytyping.overload
1281+
def pl(
1282+
rows_per_batch: pytyping.SupportsInt = 1000000,
1283+
*,
1284+
lazy: bool = False,
1285+
connection: DuckDBPyConnection | None = None,
1286+
) -> pytyping.Union[polars.DataFrame, polars.LazyFrame]: ...
12501287
def project(
12511288
df: pandas.DataFrame, *args: str | Expression, groups: str = "", connection: DuckDBPyConnection | None = None
12521289
) -> DuckDBPyRelation: ...

‎duckdb/experimental/__init__.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
11
from . import spark # noqa: D104
22

3-
__all__ = spark.__all__
3+
__all__ = [
4+
"spark",
5+
]

0 commit comments

Comments
 (0)