Merge branch 'main' into fix-DuckDBPyRelation.aggregate-incorrect-typing

Author: evertlammerts

.github/workflows/InternalIssuesCreateMirror.yml

@@ -0,0 +1,82 @@
+name: Create or Label Mirror Issue
+on:
+  discussion:
+    types:
+      - labeled
+  issues:
+    types:
+      - labeled
+
+env:
+  GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
+  TITLE_PREFIX: "[duckdb-python/#${{ github.event.issue.number || github.event.discussion.number }}]"
+  PUBLIC_ISSUE_TITLE: ${{ github.event.issue.title }}
+
+jobs:
+  add_needs_reproducible_example_comment:
+    if: github.event.label.name == 'needs reproducible example'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Add comment
+        run: |
+          cat > needs-reproducible-example-comment.md << EOF
+          Thanks for opening this issue in the DuckDB-Python project's issue tracker! To resolve this issue, our team needs a reproducible example. This includes:
+
+          * A source code snippet which reproduces the issue.
+          * The snippet should be self-contained, i.e., it should contain all imports and should use relative paths instead of hard coded paths (please avoid \`/Users/JohnDoe/...\`).
+          * A lot of issues can be reproduced with plain SQL code executed in the [DuckDB command line client](https://duckdb.org/docs/api/cli/overview). For such issues, please open an issue in the [main DuckDB repository](https://github.com/duckdb/duckdb/).
+          * If the script needs additional data, please share the data as a CSV, JSON, or Parquet file. Unfortunately, we cannot fix issues that can only be reproduced with a confidential data set. [Support contracts](https://duckdblabs.com/#support) allow sharing confidential data with the core DuckDB team under NDA.
+
+          For more detailed guidelines on how to create reproducible examples, please visit Stack Overflow's [“Minimal, Reproducible Example”](https://stackoverflow.com/help/minimal-reproducible-example) page.
+          EOF
+          gh issue comment --repo duckdb/duckdb-python ${{ github.event.issue.number || github.event.discussion.number }} --body-file needs-reproducible-example-comment.md
+
+  create_or_label_mirror_issue:
+    if: github.event.label.name == 'reproduced' || github.event.label.name == 'under review'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Remove 'needs triage' / 'under review' if 'reproduced'
+        if: github.event.label.name == 'reproduced'
+        run: |
+          gh issue edit --repo duckdb/duckdb-python ${{ github.event.issue.number || github.event.discussion.number }} --remove-label "needs triage" --remove-label "under review" --remove-label "needs reproducible example"
+
+      - name: Remove 'needs triage' / 'reproduced' if 'under review'
+        if: github.event.label.name == 'under review'
+        run: |
+          gh issue edit --repo duckdb/duckdb-python ${{ github.event.issue.number || github.event.discussion.number }} --remove-label "needs triage" --remove-label "reproduced"
+
+      - name: Remove 'needs triage' if 'expected behavior'
+        if: github.event.label.name == 'expected behavior'
+        run: |
+          gh issue edit --repo duckdb/duckdb-python ${{ github.event.issue.number || github.event.discussion.number }} --remove-label "needs triage"
+
+      - name: Get mirror issue number
+        run: |
+          gh issue list --repo duckdblabs/duckdb-internal --search "${TITLE_PREFIX}" --json title,number --state all --jq ".[] | select(.title | startswith(\"$TITLE_PREFIX\")).number" > mirror_issue_number.txt
+          echo "MIRROR_ISSUE_NUMBER=$(cat mirror_issue_number.txt)" >> $GITHUB_ENV
+
+      - name: Print whether mirror issue exists
+        run: |
+          if [ "$MIRROR_ISSUE_NUMBER" == "" ]; then
+            echo "Mirror issue with title prefix '$TITLE_PREFIX' does not exist yet"
+          else
+            echo "Mirror issue with title prefix '$TITLE_PREFIX' exists with number $MIRROR_ISSUE_NUMBER"
+          fi
+
+      - name: Set label environment variable
+        run: |
+          if ${{ github.event.label.name == 'reproduced' }}; then
+            echo "LABEL=reproduced" >> $GITHUB_ENV
+            echo "UNLABEL=under review" >> $GITHUB_ENV
+          else
+            echo "LABEL=under review" >> $GITHUB_ENV
+            echo "UNLABEL=reproduced" >> $GITHUB_ENV
+          fi
+
+      - name: Create or label issue
+        run: |
+          if [ "$MIRROR_ISSUE_NUMBER" == "" ]; then
+            gh issue create --repo duckdblabs/duckdb-internal --label "$LABEL" --label "Python" --title "$TITLE_PREFIX - $PUBLIC_ISSUE_TITLE" --body "See https://github.com/duckdb/duckdb-python/issues/${{ github.event.issue.number || github.event.discussion.number }}"
+          else
+            gh issue edit --repo duckdblabs/duckdb-internal $MIRROR_ISSUE_NUMBER --remove-label "$UNLABEL" --add-label "$LABEL"
+          fi

.github/workflows/InternalIssuesUpdateMirror.yml

@@ -0,0 +1,51 @@
+name: Update Mirror Issue
+on:
+  discussion:
+    types:
+      - labeled
+  issues:
+    types:
+      - closed
+      - reopened
+
+env:
+  GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
+  TITLE_PREFIX: "[duckdb-python/#${{ github.event.issue.number || github.event.discussion.number }}]"
+
+jobs:
+  update_mirror_issue:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get mirror issue number
+        run: |
+          gh issue list --repo duckdblabs/duckdb-internal --search "${TITLE_PREFIX}" --json title,number --state all --jq ".[] | select(.title | startswith(\"$TITLE_PREFIX\")).number" > mirror_issue_number.txt
+          echo "MIRROR_ISSUE_NUMBER=$(cat mirror_issue_number.txt)" >> $GITHUB_ENV
+
+      - name: Print whether mirror issue exists
+        run: |
+          if [ "$MIRROR_ISSUE_NUMBER" == "" ]; then
+            echo "Mirror issue with title prefix '$TITLE_PREFIX' does not exist yet"
+          else
+            echo "Mirror issue with title prefix '$TITLE_PREFIX' exists with number $MIRROR_ISSUE_NUMBER"
+          fi
+
+      - name: Add comment with status to mirror issue
+        run: |
+          if [ "$MIRROR_ISSUE_NUMBER" != "" ]; then
+            gh issue comment --repo duckdblabs/duckdb-internal $MIRROR_ISSUE_NUMBER --body "The issue has been ${{ github.event.action }} (https://github.com/duckdb/duckdb-python/issues/${{ github.event.issue.number || github.event.discussion.number }})."
+          fi
+
+      - name: Add closed label to mirror issue
+        if: github.event.action == 'closed'
+        run: |
+          if [ "$MIRROR_ISSUE_NUMBER" != "" ]; then
+            gh issue edit --repo duckdblabs/duckdb-internal $MIRROR_ISSUE_NUMBER --add-label "public closed" --remove-label "public reopened"
+          fi
+
+      - name: Reopen mirror issue and add reopened label
+        if: github.event.action == 'reopened'
+        run: |
+          if [ "$MIRROR_ISSUE_NUMBER" != "" ]; then
+            gh issue reopen --repo duckdblabs/duckdb-internal $MIRROR_ISSUE_NUMBER
+            gh issue edit --repo duckdblabs/duckdb-internal $MIRROR_ISSUE_NUMBER --add-label "public reopened" --remove-label "public closed"
+          fi

CMakeLists.txt

@@ -6,6 +6,7 @@ project(duckdb_py LANGUAGES CXX)
 set(CMAKE_CXX_STANDARD 11)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
 
 # Set the library name
 set(DUCKDB_PYTHON_LIB_NAME "_duckdb")
@@ -78,6 +79,7 @@ pybind11_add_module(
   $<TARGET_OBJECTS:python_type>)
 # add _duckdb_dependencies
 target_link_libraries(_duckdb PRIVATE _duckdb_dependencies)
+duckdb_link_extensions(_duckdb)
 
 # ────────────────────────────────────────────
 # Controlling symbol export

cmake/duckdb_loader.cmake

@@ -7,7 +7,7 @@
 # debugging but will never allow jemalloc in a release build if not on Linux.
 #
 # Usage: include(cmake/duckdb_loader.cmake) # Optionally load extensions
-# set(CORE_EXTENSIONS "json;parquet;icu")
+# set(BUILD_EXTENSIONS "json;parquet;icu")
 #
 # # set sensible defaults for a debug build: duckdb_configure_for_debug()
 #
@@ -35,7 +35,7 @@ _duckdb_set_default(DUCKDB_SOURCE_PATH
                     "${CMAKE_CURRENT_SOURCE_DIR}/external/duckdb")
 
 # Extension list - commonly used extensions for Python
-_duckdb_set_default(CORE_EXTENSIONS "core_functions;parquet;icu;json")
+_duckdb_set_default(BUILD_EXTENSIONS "core_functions;parquet;icu;json")
 
 # Core build options - disable unnecessary components for Python builds
 _duckdb_set_default(BUILD_SHELL OFF)
@@ -64,8 +64,8 @@ _duckdb_set_default(DEBUG_STACKTRACE OFF)
 set(DUCKDB_SOURCE_PATH
     "${DUCKDB_SOURCE_PATH}"
     CACHE PATH "Path to DuckDB source directory")
-set(CORE_EXTENSIONS
-    "${CORE_EXTENSIONS}"
+set(BUILD_EXTENSIONS
+    "${BUILD_EXTENSIONS}"
     CACHE STRING "Semicolon-separated list of extensions to enable")
 set(BUILD_SHELL
     "${BUILD_SHELL}"
@@ -110,40 +110,32 @@ set(DEBUG_STACKTRACE
 
 function(_duckdb_validate_jemalloc_config)
   # Check if jemalloc is in the extension list
-  if(NOT CORE_EXTENSIONS MATCHES "jemalloc")
+  if(NOT BUILD_EXTENSIONS MATCHES "jemalloc")
     return()
   endif()
 
-  # If we're on Linux then using jemalloc is fine, otherwise we only allow it in
-  # debug builds
-  if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux")
-    set(is_debug_build FALSE)
-    if(CMAKE_BUILD_TYPE STREQUAL "Debug")
-      set(is_debug_build TRUE)
-    endif()
-    if(is_debug_build)
-      message(
-        WARNING
-          "jemalloc extension enabled on ${CMAKE_SYSTEM_NAME} in Debug build.\n"
-          "This is only recommended for debugging purposes.\n"
-          "jemalloc is officially supported only on Linux.")
-    else()
-      message(
-        WARNING
-          "jemalloc extension is only supported on ${CMAKE_SYSTEM_NAME} in Debug builds.\n"
-          "Removing jemalloc from extension list.\n"
-          "In non-debug builds, jemalloc is only supported on Linux.")
-      # Remove jemalloc from the extension list
-      string(REPLACE "jemalloc" "" CORE_EXTENSIONS_FILTERED
-                     "${CORE_EXTENSIONS}")
-      string(REGEX REPLACE ";+" ";" CORE_EXTENSIONS_FILTERED
-                           "${CORE_EXTENSIONS_FILTERED}")
-      string(REGEX REPLACE "^;|;$" "" CORE_EXTENSIONS_FILTERED
-                           "${CORE_EXTENSIONS_FILTERED}")
-      set(CORE_EXTENSIONS
-          "${CORE_EXTENSIONS_FILTERED}"
-          PARENT_SCOPE)
-    endif()
+  # jemalloc is only enabled on 64bit x86 linux builds
+  if(CMAKE_SIZEOF_VOID_P EQUAL 8
+     AND CMAKE_SYSTEM_NAME STREQUAL "Linux"
+     AND NOT BSD)
+    set(jemalloc_allowed TRUE)
+  else()
+    set(jemalloc_allowed FALSE)
+  endif()
+
+  if(NOT jemalloc_allowed)
+    message(WARNING "jemalloc extension is only supported on Linux.\n"
+                    "Removing jemalloc from extension list.")
+    # Remove jemalloc from the extension list
+    string(REPLACE "jemalloc" "" BUILD_EXTENSIONS_FILTERED
+                   "${BUILD_EXTENSIONS}")
+    string(REGEX REPLACE ";+" ";" BUILD_EXTENSIONS_FILTERED
+                         "${BUILD_EXTENSIONS_FILTERED}")
+    string(REGEX REPLACE "^;|;$" "" BUILD_EXTENSIONS_FILTERED
+                         "${BUILD_EXTENSIONS_FILTERED}")
+    set(BUILD_EXTENSIONS
+        "${BUILD_EXTENSIONS_FILTERED}"
+        PARENT_SCOPE)
   endif()
 endfunction()
 
@@ -188,19 +180,17 @@ function(_duckdb_create_interface_target target_name)
   if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
     target_compile_options(
       ${target_name}
-      INTERFACE
-        /wd4244 # suppress Conversion from 'type1' to 'type2', possible loss of
-                # data
-        /wd4267 # suppress Conversion from ‘size_t’ to ‘type’, possible loss of
-                # data
-        /wd4200 # suppress Nonstandard extension used: zero-sized array in
-                # struct/union
-        /wd26451
-        /wd26495 # suppress Code Analysis
-        /D_CRT_SECURE_NO_WARNINGS # suppress warnings about unsafe functions
-        /D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR # see
-                                               # https://github.com/duckdblabs/duckdb-internal/issues/5151
-        /utf-8 # treat source files as UTF-8 encoded
+      INTERFACE /wd4244 # suppress Conversion from 'type1' to 'type2', possible
+                        # loss of data
+                /wd4267 # suppress Conversion from ‘size_t’ to ‘type’, possible
+                        # loss of data
+                /wd4200 # suppress Nonstandard extension used: zero-sized array
+                        # in struct/union
+                /wd26451
+                /wd26495 # suppress Code Analysis
+                /D_CRT_SECURE_NO_WARNINGS # suppress warnings about unsafe
+                                          # functions
+                /utf-8 # treat source files as UTF-8 encoded
     )
   elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
     target_compile_options(
@@ -225,10 +215,6 @@ function(_duckdb_print_summary)
   message(STATUS "  Native Arch: ${NATIVE_ARCH}")
   message(STATUS "  Unity Build Disabled: ${DISABLE_UNITY}")
 
-  if(CORE_EXTENSIONS)
-    message(STATUS "  Extensions: ${CORE_EXTENSIONS}")
-  endif()
-
   set(debug_opts)
   if(FORCE_ASSERT)
     list(APPEND debug_opts "FORCE_ASSERT")
@@ -256,6 +242,26 @@ function(duckdb_add_library target_name)
 
   # Create clean interface target
   _duckdb_create_interface_target(${target_name})
+
+  # Propagate BUILD_EXTENSIONS back to caller scope in case it was modified
+  set(BUILD_EXTENSIONS
+      "${BUILD_EXTENSIONS}"
+      PARENT_SCOPE)
+endfunction()
+
+function(duckdb_link_extensions target_name)
+  # Link to the DuckDB static library and extensions
+  target_link_libraries(${target_name}
+                        PRIVATE duckdb_generated_extension_loader)
+  if(BUILD_EXTENSIONS)
+    message(STATUS "Linking DuckDB extensions:")
+    foreach(ext IN LISTS BUILD_EXTENSIONS)
+      message(STATUS "- ${ext}")
+      target_link_libraries(${target_name} PRIVATE ${ext}_extension)
+    endforeach()
+  else()
+    message(STATUS "No DuckDB extensions linked in")
+  endif()
 endfunction()
 
 # ════════════════════════════════════════════════════════════════════════════════

duckdb_packaging/setuptools_scm_version.py

@@ -12,7 +12,7 @@
 from ._versioning import format_version, parse_version
 
 # MAIN_BRANCH_VERSIONING should be 'True' on main branch only
-MAIN_BRANCH_VERSIONING = False
+MAIN_BRANCH_VERSIONING = True
 
 SCM_PRETEND_ENV_VAR = "SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DUCKDB"
 SCM_GLOBAL_PRETEND_ENV_VAR = "SETUPTOOLS_SCM_PRETEND_VERSION"

external/duckdb

@@ -82,7 +82,7 @@ packages.adbc_driver_duckdb = "adbc_driver_duckdb"
 packages._duckdb-stubs = "_duckdb-stubs"
 
 [tool.scikit-build.cmake.define]
-CORE_EXTENSIONS = "core_functions;json;parquet;icu;jemalloc"
+BUILD_EXTENSIONS = "core_functions;json;parquet;icu;jemalloc"
 
 [tool.setuptools_scm]
 version_scheme = "duckdb_packaging.setuptools_scm_version:version_scheme"