Merge branch 'main' into fill-null

Author: kosiew

Cargo.lock

@@ -34,25 +34,25 @@ protoc = [ "datafusion-substrait/protoc" ]
 substrait = ["dep:datafusion-substrait"]
 
 [dependencies]
-tokio = { version = "1.43", features = ["macros", "rt", "rt-multi-thread", "sync"] }
-pyo3 = { version = "0.23", features = ["extension-module", "abi3", "abi3-py39"] }
-pyo3-async-runtimes = { version = "0.23", features = ["tokio-runtime"]}
-arrow = { version = "54.2.1", features = ["pyarrow"] }
-datafusion = { version = "46.0.1", features = ["avro", "unicode_expressions"] }
-datafusion-substrait = { version = "46.0.1", optional = true }
-datafusion-proto = { version = "46.0.1" }
-datafusion-ffi = { version = "46.0.1" }
+tokio = { version = "1.44", features = ["macros", "rt", "rt-multi-thread", "sync"] }
+pyo3 = { version = "0.24", features = ["extension-module", "abi3", "abi3-py39"] }
+pyo3-async-runtimes = { version = "0.24", features = ["tokio-runtime"]}
+arrow = { version = "55.0.0", features = ["pyarrow"] }
+datafusion = { version = "47.0.0", features = ["avro", "unicode_expressions"] }
+datafusion-substrait = { version = "47.0.0", optional = true }
+datafusion-proto = { version = "47.0.0" }
+datafusion-ffi = { version = "47.0.0" }
 prost = "0.13.1" # keep in line with `datafusion-substrait`
-uuid = { version = "1.12", features = ["v4"] }
+uuid = { version = "1.16", features = ["v4"] }
 mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] }
-async-trait = "0.1.73"
+async-trait = "0.1.88"
 futures = "0.3"
-object_store = { version = "0.11.0", features = ["aws", "gcp", "azure", "http"] }
+object_store = { version = "0.12.0", features = ["aws", "gcp", "azure", "http"] }
 url = "2"
 
 [build-dependencies]
 prost-types = "0.13.1" # keep in line with `datafusion-substrait`
-pyo3-build-config = "0.23"
+pyo3-build-config = "0.24"
 
 [lib]
 name = "datafusion_python"

Cargo.toml

@@ -72,6 +72,7 @@ Example
    user-guide/introduction
    user-guide/basics
    user-guide/data-sources
+   user-guide/dataframe
    user-guide/common-operations/index
    user-guide/io/index
    user-guide/configuration

docs/source/index.rst

@@ -21,7 +21,8 @@ Concepts
 ========
 
 In this section, we will cover a basic example to introduce a few key concepts. We will use the
-2021 Yellow Taxi Trip Records ([download](https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-01.parquet)), from the [TLC Trip Record Data](https://www.nyc.gov/site/tlc/about/tlc-trip-record-data.page).
+2021 Yellow Taxi Trip Records (`download <https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-01.parquet>`_),
+from the `TLC Trip Record Data <https://www.nyc.gov/site/tlc/about/tlc-trip-record-data.page>`_.
 
 .. ipython:: python
 
@@ -72,6 +73,8 @@ DataFrames are typically created by calling a method on :py:class:`~datafusion.c
 calling the transformation methods, such as :py:func:`~datafusion.dataframe.DataFrame.filter`, :py:func:`~datafusion.dataframe.DataFrame.select`, :py:func:`~datafusion.dataframe.DataFrame.aggregate`,
 and :py:func:`~datafusion.dataframe.DataFrame.limit` to build up a query definition.
 
+For more details on working with DataFrames, including visualization options and conversion to other formats, see :doc:`dataframe`.
+
 Expressions
 -----------
 

docs/source/user-guide/basics.rst

@@ -0,0 +1,179 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+DataFrames
+==========
+
+Overview
+--------
+
+DataFusion's DataFrame API provides a powerful interface for building and executing queries against data sources. 
+It offers a familiar API similar to pandas and other DataFrame libraries, but with the performance benefits of Rust 
+and Arrow.
+
+A DataFrame represents a logical plan that can be composed through operations like filtering, projection, and aggregation.
+The actual execution happens when terminal operations like ``collect()`` or ``show()`` are called.
+
+Basic Usage
+-----------
+
+.. code-block:: python
+
+    import datafusion
+    from datafusion import col, lit
+
+    # Create a context and register a data source
+    ctx = datafusion.SessionContext()
+    ctx.register_csv("my_table", "path/to/data.csv")
+    
+    # Create and manipulate a DataFrame
+    df = ctx.sql("SELECT * FROM my_table")
+    
+    # Or use the DataFrame API directly
+    df = (ctx.table("my_table")
+          .filter(col("age") > lit(25))
+          .select([col("name"), col("age")]))
+    
+    # Execute and collect results
+    result = df.collect()
+    
+    # Display the first few rows
+    df.show()
+
+HTML Rendering
+--------------
+
+When working in Jupyter notebooks or other environments that support HTML rendering, DataFrames will
+automatically display as formatted HTML tables, making it easier to visualize your data.
+
+The ``_repr_html_`` method is called automatically by Jupyter to render a DataFrame. This method 
+controls how DataFrames appear in notebook environments, providing a richer visualization than
+plain text output.
+
+Customizing HTML Rendering
+--------------------------
+
+You can customize how DataFrames are rendered in HTML by configuring the formatter:
+
+.. code-block:: python
+
+    from datafusion.html_formatter import configure_formatter
+    
+    # Change the default styling
+    configure_formatter(
+        max_rows=50,           # Maximum number of rows to display
+        max_width=None,        # Maximum width in pixels (None for auto)
+        theme="light",         # Theme: "light" or "dark" 
+        precision=2,           # Floating point precision
+        thousands_separator=",", # Separator for thousands
+        date_format="%Y-%m-%d", # Date format
+        truncate_width=20      # Max width for string columns before truncating
+    )
+
+The formatter settings affect all DataFrames displayed after configuration.
+
+Custom Style Providers
+----------------------
+
+For advanced styling needs, you can create a custom style provider:
+
+.. code-block:: python
+
+    from datafusion.html_formatter import StyleProvider, configure_formatter
+    
+    class MyStyleProvider(StyleProvider):
+        def get_table_styles(self):
+            return {
+                "table": "border-collapse: collapse; width: 100%;",
+                "th": "background-color: #007bff; color: white; padding: 8px; text-align: left;",
+                "td": "border: 1px solid #ddd; padding: 8px;",
+                "tr:nth-child(even)": "background-color: #f2f2f2;",
+            }
+            
+        def get_value_styles(self, dtype, value):
+            """Return custom styles for specific values"""
+            if dtype == "float" and value < 0:
+                return "color: red;"
+            return None
+    
+    # Apply the custom style provider
+    configure_formatter(style_provider=MyStyleProvider())
+
+Creating a Custom Formatter
+---------------------------
+
+For complete control over rendering, you can implement a custom formatter:
+
+.. code-block:: python
+
+    from datafusion.html_formatter import Formatter, get_formatter
+    
+    class MyFormatter(Formatter):
+        def format_html(self, batches, schema, has_more=False, table_uuid=None):
+            # Create your custom HTML here
+            html = "<div class='my-custom-table'>"
+            # ... formatting logic ...
+            html += "</div>"
+            return html
+    
+    # Set as the global formatter
+    configure_formatter(formatter_class=MyFormatter)
+    
+    # Or use the formatter just for specific operations
+    formatter = get_formatter()
+    custom_html = formatter.format_html(batches, schema)
+
+Managing Formatters
+-------------------
+
+Reset to default formatting:
+
+.. code-block:: python
+
+    from datafusion.html_formatter import reset_formatter
+    
+    # Reset to default settings
+    reset_formatter()
+
+Get the current formatter settings:
+
+.. code-block:: python
+
+    from datafusion.html_formatter import get_formatter
+    
+    formatter = get_formatter()
+    print(formatter.max_rows)
+    print(formatter.theme)
+
+Contextual Formatting
+---------------------
+
+You can also use a context manager to temporarily change formatting settings:
+
+.. code-block:: python
+
+    from datafusion.html_formatter import formatting_context
+    
+    # Default formatting
+    df.show()
+    
+    # Temporarily use different formatting
+    with formatting_context(max_rows=100, theme="dark"):
+        df.show()  # Will use the temporary settings
+    
+    # Back to default formatting
+    df.show()

docs/source/user-guide/dataframe.rst

@@ -26,6 +26,8 @@
 except ImportError:
     import importlib_metadata
 
+from datafusion.col import col, column
+
 from . import functions, object_store, substrait, unparser
 
 # The following imports are okay to remain as opaque to the user.
@@ -45,6 +47,7 @@
     Expr,
     WindowFrame,
 )
+from .html_formatter import configure_formatter
 from .io import read_avro, read_csv, read_json, read_parquet
 from .plan import ExecutionPlan, LogicalPlan
 from .record_batch import RecordBatch, RecordBatchStream
@@ -76,6 +79,7 @@
     "col",
     "column",
     "common",
+    "configure_formatter",
     "expr",
     "functions",
     "lit",
@@ -93,16 +97,6 @@
 ]
 
 
-def column(value: str) -> Expr:
-    """Create a column expression."""
-    return Expr.column(value)
-
-
-def col(value: str) -> Expr:
-    """Create a column expression."""
-    return Expr.column(value)
-
-
 def literal(value) -> Expr:
     """Create a literal expression."""
     return Expr.literal(value)

python/datafusion/__init__.py

@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Col class."""
+
+from datafusion.expr import Expr
+
+
+class Col:
+    """Create a column expression.
+
+    This helper class allows an extra syntax of creating columns using the __getattr__
+    method.
+    """
+
+    def __call__(self, value: str) -> Expr:
+        """Create a column expression."""
+        return Expr.column(value)
+
+    def __getattr__(self, value: str) -> Expr:
+        """Create a column using attribute syntax."""
+        # For autocomplete to work with IPython
+        if value.startswith("__wrapped__"):
+            return getattr(type(self), value)
+
+        return Expr.column(value)
+
+
+col: Col = Col()
+column: Col = Col()
+__all__ = ["col", "column"]