Move ffi-table-provider example to ffi-library and correct doc building

Author: timsaucer

.github/workflows/test.yaml

@@ -91,7 +91,7 @@ jobs:
 
       - name: FFI unit tests
         run: |
-          cd examples/ffi-table-provider
+          cd examples/ffi-library
           uv run --no-project maturin develop --uv
           uv run --no-project pytest python/tests/_test_table_provider.py
 

docs/source/conf.py

@@ -71,6 +71,7 @@
 autoapi_member_order = "groupwise"
 suppress_warnings = ["autoapi.python_import_resolution"]
 autoapi_python_class_content = "both"
+autoapi_keep_files = False  # Set to True for debugging documentation generated
 
 
 def autoapi_skip_member_fn(app, what, name, obj, skip, options) -> bool:  # noqa: ARG001

docs/source/contributor-guide/ffi.rst

@@ -176,7 +176,7 @@ By convention the ``datafusion-python`` library expects a Python object that has
 ``TableProvider`` PyCapsule to have this capsule accessible by calling a function named
 ``__datafusion_table_provider__``. You can see a complete working example of how to
 share a ``TableProvider`` from one python library to DataFusion Python in the
-`repository examples folder <https://github.com/apache/datafusion-python/tree/main/examples/ffi-table-provider>`_.
+`repository examples folder <https://github.com/apache/datafusion-python/tree/main/examples/ffi-library>`_.
 
 This section has been written using ``TableProvider`` as an example. It is the first
 extension that has been written using this approach and the most thoroughly implemented.

examples/ffi-table-provider/.cargo/config.toml renamed to examples/ffi-library/.cargo/config.toml

@@ -16,7 +16,7 @@
 # under the License.
 
 [package]
-name = "ffi-table-provider"
+name = "ffi-library"
 version = "0.1.0"
 edition = "2021"
 

examples/ffi-table-provider/Cargo.lock renamed to examples/ffi-library/Cargo.lock

@@ -22,7 +22,7 @@
 from datafusion import col, lit, udwf
 from datafusion import functions as f
 from datafusion.expr import WindowFrame
-from datafusion.udf import WindowEvaluator
+from datafusion.user_defined import WindowEvaluator
 
 # This example creates five different examples of user defined window functions in order
 # to demonstrate the variety of ways a user may need to implement.

examples/ffi-table-provider/Cargo.toml renamed to examples/ffi-library/Cargo.toml

@@ -48,7 +48,15 @@
 from .io import read_avro, read_csv, read_json, read_parquet
 from .plan import ExecutionPlan, LogicalPlan
 from .record_batch import RecordBatch, RecordBatchStream
-from .udf import Accumulator, AggregateUDF, ScalarUDF, WindowUDF, udaf, udf, udwf
+from .user_defined import (
+    Accumulator,
+    AggregateUDF,
+    ScalarUDF,
+    WindowUDF,
+    udaf,
+    udf,
+    udwf,
+)
 
 __version__ = importlib_metadata.version(__name__)
 

examples/ffi-table-provider/build.rs renamed to examples/ffi-library/build.rs

@@ -30,7 +30,7 @@
 from datafusion.dataframe import DataFrame
 from datafusion.expr import Expr, SortExpr, sort_list_to_raw_sort_list
 from datafusion.record_batch import RecordBatchStream
-from datafusion.udf import AggregateUDF, ScalarUDF, WindowUDF
+from datafusion.user_defined import AggregateUDF, ScalarUDF, WindowUDF
 
 from ._internal import RuntimeEnvBuilder as RuntimeEnvBuilderInternal
 from ._internal import SessionConfig as SessionConfigInternal

examples/ffi-table-provider/pyproject.toml renamed to examples/ffi-library/pyproject.toml

@@ -153,10 +153,9 @@ def udf(*args: Any, **kwargs: Any):  # noqa: D417
         This class can be used both as a **function** and as a **decorator**.
 
         Usage:
-            - **As a function**: Call `udf(func, input_types, return_type, volatility,
-              name)`.
-            - **As a decorator**: Use `@udf(input_types, return_type, volatility,
-              name)`. In this case, do **not** pass `func` explicitly.
+            - **As a function**: Call :py:func:`udf(func, input_types, return_type, volatility, name)`.
+            - **As a decorator**: Use :py:func:`@udf(input_types, return_type, volatility, name)`.
+              In this case, do **not** pass `func` explicitly.
 
         Args:
             func (Callable, optional): **Only needed when calling as a function.**
@@ -175,9 +174,8 @@ def udf(*args: Any, **kwargs: Any):  # noqa: D417
             A user-defined function that can be used in SQL expressions,
             data aggregation, or window function calls.
 
-        Example:
-            **Using `udf` as a function:**
-            ```
+        Example: **Using `udf` as a function**::
+
             def double_func(x):
                 return x * 2
             double_udf = udf(double_func, [pa.int32()], pa.int32(),
@@ -189,8 +187,7 @@ def double_func(x):
             @udf([pa.int32()], pa.int32(), "volatile", "double_it")
             def double_udf(x):
                 return x * 2
-            ```
-        """
+        """  # noqa: E501 W505
 
         def _function(
             func: Callable[..., _R],
@@ -348,20 +345,18 @@ def udaf(*args: Any, **kwargs: Any):  # noqa: D417
         data aggregation or window function calls.
 
         Usage:
-            - **As a function**: Call `udaf(accum, input_types, return_type, state_type,
-                volatility, name)`.
-            - **As a decorator**: Use `@udaf(input_types, return_type, state_type,
-                volatility, name)`.
-            When using `udaf` as a decorator, **do not pass `accum` explicitly**.
+            - **As a function**: Call :py:func:`udaf(accum, input_types, return_type, state_type, volatility, name)`.
+            - **As a decorator**: Use :py:func:`@udaf(input_types, return_type, state_type, volatility, name)`.
+              When using `udaf` as a decorator, **do not pass `accum` explicitly**.
 
         **Function example:**
 
-            If your `:py:class:Accumulator` can be instantiated with no arguments, you
-            can simply pass it's type as `accum`. If you need to pass additional
-            arguments to it's constructor, you can define a lambda or a factory method.
-            During runtime the `:py:class:Accumulator` will be constructed for every
-            instance in which this UDAF is used. The following examples are all valid.
-            ```
+        If your :py:class:`Accumulator` can be instantiated with no arguments, you
+        can simply pass it's type as `accum`. If you need to pass additional
+        arguments to it's constructor, you can define a lambda or a factory method.
+        During runtime the :py:class:`Accumulator` will be constructed for every
+        instance in which this UDAF is used. The following examples are all valid::
+
             import pyarrow as pa
             import pyarrow.compute as pc
 
@@ -390,14 +385,12 @@ def sum_bias_10() -> Summarize:
                 "immutable")
             udaf3 = udaf(lambda: Summarize(20.0), pa.float64(), pa.float64(),
                 [pa.float64()], "immutable")
-            ```
 
-        **Decorator example:**
-            ```
+        **Decorator example:**::
+
             @udaf(pa.float64(), pa.float64(), [pa.float64()], "immutable")
             def udf4() -> Summarize:
                 return Summarize(10.0)
-            ```
 
         Args:
             accum: The accumulator python function. **Only needed when calling as a
@@ -411,7 +404,7 @@ def udf4() -> Summarize:
         Returns:
             A user-defined aggregate function, which can be used in either data
             aggregation or window function calls.
-        """
+        """  # noqa: E501 W505
 
         def _function(
             accum: Callable[[], Accumulator],
@@ -691,8 +684,8 @@ def udwf(*args: Any, **kwargs: Any):  # noqa: D417
               name)`. When using `udwf` as a decorator, **do not pass `func`
               explicitly**.
 
-        **Function example:**
-            ```
+        **Function example:**::
+
             import pyarrow as pa
 
             class BiasedNumbers(WindowEvaluator):
@@ -710,14 +703,11 @@ def bias_10() -> BiasedNumbers:
             udwf2 = udwf(bias_10, pa.int64(), pa.int64(), "immutable")
             udwf3 = udwf(lambda: BiasedNumbers(20), pa.int64(), pa.int64(), "immutable")
 
-            ```
+        **Decorator example:**::
 
-        **Decorator example:**
-            ```
             @udwf(pa.int64(), pa.int64(), "immutable")
             def biased_numbers() -> BiasedNumbers:
                 return BiasedNumbers(10)
-            ```
 
         Args:
             func: **Only needed when calling as a function. Skip this argument when

examples/ffi-table-provider/python/tests/_test_table_provider.py renamed to examples/ffi-library/python/tests/_test_table_provider.py

@@ -107,7 +107,7 @@ def test_class_module_is_datafusion():
         AggregateUDF,
         ScalarUDF,
     ]:
-        assert klass.__module__ == "datafusion.udf"
+        assert klass.__module__ == "datafusion.user_defined"
 
     # expressions
     for klass in [Expr, Column, Literal, BinaryExpr, AggregateFunction]:

examples/ffi-table-provider/src/lib.rs renamed to examples/ffi-library/src/lib.rs

@@ -22,7 +22,7 @@
 from datafusion import SessionContext, column, lit, udwf
 from datafusion import functions as f
 from datafusion.expr import WindowFrame
-from datafusion.udf import WindowEvaluator
+from datafusion.user_defined import WindowEvaluator
 
 
 class ExponentialSmoothDefault(WindowEvaluator):

examples/python-udwf.py

@@ -835,7 +835,7 @@ fn record_batch_into_schema(
 ) -> Result<RecordBatch, ArrowError> {
     let schema = Arc::new(schema.clone());
     let base_schema = record_batch.schema();
-    if base_schema.fields().len() == 0 {
+    if base_schema.fields().is_empty() {
         // Nothing to project
         return Ok(RecordBatch::new_empty(schema));
     }