Skip to content

Commit 8a8c17f

Browse files
committed
refactor: simplify file_sort_order handling in SessionContext
1 parent dd9771c commit 8a8c17f

1 file changed

Lines changed: 13 additions & 14 deletions

File tree

python/datafusion/context.py

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -572,18 +572,13 @@ def register_listing_table(
572572
if table_partition_cols is None:
573573
table_partition_cols = []
574574
table_partition_cols = self._convert_table_partition_cols(table_partition_cols)
575-
file_sort_order_raw = (
576-
[sort_list_to_raw_sort_list(f) for f in file_sort_order]
577-
if file_sort_order is not None
578-
else None
579-
)
580575
self.ctx.register_listing_table(
581576
name,
582577
str(path),
583578
table_partition_cols,
584579
file_extension,
585580
schema,
586-
file_sort_order_raw,
581+
self._convert_file_sort_order(file_sort_order),
587582
)
588583

589584
def sql(self, query: str, options: SQLOptions | None = None) -> DataFrame:
@@ -840,9 +835,7 @@ def register_parquet(
840835
file_extension,
841836
skip_metadata,
842837
schema,
843-
[sort_list_to_raw_sort_list(exprs) for exprs in file_sort_order]
844-
if file_sort_order is not None
845-
else None,
838+
self._convert_file_sort_order(file_sort_order),
846839
)
847840

848841
def register_csv(
@@ -1124,11 +1117,7 @@ def read_parquet(
11241117
if table_partition_cols is None:
11251118
table_partition_cols = []
11261119
table_partition_cols = self._convert_table_partition_cols(table_partition_cols)
1127-
file_sort_order = (
1128-
[sort_list_to_raw_sort_list(f) for f in file_sort_order]
1129-
if file_sort_order is not None
1130-
else None
1131-
)
1120+
file_sort_order = self._convert_file_sort_order(file_sort_order)
11321121
return DataFrame(
11331122
self.ctx.read_parquet(
11341123
str(path),
@@ -1179,6 +1168,16 @@ def execute(self, plan: ExecutionPlan, partitions: int) -> RecordBatchStream:
11791168
"""Execute the ``plan`` and return the results."""
11801169
return RecordBatchStream(self.ctx.execute(plan._raw_plan, partitions))
11811170

1171+
@staticmethod
1172+
def _convert_file_sort_order(
1173+
file_sort_order: list[list[Expr | SortExpr | str]] | None,
1174+
) -> list[list[Any]] | None:
1175+
return (
1176+
[sort_list_to_raw_sort_list(f) for f in file_sort_order]
1177+
if file_sort_order is not None
1178+
else None
1179+
)
1180+
11821181
@staticmethod
11831182
def _convert_table_partition_cols(
11841183
table_partition_cols: list[tuple[str, str | pa.DataType]],

0 commit comments

Comments
 (0)