More documentation on FileSource::table_schema and FileSource::projection (#20242)

## Which issue does this PR close?

- Follow on to #20188

## Rationale for this change

@zhuqi-lucas and @adriangb had some good ideas on how to further improve
the documentation on #20188,
which I tried to implement in this PR

## What changes are included in this PR?

Add more clarity about what TableSource and FileSource::projection are

## Are these changes tested?

By CI
## Are there any user-facing changes?

Additional documentation

Author: alamb

datafusion/datasource/src/file.rs

@@ -39,16 +39,17 @@ use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;
 use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
 use object_store::ObjectStore;
 
-/// Helper function to convert any type implementing FileSource to Arc<dyn FileSource>
+/// Helper function to convert any type implementing [`FileSource`] to `Arc<dyn FileSource>`
 pub fn as_file_source<T: FileSource + 'static>(source: T) -> Arc<dyn FileSource> {
     Arc::new(source)
 }
 
-/// file format specific behaviors for elements in [`DataSource`]
+/// File format specific behaviors for [`DataSource`]
 ///
 /// # Schema information
 /// There are two important schemas for a [`FileSource`]:
-/// 1. [`Self::table_schema`] -- the schema for the overall "table"
+/// 1. [`Self::table_schema`] -- the schema for the overall table
+///    (file data plus partition columns)
 /// 2. The logical output schema, comprised of [`Self::table_schema`] with
 ///    [`Self::projection`] applied
 ///
@@ -71,13 +72,16 @@ pub trait FileSource: Send + Sync {
     /// Any
     fn as_any(&self) -> &dyn Any;
 
-    /// Returns the table schema for this file source.
+    /// Returns the table schema for the overall table (including partition columns, if any)
     ///
-    /// This always returns the unprojected schema (the full schema of the data)
+    /// This method returns the unprojected schema: the full schema of the data
     /// without [`Self::projection`] applied.
     ///
     /// The output schema of this `FileSource` is this TableSchema
     /// with [`Self::projection`] applied.
+    ///
+    /// Use [`ProjectionExprs::project_schema`] to get the projected schema
+    /// after applying the projection.
     fn table_schema(&self) -> &crate::table_schema::TableSchema;
 
     /// Initialize new type with batch size configuration
@@ -92,6 +96,9 @@ pub trait FileSource: Send + Sync {
 
     /// Return the projection that will be applied to the output stream on top
     /// of [`Self::table_schema`].
+    ///
+    /// Note you can use [`ProjectionExprs::project_schema`] on the table
+    /// schema to get the effective output schema of this source.
     fn projection(&self) -> Option<&ProjectionExprs> {
         None
     }

datafusion/datasource/src/table_schema.rs

@@ -20,13 +20,13 @@
 use arrow::datatypes::{FieldRef, SchemaBuilder, SchemaRef};
 use std::sync::Arc;
 
-/// Helper to hold table schema information for partitioned data sources.
+/// The overall schema for potentially partitioned data sources.
 ///
-/// When reading partitioned data (such as Hive-style partitioning), a table's schema
+/// When reading partitioned data (such as Hive-style partitioning), a [`TableSchema`]
 /// consists of two parts:
 /// 1. **File schema**: The schema of the actual data files on disk
-/// 2. **Partition columns**: Columns that are encoded in the directory structure,
-///    not stored in the files themselves
+/// 2. **Partition columns**: Columns whose values are encoded in the directory structure,
+///    but not stored in the files themselves
 ///
 /// # Example: Partitioned Table
 ///