Skip to content

Commit d46015a

Browse files
dd-annaroseJefffreyblaginin
authored
fix: datatype_is_logically_equal for dictionaries (#20153)
## Which issue does this PR close? When checking logical equivalence with `Dictionary<_, Utf8>` and `Utf8View`, the response was `false` which is not what we expect (logical equivalence should be a transitive property). ## What changes are included in this PR? This PR introduces a test and a fix. The test fails without the fix. The fix is simply calling `datatype_is_logically_equal` again on the `v1` and `othertype` when called with `Dictionary<K1, V1>` and `othertype`. ## Are these changes tested? Yes. ## Are there any user-facing changes? No. --------- Co-authored-by: Jeffrey Vo <jeffrey.vo.australia@gmail.com> Co-authored-by: Dmitrii Blaginin <dmitrii@blaginin.me> Co-authored-by: blaginin <github@blaginin.me>
1 parent ee01acf commit d46015a

1 file changed

Lines changed: 26 additions & 3 deletions

File tree

datafusion/common/src/dfschema.rs

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -698,10 +698,12 @@ impl DFSchema {
698698
// check nested fields
699699
match (dt1, dt2) {
700700
(DataType::Dictionary(_, v1), DataType::Dictionary(_, v2)) => {
701-
v1.as_ref() == v2.as_ref()
701+
Self::datatype_is_logically_equal(v1.as_ref(), v2.as_ref())
702+
}
703+
(DataType::Dictionary(_, v1), othertype)
704+
| (othertype, DataType::Dictionary(_, v1)) => {
705+
Self::datatype_is_logically_equal(v1.as_ref(), othertype)
702706
}
703-
(DataType::Dictionary(_, v1), othertype) => v1.as_ref() == othertype,
704-
(othertype, DataType::Dictionary(_, v1)) => v1.as_ref() == othertype,
705707
(DataType::List(f1), DataType::List(f2))
706708
| (DataType::LargeList(f1), DataType::LargeList(f2))
707709
| (DataType::FixedSizeList(f1, _), DataType::FixedSizeList(f2, _)) => {
@@ -1798,6 +1800,27 @@ mod tests {
17981800
&DataType::Utf8,
17991801
&DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8))
18001802
));
1803+
1804+
// Dictionary is logically equal to the logically equivalent value type
1805+
assert!(DFSchema::datatype_is_logically_equal(
1806+
&DataType::Utf8View,
1807+
&DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8))
1808+
));
1809+
1810+
assert!(DFSchema::datatype_is_logically_equal(
1811+
&DataType::Dictionary(
1812+
Box::new(DataType::Int32),
1813+
Box::new(DataType::List(
1814+
Field::new("element", DataType::Utf8, false).into()
1815+
))
1816+
),
1817+
&DataType::Dictionary(
1818+
Box::new(DataType::Int32),
1819+
Box::new(DataType::List(
1820+
Field::new("element", DataType::Utf8View, false).into()
1821+
))
1822+
)
1823+
));
18011824
}
18021825

18031826
#[test]

0 commit comments

Comments
 (0)