Skip to content

Commit 88e9856

Browse files
erenavsarogullarierenavsarogullari
committed
Fix CI test failures
1 parent 6a44ab9 commit 88e9856

File tree

6 files changed

+93
-19
lines changed

6 files changed

+93
-19
lines changed

‎datafusion-cli/tests/snapshots/cli_top_memory_consumers@no_track.snap‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,6 @@ exit_code: 1
1616
[CLI_VERSION]
1717
Error: Not enough memory to continue external sort. Consider increasing the memory limit config: 'datafusion.runtime.memory_limit', or decreasing the config: 'datafusion.execution.sort_spill_reservation_bytes'.
1818
caused by
19-
Resources exhausted: Failed to allocate
19+
Resources exhausted: Failed to allocate additional 128.0 KB for ExternalSorter[0] with 0.0 B already allocated for this reservation - 0.0 B remain available for the total memory pool: greedy(used: 10.0 MB, pool_size: 10.0 MB)
2020

2121
----- stderr -----

‎datafusion-cli/tests/snapshots/cli_top_memory_consumers@top2.snap‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,6 @@ caused by
1919
Resources exhausted: Additional allocation failed for ExternalSorter[0] with top memory consumers (across reservations) as:
2020
Consumer(can spill: bool) consumed XB, peak XB,
2121
Consumer(can spill: bool) consumed XB, peak XB.
22-
Error: Failed to allocate
22+
Error: Failed to allocate additional 128.0 KB for ExternalSorter[0] with 0.0 B already allocated for this reservation - 0.0 B remain available for the total memory pool: greedy(used: 10.0 MB, pool_size: 10.0 MB)
2323

2424
----- stderr -----

‎datafusion-cli/tests/snapshots/cli_top_memory_consumers@top3_default.snap‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,6 @@ Resources exhausted: Additional allocation failed for ExternalSorter[0] with top
1818
Consumer(can spill: bool) consumed XB, peak XB,
1919
Consumer(can spill: bool) consumed XB, peak XB,
2020
Consumer(can spill: bool) consumed XB, peak XB.
21-
Error: Failed to allocate
21+
Error: Failed to allocate additional 128.0 KB for ExternalSorter[0] with 0.0 B already allocated for this reservation - 0.0 B remain available for the total memory pool: greedy(used: 10.0 MB, pool_size: 10.0 MB)
2222

2323
----- stderr -----

‎datafusion-cli/tests/snapshots/cli_top_memory_consumers_with_mem_pool_type@no_track.snap‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,6 @@ exit_code: 1
1818
[CLI_VERSION]
1919
Error: Not enough memory to continue external sort. Consider increasing the memory limit config: 'datafusion.runtime.memory_limit', or decreasing the config: 'datafusion.execution.sort_spill_reservation_bytes'.
2020
caused by
21-
Resources exhausted: Failed to allocate
21+
Resources exhausted: Failed to allocate additional 128.0 KB for ExternalSorter[0] with 0.0 B already allocated for this reservation - 0.0 B remain available for the total memory pool: fair(pool_size: 10.0 MB)
2222

2323
----- stderr -----

‎datafusion-cli/tests/snapshots/cli_top_memory_consumers_with_mem_pool_type@top2.snap‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,6 @@ caused by
2121
Resources exhausted: Additional allocation failed for ExternalSorter[0] with top memory consumers (across reservations) as:
2222
Consumer(can spill: bool) consumed XB, peak XB,
2323
Consumer(can spill: bool) consumed XB, peak XB.
24-
Error: Failed to allocate
24+
Error: Failed to allocate additional 128.0 KB for ExternalSorter[0] with 0.0 B already allocated for this reservation - 0.0 B remain available for the total memory pool: fair(pool_size: 10.0 MB)
2525

2626
----- stderr -----

‎datafusion/execution/src/memory_pool/pool.rs‎

Lines changed: 88 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -654,10 +654,10 @@ mod tests {
654654
assert_eq!(pool.reserved(), 4000);
655655

656656
let err = r2.try_grow(1).unwrap_err().strip_backtrace();
657-
assert_snapshot!(err, @"Resources exhausted: Failed to allocate additional 1.0 B for r2 with 2000.0 B already allocated for this reservation - 0.0 B remain available for the total memory pool:");
657+
assert_snapshot!(err, @"Resources exhausted: Failed to allocate additional 1.0 B for r2 with 2000.0 B already allocated for this reservation - 0.0 B remain available for the total memory pool: fair(pool_size: 100.0 B)");
658658

659659
let err = r2.try_grow(1).unwrap_err().strip_backtrace();
660-
assert_snapshot!(err, @"Resources exhausted: Failed to allocate additional 1.0 B for r2 with 2000.0 B already allocated for this reservation - 0.0 B remain available for the total memory pool:");
660+
assert_snapshot!(err, @"Resources exhausted: Failed to allocate additional 1.0 B for r2 with 2000.0 B already allocated for this reservation - 0.0 B remain available for the total memory pool: fair(pool_size: 100.0 B)");
661661

662662
r1.shrink(1990);
663663
r2.shrink(2000);
@@ -682,12 +682,12 @@ mod tests {
682682
.register(&pool);
683683

684684
let err = r3.try_grow(70).unwrap_err().strip_backtrace();
685-
assert_snapshot!(err, @"Resources exhausted: Failed to allocate additional 70.0 B for r3 with 0.0 B already allocated for this reservation - 40.0 B remain available for the total memory pool:");
685+
assert_snapshot!(err, @"Resources exhausted: Failed to allocate additional 70.0 B for r3 with 0.0 B already allocated for this reservation - 40.0 B remain available for the total memory pool: fair(pool_size: 100.0 B)");
686686

687687
//Shrinking r2 to zero doesn't allow a3 to allocate more than 45
688688
r2.free();
689689
let err = r3.try_grow(70).unwrap_err().strip_backtrace();
690-
assert_snapshot!(err, @"Resources exhausted: Failed to allocate additional 70.0 B for r3 with 0.0 B already allocated for this reservation - 40.0 B remain available for the total memory pool:");
690+
assert_snapshot!(err, @"Resources exhausted: Failed to allocate additional 70.0 B for r3 with 0.0 B already allocated for this reservation - 40.0 B remain available for the total memory pool: fair(pool_size: 100.0 B)");
691691

692692
// But dropping r2 does
693693
drop(r2);
@@ -700,7 +700,7 @@ mod tests {
700700

701701
let r4 = MemoryConsumer::new("s4").register(&pool);
702702
let err = r4.try_grow(30).unwrap_err().strip_backtrace();
703-
assert_snapshot!(err, @"Resources exhausted: Failed to allocate additional 30.0 B for s4 with 0.0 B already allocated for this reservation - 20.0 B remain available for the total memory pool:");
703+
assert_snapshot!(err, @"Resources exhausted: Failed to allocate additional 30.0 B for s4 with 0.0 B already allocated for this reservation - 20.0 B remain available for the total memory pool: fair(pool_size: 100.0 B)");
704704
}
705705

706706
#[test]
@@ -748,7 +748,7 @@ mod tests {
748748
r1#[ID](can spill: false) consumed 50.0 B, peak 70.0 B,
749749
r3#[ID](can spill: false) consumed 20.0 B, peak 25.0 B,
750750
r2#[ID](can spill: false) consumed 15.0 B, peak 15.0 B.
751-
Error: Failed to allocate additional 150.0 B for r5 with 0.0 B already allocated for this reservation - 5.0 B remain available for the total memory pool:
751+
Error: Failed to allocate additional 150.0 B for r5 with 0.0 B already allocated for this reservation - 5.0 B remain available for the total memory pool: greedy(used: 95.0 B, pool_size: 100.0 B)
752752
");
753753
}
754754

@@ -771,7 +771,7 @@ mod tests {
771771
assert_snapshot!(error, @r"
772772
Resources exhausted: Additional allocation failed for foo with top memory consumers (across reservations) as:
773773
foo#[ID](can spill: false) consumed 0.0 B, peak 0.0 B.
774-
Error: Failed to allocate additional 150.0 B for foo with 0.0 B already allocated for this reservation - 100.0 B remain available for the total memory pool:
774+
Error: Failed to allocate additional 150.0 B for foo with 0.0 B already allocated for this reservation - 100.0 B remain available for the total memory pool: greedy(used: 0.0 B, pool_size: 100.0 B)
775775
");
776776

777777
// API: multiple registrations using the same hashed consumer,
@@ -789,7 +789,7 @@ mod tests {
789789
Resources exhausted: Additional allocation failed for foo with top memory consumers (across reservations) as:
790790
foo#[ID](can spill: false) consumed 10.0 B, peak 10.0 B,
791791
foo#[ID](can spill: false) consumed 0.0 B, peak 0.0 B.
792-
Error: Failed to allocate additional 150.0 B for foo with 0.0 B already allocated for this reservation - 90.0 B remain available for the total memory pool:
792+
Error: Failed to allocate additional 150.0 B for foo with 0.0 B already allocated for this reservation - 90.0 B remain available for the total memory pool: greedy(used: 10.0 B, pool_size: 100.0 B)
793793
");
794794

795795
// Test: will accumulate size changes per consumer, not per reservation
@@ -802,7 +802,7 @@ mod tests {
802802
Resources exhausted: Additional allocation failed for foo with top memory consumers (across reservations) as:
803803
foo#[ID](can spill: false) consumed 20.0 B, peak 20.0 B,
804804
foo#[ID](can spill: false) consumed 10.0 B, peak 10.0 B.
805-
Error: Failed to allocate additional 150.0 B for foo with 20.0 B already allocated for this reservation - 70.0 B remain available for the total memory pool:
805+
Error: Failed to allocate additional 150.0 B for foo with 20.0 B already allocated for this reservation - 70.0 B remain available for the total memory pool: greedy(used: 30.0 B, pool_size: 100.0 B)
806806
");
807807

808808
// Test: different hashed consumer, (even with the same name),
@@ -818,7 +818,7 @@ mod tests {
818818
foo#[ID](can spill: false) consumed 20.0 B, peak 20.0 B,
819819
foo#[ID](can spill: false) consumed 10.0 B, peak 10.0 B,
820820
foo#[ID](can spill: true) consumed 0.0 B, peak 0.0 B.
821-
Error: Failed to allocate additional 150.0 B for foo with 0.0 B already allocated for this reservation - 70.0 B remain available for the total memory pool:
821+
Error: Failed to allocate additional 150.0 B for foo with 0.0 B already allocated for this reservation - 70.0 B remain available for the total memory pool: greedy(used: 30.0 B, pool_size: 100.0 B)
822822
");
823823
}
824824

@@ -841,7 +841,7 @@ mod tests {
841841
Resources exhausted: Additional allocation failed for r0 with top memory consumers (across reservations) as:
842842
r1#[ID](can spill: false) consumed 20.0 B, peak 20.0 B,
843843
r0#[ID](can spill: false) consumed 10.0 B, peak 10.0 B.
844-
Error: Failed to allocate additional 150.0 B for r0 with 10.0 B already allocated for this reservation - 70.0 B remain available for the total memory pool:
844+
Error: Failed to allocate additional 150.0 B for r0 with 10.0 B already allocated for this reservation - 70.0 B remain available for the total memory pool: greedy(used: 30.0 B, pool_size: 100.0 B)
845845
"));
846846

847847
// Test: unregister one
@@ -853,7 +853,7 @@ mod tests {
853853
allow_duplicates!(assert_snapshot!(error, @r"
854854
Resources exhausted: Additional allocation failed for r0 with top memory consumers (across reservations) as:
855855
r0#[ID](can spill: false) consumed 10.0 B, peak 10.0 B.
856-
Error: Failed to allocate additional 150.0 B for r0 with 10.0 B already allocated for this reservation - 90.0 B remain available for the total memory pool:
856+
Error: Failed to allocate additional 150.0 B for r0 with 10.0 B already allocated for this reservation - 90.0 B remain available for the total memory pool: fair(pool_size: 100.0 B)
857857
"));
858858

859859
// Test: actual message we see is the `available is 70`. When it should be `available is 90`.
@@ -864,7 +864,7 @@ mod tests {
864864
allow_duplicates!(assert_snapshot!(error, @r"
865865
Resources exhausted: Additional allocation failed for r0 with top memory consumers (across reservations) as:
866866
r0#[ID](can spill: false) consumed 10.0 B, peak 10.0 B.
867-
Error: Failed to allocate additional 150.0 B for r0 with 10.0 B already allocated for this reservation - 90.0 B remain available for the total memory pool:
867+
Error: Failed to allocate additional 150.0 B for r0 with 10.0 B already allocated for this reservation - 90.0 B remain available for the total memory pool: fair(pool_size: 100.0 B)
868868
"));
869869

870870
// Test: the registration needs to free itself (or be dropped),
@@ -875,7 +875,7 @@ mod tests {
875875
allow_duplicates!(assert_snapshot!(error, @r"
876876
Resources exhausted: Additional allocation failed for r0 with top memory consumers (across reservations) as:
877877
r0#[ID](can spill: false) consumed 10.0 B, peak 10.0 B.
878-
Error: Failed to allocate additional 150.0 B for r0 with 10.0 B already allocated for this reservation - 90.0 B remain available for the total memory pool:
878+
Error: Failed to allocate additional 150.0 B for r0 with 10.0 B already allocated for this reservation - 90.0 B remain available for the total memory pool: fair(pool_size: 100.0 B)
879879
"));
880880
}
881881

@@ -973,4 +973,78 @@ mod tests {
973973
r1#[ID](can spill: false) consumed 20.0 B, peak 20.0 B.
974974
");
975975
}
976+
977+
#[test]
978+
fn test_memory_pool_display_fmt() {
979+
let top = NonZeroUsize::new(5).unwrap();
980+
981+
// UnboundedMemoryPool Display with default allocation: 0.0B
982+
let unbounded = UnboundedMemoryPool::default();
983+
assert_eq!(
984+
unbounded.to_string(),
985+
"unbounded(used: 0.0 B)",
986+
"UnboundedMemoryPool Display"
987+
);
988+
989+
// UnboundedMemoryPool Display with reservations
990+
let unbounded_arc: Arc<dyn MemoryPool> = Arc::new(UnboundedMemoryPool::default());
991+
let r = MemoryConsumer::new("u").register(&unbounded_arc);
992+
r.grow(2048);
993+
assert_eq!(
994+
unbounded_arc.as_ref().to_string(),
995+
"unbounded(used: 2.0 KB)",
996+
"UnboundedMemoryPool Display with reservations"
997+
);
998+
999+
// GreedyMemoryPool Display with default allocation: 100.0B
1000+
let greedy = GreedyMemoryPool::new(100);
1001+
assert_eq!(
1002+
greedy.to_string(),
1003+
"greedy(used: 0.0 B, pool_size: 100.0 B)",
1004+
"GreedyMemoryPool Display"
1005+
);
1006+
1007+
// GreedyMemoryPool Display with reservations
1008+
let greedy_arc: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(100));
1009+
let r = MemoryConsumer::new("g").register(&greedy_arc);
1010+
r.grow(50);
1011+
assert_eq!(
1012+
greedy_arc.as_ref().to_string(),
1013+
"greedy(used: 50.0 B, pool_size: 100.0 B)",
1014+
"GreedyMemoryPool Display with reservations"
1015+
);
1016+
1017+
// FairSpillPool Display with default allocation: 4.0KB and without reservations
1018+
let fair = FairSpillPool::new(4096);
1019+
assert_eq!(
1020+
fair.to_string(),
1021+
"fair(pool_size: 4.0 KB)",
1022+
"FairSpillPool Display"
1023+
);
1024+
1025+
// TrackConsumersPool<GreedyMemoryPool> Display with default allocation: 128.0B and without reservations
1026+
let tracked_greedy = TrackConsumersPool::new(GreedyMemoryPool::new(128), top);
1027+
assert_eq!(
1028+
tracked_greedy.to_string(),
1029+
"track_consumers(inner_pool: greedy(used: 0.0 B, pool_size: 128.0 B), num_of_top_consumers: 5)",
1030+
"TrackConsumersPool<GreedyMemoryPool> Display"
1031+
);
1032+
1033+
// TrackConsumersPool<FairSpillPool> Display with default allocation: 256.0B and without reservations
1034+
let tracked_fair = TrackConsumersPool::new(FairSpillPool::new(256), top);
1035+
assert_eq!(
1036+
tracked_fair.to_string(),
1037+
"track_consumers(inner_pool: fair(pool_size: 256.0 B), num_of_top_consumers: 5)",
1038+
"TrackConsumersPool<FairSpillPool> Display"
1039+
);
1040+
1041+
// TrackConsumersPool<UnboundedMemoryPool> Display without reservations
1042+
let tracked_unbounded =
1043+
TrackConsumersPool::new(UnboundedMemoryPool::default(), top);
1044+
assert_eq!(
1045+
tracked_unbounded.to_string(),
1046+
"track_consumers(inner_pool: unbounded(used: 0.0 B), num_of_top_consumers: 5)",
1047+
"TrackConsumersPool<UnboundedMemoryPool> Display"
1048+
);
1049+
}
9761050
}

0 commit comments

Comments
 (0)