fix(shuffle): fuse repartition flush buffers before partitioning to bound map memory
#7064Comparing
colin/flight-shuffle-map-side-memory
(a5b09dc
) withmain
(7b7658f
)0%
- Untouched
- 40
- Ignored
- 4
- Skipped
- 10
Benchmarks
54 total
test_clickbench_sql[4]
tests/benchmarks/test_clickbench.py+3%
81.1 ms78.8 ms
test_iter_rows_first_row[1 Small File]
tests/benchmarks/test_interactive_reads.py+3%
34.6 ms33.6 ms
test_clickbench_sql[0]
tests/benchmarks/test_clickbench.py+3%
4.7 ms4.6 ms
test_tpch_sql[1-in-memory-2]
tests/benchmarks/test_local_tpch.py+2%
119 ms116.4 ms
test_explain[100 Small Files]
tests/benchmarks/test_interactive_reads.py+2%
10.4 ms10.2 ms
test_read_jsonl_string
tests/benchmarks/test_json_reads.py+2%
333.7 ms328 ms
test_clickbench_sql[1]
tests/benchmarks/test_clickbench.py+1%
16.5 ms16.3 ms
test_read_jsonl_high_nulls
tests/benchmarks/test_json_reads.py+1%
417 ms413.4 ms
test_show[1 Small File]
tests/benchmarks/test_interactive_reads.py+1%
12 ms11.9 ms
test_read_jsonl_nested_list
tests/benchmarks/test_json_reads.py+1%
454.3 ms451.4 ms
test_tpch_sql[1-in-memory-10]
tests/benchmarks/test_local_tpch.py+1%
170.4 ms169.3 ms
test_read_jsonl_int64
tests/benchmarks/test_json_reads.py+1%
289 ms287.3 ms
test_tpch_sql[1-in-memory-5]
tests/benchmarks/test_local_tpch.py+1%
89 ms88.5 ms
test_clickbench_sql[37]
tests/benchmarks/test_clickbench.py+1%
183.3 ms182.4 ms
test_read_json_array_format
tests/benchmarks/test_json_reads.py0%
1.8 s1.8 s
test_read_jsonl_with_limit
tests/benchmarks/test_json_reads.py0%
68.4 ms68.1 ms
test_read_jsonl_boolean
tests/benchmarks/test_json_reads.py0%
359.2 ms357.6 ms
test_tpch_sql[1-in-memory-9]
tests/benchmarks/test_local_tpch.py0%
225.1 ms224.2 ms
test_tpch_sql[1-in-memory-4]
tests/benchmarks/test_local_tpch.py0%
69.6 ms69.5 ms
test_read_jsonl_float64
tests/benchmarks/test_json_reads.py0%
284.9 ms284.6 ms
test_tpch_sql[1-in-memory-3]
tests/benchmarks/test_local_tpch.py0%
100.9 ms100.8 ms
test_read_jsonl_with_explicit_schema
tests/benchmarks/test_json_reads.py0%
1.7 s1.7 s
test_read_jsonl_wide_table
tests/benchmarks/test_json_reads.py0%
1.7 s1.7 s
test_clickbench_sql[40]
tests/benchmarks/test_clickbench.py0%
76.7 ms76.7 ms
test_read_jsonl_with_column_projection
tests/benchmarks/test_json_reads.py0%
1.3 s1.3 s
Commits
Click on a commit to change the comparison rangeBase
main
7b7658f
+0.12%
fix(shuffle): compact flight repartition accumulators to bound map-task memory
a5b09dc
3 hours ago
by colin-ho