Skip to content

perf : experiment roaring bitmap for int32 anti and semi joins #21817

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
coderfender wants to merge 16 commits into
base: main
Choose a base branch
from
Draft

perf : experiment roaring bitmap for int32 anti and semi joins #21817

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
1c773e1
roaring_bitmap_for_blazingly_fast_semi_joins_int32_join_key
coderfender Apr 21, 2026
40bea15
rebase_main
coderfender Apr 23, 2026
5a75ee0
rebase_main
coderfender Apr 23, 2026
0a2d0e1
Merge branch 'main' into experiment_roaring_bitmap_for_int32_anti_sem…
coderfender Apr 23, 2026
cc6607d
rebase_main
coderfender Apr 23, 2026
86c0f30
null_handling
coderfender Apr 24, 2026
d109417
null_handling
coderfender Apr 24, 2026
31f2afc
update_slt_to_reflect_roaring_bitmaps
coderfender Apr 24, 2026
a470f37
fix_tests
coderfender Apr 24, 2026
fb3f214
fix_tests
coderfender Apr 24, 2026
4fa2df7
Merge branch 'main' into experiment_roaring_bitmap_for_int32_anti_sem…
coderfender Apr 24, 2026
698dcbb
support_partition_eval_mode
coderfender Apr 24, 2026
16c82bb
add_bool_buffers_memebership_test
coderfender Apr 24, 2026
c6f059a
Merge branch 'main' into experiment_roaring_bitmap_for_int32_anti_sem…
coderfender Apr 24, 2026
173bd10
add_existence_benchmarks_hashjoin
coderfender Apr 24, 2026
20a42c2
Merge branch 'main' into experiment_roaring_bitmap_for_int32_anti_sem…
coderfender Apr 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,7 @@ prost = "0.14.1"
rand = "0.9"
recursive = "0.1.1"
regex = "1.12"
roaring = "0.11.3"
rstest = "0.26.1"
serde_json = "1"
sha2 = "^0.10.9"
Expand Down
192 changes: 192 additions & 0 deletions benchmarks/src/hj.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,198 @@ const HASH_QUERIES: &[HashJoinQuery] = &[
build_size: "100K_(20%_dups)",
probe_size: "60M",
},
// RightSemi Join benchmarks with Int32 keys
// Q16: RightSemi, 100% Density, 100% Hit rate
HashJoinQuery {
sql: r###"SELECT l.k
FROM (
SELECT CAST(l_suppkey AS INT) as k FROM lineitem
) l
WHERE EXISTS (
SELECT 1 FROM (SELECT CAST(s_suppkey AS INT) as k FROM supplier) s WHERE s.k = l.k
)"###,
density: 1.0,
prob_hit: 1.0,
build_size: "100K",
probe_size: "60M_RightSemi",
},
// Q17: RightSemi, 100% Density, 10% Hit rate
HashJoinQuery {
sql: r###"SELECT l.k
FROM (
SELECT CAST(CASE WHEN l_suppkey % 10 = 0 THEN l_suppkey ELSE l_suppkey + 1000000 END AS INT) as k
FROM lineitem
) l
WHERE EXISTS (
SELECT 1 FROM (SELECT CAST(s_suppkey AS INT) as k FROM supplier) s WHERE s.k = l.k
)"###,
density: 1.0,
prob_hit: 0.1,
build_size: "100K",
probe_size: "60M_RightSemi",
},
// Q18: RightSemi, 50% Density, 100% Hit rate
HashJoinQuery {
sql: r###"SELECT l.k
FROM (
SELECT CAST(l_suppkey * 2 AS INT) as k FROM lineitem
) l
WHERE EXISTS (
SELECT 1 FROM (SELECT CAST(s_suppkey * 2 AS INT) as k FROM supplier) s WHERE s.k = l.k
)"###,
density: 0.5,
prob_hit: 1.0,
build_size: "100K",
probe_size: "60M_RightSemi",
},
// Q19: RightSemi, 50% Density, 10% Hit rate
HashJoinQuery {
sql: r###"SELECT l.k
FROM (
SELECT CAST(CASE
WHEN l_suppkey % 10 = 0 THEN l_suppkey * 2
WHEN l_suppkey % 10 < 9 THEN l_suppkey * 2 + 1
ELSE l_suppkey * 2 + 1000000
END AS INT) as k
FROM lineitem
) l
WHERE EXISTS (
SELECT 1 FROM (SELECT CAST(s_suppkey * 2 AS INT) as k FROM supplier) s WHERE s.k = l.k
)"###,
density: 0.5,
prob_hit: 0.1,
build_size: "100K",
probe_size: "60M_RightSemi",
},
// Q20: RightSemi, 10% Density, 100% Hit rate
HashJoinQuery {
sql: r###"SELECT l.k
FROM (
SELECT CAST(l_suppkey * 10 AS INT) as k FROM lineitem
) l
WHERE EXISTS (
SELECT 1 FROM (SELECT CAST(s_suppkey * 10 AS INT) as k FROM supplier) s WHERE s.k = l.k
)"###,
density: 0.1,
prob_hit: 1.0,
build_size: "100K",
probe_size: "60M_RightSemi",
},
// Q21: RightSemi, 10% Density, 10% Hit rate
HashJoinQuery {
sql: r###"SELECT l.k
FROM (
SELECT CAST(CASE
WHEN l_suppkey % 10 = 0 THEN l_suppkey * 10
WHEN l_suppkey % 10 < 9 THEN l_suppkey * 10 + 1
ELSE l_suppkey * 10 + 1000000
END AS INT) as k
FROM lineitem
) l
WHERE EXISTS (
SELECT 1 FROM (SELECT CAST(s_suppkey * 10 AS INT) as k FROM supplier) s WHERE s.k = l.k
)"###,
density: 0.1,
prob_hit: 0.1,
build_size: "100K",
probe_size: "60M_RightSemi",
},
// RightAnti Join benchmarks with Int32 keys
// Q22: RightAnti, 100% Density, 100% Hit rate (no output)
HashJoinQuery {
sql: r###"SELECT l.k
FROM (
SELECT CAST(l_suppkey AS INT) as k FROM lineitem
) l
WHERE NOT EXISTS (
SELECT 1 FROM (SELECT CAST(s_suppkey AS INT) as k FROM supplier) s WHERE s.k = l.k
)"###,
density: 1.0,
prob_hit: 1.0,
build_size: "100K",
probe_size: "60M_RightAnti",
},
// Q23: RightAnti, 100% Density, 10% Hit rate (90% output)
HashJoinQuery {
sql: r###"SELECT l.k
FROM (
SELECT CAST(CASE WHEN l_suppkey % 10 = 0 THEN l_suppkey ELSE l_suppkey + 1000000 END AS INT) as k
FROM lineitem
) l
WHERE NOT EXISTS (
SELECT 1 FROM (SELECT CAST(s_suppkey AS INT) as k FROM supplier) s WHERE s.k = l.k
)"###,
density: 1.0,
prob_hit: 0.1,
build_size: "100K",
probe_size: "60M_RightAnti",
},
// Q24: RightAnti, 50% Density, 100% Hit rate (no output)
HashJoinQuery {
sql: r###"SELECT l.k
FROM (
SELECT CAST(l_suppkey * 2 AS INT) as k FROM lineitem
) l
WHERE NOT EXISTS (
SELECT 1 FROM (SELECT CAST(s_suppkey * 2 AS INT) as k FROM supplier) s WHERE s.k = l.k
)"###,
density: 0.5,
prob_hit: 1.0,
build_size: "100K",
probe_size: "60M_RightAnti",
},
// Q25: RightAnti, 50% Density, 10% Hit rate (90% output)
HashJoinQuery {
sql: r###"SELECT l.k
FROM (
SELECT CAST(CASE
WHEN l_suppkey % 10 = 0 THEN l_suppkey * 2
WHEN l_suppkey % 10 < 9 THEN l_suppkey * 2 + 1
ELSE l_suppkey * 2 + 1000000
END AS INT) as k
FROM lineitem
) l
WHERE NOT EXISTS (
SELECT 1 FROM (SELECT CAST(s_suppkey * 2 AS INT) as k FROM supplier) s WHERE s.k = l.k
)"###,
density: 0.5,
prob_hit: 0.1,
build_size: "100K",
probe_size: "60M_RightAnti",
},
// Q26: RightAnti, 10% Density, 100% Hit rate (no output)
HashJoinQuery {
sql: r###"SELECT l.k
FROM (
SELECT CAST(l_suppkey * 10 AS INT) as k FROM lineitem
) l
WHERE NOT EXISTS (
SELECT 1 FROM (SELECT CAST(s_suppkey * 10 AS INT) as k FROM supplier) s WHERE s.k = l.k
)"###,
density: 0.1,
prob_hit: 1.0,
build_size: "100K",
probe_size: "60M_RightAnti",
},
// Q27: RightAnti, 10% Density, 10% Hit rate (90% output)
HashJoinQuery {
sql: r###"SELECT l.k
FROM (
SELECT CAST(CASE
WHEN l_suppkey % 10 = 0 THEN l_suppkey * 10
WHEN l_suppkey % 10 < 9 THEN l_suppkey * 10 + 1
ELSE l_suppkey * 10 + 1000000
END AS INT) as k
FROM lineitem
) l
WHERE NOT EXISTS (
SELECT 1 FROM (SELECT CAST(s_suppkey * 10 AS INT) as k FROM supplier) s WHERE s.k = l.k
)"###,
density: 0.1,
prob_hit: 0.1,
build_size: "100K",
probe_size: "60M_RightAnti",
},
];

impl RunOpt {
Expand Down
1 change: 1 addition & 0 deletions datafusion/physical-plan/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ log = { workspace = true }
num-traits = { workspace = true }
parking_lot = { workspace = true }
pin-project-lite = "^0.2.7"
roaring = { workspace = true }
tokio = { workspace = true }

[dev-dependencies]
Expand Down
Loading
Loading