From aca31fa093f416eb7865882c684c4ab5ce9fecae Mon Sep 17 00:00:00 2001
From: Vinzent Steinberg <Vinzent.Steinberg@gmail.com>
Date: Thu, 22 Feb 2018 15:18:42 +0100
Subject: [PATCH 1/4] Move benchmarks to criterion

---
 Cargo.toml         |  8 +++--
 benches/benches.rs | 76 ++++++++++++++++++++++++----------------------
 2 files changed, 46 insertions(+), 38 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 547d0f8..6235a58 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -3,14 +3,18 @@ name = "fixedbitset"
 version = "0.1.8"
 authors = ["bluss"]
 license = "MIT/Apache-2.0"
-
 description = "FixedBitSet is a simple bitset collection"
 documentation = "https://docs.rs/fixedbitset/"
 repository = "https://github.com/bluss/fixedbitset"
-
 keywords = ["container", "data-structure", "bitvec", "bitset"]
 categories = ["data-structures"]
 
 [package.metadata.release]
 no-dev-version = true
 
+[[bench]]
+name = "benches"
+harness = false
+
+[dev-dependencies]
+criterion = "0.2"
diff --git a/benches/benches.rs b/benches/benches.rs
index ac999b0..280538c 100644
--- a/benches/benches.rs
+++ b/benches/benches.rs
@@ -1,8 +1,8 @@
-#![feature(test)]
-
-extern crate test;
+#[macro_use]
+extern crate criterion;
 extern crate fixedbitset;
-use test::Bencher;
+
+use criterion::Criterion;
 use fixedbitset::{FixedBitSet};
 use std::mem::size_of;
 
@@ -31,103 +31,107 @@ fn iter_ones_using_slice_directly<F: FnMut(usize)>(fb: &FixedBitSet, f: &mut F)
     }
 }
 
-#[bench]
-fn bench_iter_ones_using_contains_all_zeros(b: &mut Bencher) {
+fn bench_iter_ones_using_contains_all_zeros(c: &mut Criterion) {
     const N: usize = 1_000_000;
     let fb = FixedBitSet::with_capacity(N);
 
-    b.iter(|| {
+    c.bench_function("ones using contains: all zeros", move |b| b.iter(|| {
         let mut count = 0;
         iter_ones_using_contains(&fb, &mut |_bit| count += 1);
         count
-    });
+    }));
 }
 
-#[bench]
-fn bench_iter_ones_using_contains_all_ones(b: &mut Bencher) {
+fn bench_iter_ones_using_contains_all_ones(c: &mut Criterion) {
     const N: usize = 1_000_000;
     let mut fb = FixedBitSet::with_capacity(N);
     fb.insert_range(..);
 
-    b.iter(|| {
+    c.bench_function("ones using contains: all ones", move |b| b.iter(|| {
         let mut count = 0;
         iter_ones_using_contains(&fb, &mut |_bit| count += 1);
         count
-    });
+    }));
 }
 
-#[bench]
-fn bench_iter_ones_using_slice_directly_all_zero(b: &mut Bencher) {
+fn bench_iter_ones_using_slice_directly_all_zeros(c: &mut Criterion) {
     const N: usize = 1_000_000;
     let fb = FixedBitSet::with_capacity(N);
 
-    b.iter(|| {
+    c.bench_function("ones using slice: all zeros", move |b| b.iter(|| {
        let mut count = 0;
        iter_ones_using_slice_directly(&fb, &mut |_bit| count += 1);
        count
-    });
+    }));
 }
 
-#[bench]
-fn bench_iter_ones_using_slice_directly_all_ones(b: &mut Bencher) {
+fn bench_iter_ones_using_slice_directly_all_ones(c: &mut Criterion) {
     const N: usize = 1_000_000;
     let mut fb = FixedBitSet::with_capacity(N);
     fb.insert_range(..);
 
-    b.iter(|| {
+    c.bench_function("ones using slice: all ones", move |b| b.iter(|| {
        let mut count = 0;
        iter_ones_using_slice_directly(&fb, &mut |_bit| count += 1);
        count
-    });
+    }));
 }
 
-#[bench]
-fn bench_iter_ones_all_zeros(b: &mut Bencher) {
+fn bench_iter_ones_all_zeros(c: &mut Criterion) {
     const N: usize = 1_000_000;
     let fb = FixedBitSet::with_capacity(N);
 
-    b.iter(|| {
+    c.bench_function("ones: all zeros", move |b| b.iter(|| {
         let mut count = 0;
         for _ in fb.ones() {
             count += 1;
         }
         count
-    });
+    }));
 }
 
-#[bench]
-fn bench_iter_ones_all_ones(b: &mut Bencher) {
+fn bench_iter_ones_all_ones(c: &mut Criterion) {
     const N: usize = 1_000_000;
     let mut fb = FixedBitSet::with_capacity(N);
     fb.insert_range(..);
 
-    b.iter(|| {
+    c.bench_function("ones: all ones", move |b| b.iter(|| {
         let mut count = 0;
         for _ in fb.ones() {
             count += 1;
         }
         count
-    });
+    }));
 }
 
-#[bench]
-fn bench_insert_range(b: &mut Bencher) {
+fn bench_insert_range(c: &mut Criterion) {
     const N: usize = 1_000_000;
     let mut fb = FixedBitSet::with_capacity(N);
 
-    b.iter(|| {
+    c.bench_function("insert range", move |b| b.iter(|| {
         fb.insert_range(..)
-    });
+    }));
 }
 
-#[bench]
-fn bench_insert_range_using_loop(b: &mut Bencher) {
+fn bench_insert_range_using_loop(c: &mut Criterion) {
     const N: usize = 1_000_000;
     let mut fb = FixedBitSet::with_capacity(N);
 
-    b.iter(|| {
+    c.bench_function("insert range using loop", move |b| b.iter(|| {
         for i in 0..N {
             fb.insert(i);
         }
-    });
+    }));
 }
+
+criterion_group!(benches,
+    bench_iter_ones_using_contains_all_zeros,
+    bench_iter_ones_using_contains_all_ones,
+    bench_iter_ones_using_slice_directly_all_zeros,
+    bench_iter_ones_using_slice_directly_all_ones,
+    bench_iter_ones_all_zeros,
+    bench_iter_ones_all_ones,
+    bench_insert_range,
+    bench_insert_range_using_loop
+);
+criterion_main!(benches);

From c04fe1242c72e579d4e8d8fb945e172055c9e436 Mon Sep 17 00:00:00 2001
From: Vinzent Steinberg <Vinzent.Steinberg@gmail.com>
Date: Thu, 22 Feb 2018 16:05:27 +0100
Subject: [PATCH 2/4] Bench equivalent functions together

---
 benches/benches.rs | 163 ++++++++++++++++++++++-----------------------
 1 file changed, 78 insertions(+), 85 deletions(-)

diff --git a/benches/benches.rs b/benches/benches.rs
index 280538c..67952cc 100644
--- a/benches/benches.rs
+++ b/benches/benches.rs
@@ -2,7 +2,7 @@
 extern crate criterion;
 extern crate fixedbitset;
 
-use criterion::Criterion;
+use criterion::{Criterion, Fun};
 use fixedbitset::{FixedBitSet};
 use std::mem::size_of;
 
@@ -31,107 +31,100 @@ fn iter_ones_using_slice_directly<F: FnMut(usize)>(fb: &FixedBitSet, f: &mut F)
     }
 }
 
-fn bench_iter_ones_using_contains_all_zeros(c: &mut Criterion) {
-    const N: usize = 1_000_000;
-    let fb = FixedBitSet::with_capacity(N);
-
-    c.bench_function("ones using contains: all zeros", move |b| b.iter(|| {
-        let mut count = 0;
-        iter_ones_using_contains(&fb, &mut |_bit| count += 1);
-        count
-    }));
-}
-
-fn bench_iter_ones_using_contains_all_ones(c: &mut Criterion) {
-    const N: usize = 1_000_000;
-    let mut fb = FixedBitSet::with_capacity(N);
-    fb.insert_range(..);
-
-    c.bench_function("ones using contains: all ones", move |b| b.iter(|| {
-        let mut count = 0;
-        iter_ones_using_contains(&fb, &mut |_bit| count += 1);
-        count
-    }));
-}
-
-fn bench_iter_ones_using_slice_directly_all_zeros(c: &mut Criterion) {
-    const N: usize = 1_000_000;
-    let fb = FixedBitSet::with_capacity(N);
-
-    c.bench_function("ones using slice: all zeros", move |b| b.iter(|| {
-       let mut count = 0;
-       iter_ones_using_slice_directly(&fb, &mut |_bit| count += 1);
-       count
-    }));
-}
-
-fn bench_iter_ones_using_slice_directly_all_ones(c: &mut Criterion) {
-    const N: usize = 1_000_000;
-    let mut fb = FixedBitSet::with_capacity(N);
-    fb.insert_range(..);
-
-    c.bench_function("ones using slice: all ones", move |b| b.iter(|| {
-       let mut count = 0;
-       iter_ones_using_slice_directly(&fb, &mut |_bit| count += 1);
-       count
-    }));
-}
-
 fn bench_iter_ones_all_zeros(c: &mut Criterion) {
     const N: usize = 1_000_000;
     let fb = FixedBitSet::with_capacity(N);
 
-    c.bench_function("ones: all zeros", move |b| b.iter(|| {
-        let mut count = 0;
-        for _ in fb.ones() {
-            count += 1;
-        }
-        count
-    }));
+    let default = {
+        let fb = fb.clone();
+        Fun::new("default", move |b, _| b.iter(|| {
+            let mut count = 0;
+            for _ in fb.ones() {
+                count += 1;
+            }
+            count
+        }))
+    };
+    let contains = {
+        let fb = fb.clone();
+        Fun::new("contains", move |b, _| b.iter(|| {
+            let mut count = 0;
+            iter_ones_using_contains(&fb, &mut |_bit| count += 1);
+            count
+        }))
+    };
+    let slice = {
+       let fb = fb.clone();
+       Fun::new("slice directly", move |b, _| b.iter(|| {
+           let mut count = 0;
+           iter_ones_using_slice_directly(&fb, &mut |_bit| count += 1);
+           count
+       }))
+    };
+    c.bench_functions("iter ones: all zeros", vec![default, contains, slice], ());
 }
 
 fn bench_iter_ones_all_ones(c: &mut Criterion) {
     const N: usize = 1_000_000;
-    let mut fb = FixedBitSet::with_capacity(N);
-    fb.insert_range(..);
-
-    c.bench_function("ones: all ones", move |b| b.iter(|| {
-        let mut count = 0;
-        for _ in fb.ones() {
-            count += 1;
-        }
-        count
-    }));
+    let fb = {
+        let mut tmp = FixedBitSet::with_capacity(N);
+        tmp.insert_range(..);
+        tmp
+    };
+
+    let default = {
+        let fb = fb.clone();
+        Fun::new("default", move |b, _| b.iter(|| {
+            let mut count = 0;
+            for _ in fb.ones() {
+                count += 1;
+            }
+            count
+        }))
+    };
+    let contains = {
+        let fb = fb.clone();
+        Fun::new("contains", move |b, _| b.iter(|| {
+            let mut count = 0;
+            iter_ones_using_contains(&fb, &mut |_bit| count += 1);
+            count
+        }))
+    };
+    let slice = {
+       let fb = fb.clone();
+       Fun::new("slice directly", move |b, _| b.iter(|| {
+           let mut count = 0;
+           iter_ones_using_slice_directly(&fb, &mut |_bit| count += 1);
+           count
+       }))
+    };
+    c.bench_functions("iter ones: all ones", vec![default, contains, slice], ());
 }
 
 fn bench_insert_range(c: &mut Criterion) {
     const N: usize = 1_000_000;
-    let mut fb = FixedBitSet::with_capacity(N);
-
-    c.bench_function("insert range", move |b| b.iter(|| {
-        fb.insert_range(..)
-    }));
-}
-
-fn bench_insert_range_using_loop(c: &mut Criterion) {
-    const N: usize = 1_000_000;
-    let mut fb = FixedBitSet::with_capacity(N);
+    let fb = FixedBitSet::with_capacity(N);
 
-    c.bench_function("insert range using loop", move |b| b.iter(|| {
-        for i in 0..N {
-            fb.insert(i);
-        }
-    }));
+    let default = {
+        let mut fb = fb.clone();
+        Fun::new("default", move |b, _| b.iter(|| {
+            fb.insert_range(..)
+        }))
+    };
+    let loop_ = {
+        let mut fb = fb.clone();
+        Fun::new("loop", move |b, _| b.iter(|| {
+            for i in 0..N {
+                fb.insert(i);
+            }
+        }))
+    };
+    c.bench_functions("insert range", vec![default, loop_], ());
 }
 
 criterion_group!(benches,
-    bench_iter_ones_using_contains_all_zeros,
-    bench_iter_ones_using_contains_all_ones,
-    bench_iter_ones_using_slice_directly_all_zeros,
-    bench_iter_ones_using_slice_directly_all_ones,
     bench_iter_ones_all_zeros,
     bench_iter_ones_all_ones,
-    bench_insert_range,
-    bench_insert_range_using_loop
+    bench_insert_range
 );
 criterion_main!(benches);

From 855ad8fb1b48286fa0233ed778fafc92daea68f8 Mon Sep 17 00:00:00 2001
From: Vinzent Steinberg <Vinzent.Steinberg@gmail.com>
Date: Thu, 22 Feb 2018 16:14:51 +0100
Subject: [PATCH 3/4] Reduce code duplication in benchmarks

---
 benches/benches.rs | 49 +++++++++++-----------------------------------
 1 file changed, 11 insertions(+), 38 deletions(-)

diff --git a/benches/benches.rs b/benches/benches.rs
index 67952cc..c56bb9a 100644
--- a/benches/benches.rs
+++ b/benches/benches.rs
@@ -31,10 +31,7 @@ fn iter_ones_using_slice_directly<F: FnMut(usize)>(fb: &FixedBitSet, f: &mut F)
     }
 }
 
-fn bench_iter_ones_all_zeros(c: &mut Criterion) {
-    const N: usize = 1_000_000;
-    let fb = FixedBitSet::with_capacity(N);
-
+fn make_bench_iter_ones(fb: &FixedBitSet) -> Vec<Fun<()>> {
     let default = {
         let fb = fb.clone();
         Fun::new("default", move |b, _| b.iter(|| {
@@ -61,44 +58,20 @@ fn bench_iter_ones_all_zeros(c: &mut Criterion) {
            count
        }))
     };
-    c.bench_functions("iter ones: all zeros", vec![default, contains, slice], ());
+    vec![default, contains, slice]
 }
 
-fn bench_iter_ones_all_ones(c: &mut Criterion) {
+fn bench_iter_ones_all_zeros(c: &mut Criterion) {
     const N: usize = 1_000_000;
-    let fb = {
-        let mut tmp = FixedBitSet::with_capacity(N);
-        tmp.insert_range(..);
-        tmp
-    };
+    let fb = FixedBitSet::with_capacity(N);
+    c.bench_functions("iter ones: all zeros", make_bench_iter_ones(&fb), ());
+}
 
-    let default = {
-        let fb = fb.clone();
-        Fun::new("default", move |b, _| b.iter(|| {
-            let mut count = 0;
-            for _ in fb.ones() {
-                count += 1;
-            }
-            count
-        }))
-    };
-    let contains = {
-        let fb = fb.clone();
-        Fun::new("contains", move |b, _| b.iter(|| {
-            let mut count = 0;
-            iter_ones_using_contains(&fb, &mut |_bit| count += 1);
-            count
-        }))
-    };
-    let slice = {
-       let fb = fb.clone();
-       Fun::new("slice directly", move |b, _| b.iter(|| {
-           let mut count = 0;
-           iter_ones_using_slice_directly(&fb, &mut |_bit| count += 1);
-           count
-       }))
-    };
-    c.bench_functions("iter ones: all ones", vec![default, contains, slice], ());
+fn bench_iter_ones_all_ones(c: &mut Criterion) {
+    const N: usize = 1_000_000;
+    let mut fb = FixedBitSet::with_capacity(N);
+    fb.insert_range(..);
+    c.bench_functions("iter ones: all ones", make_bench_iter_ones(&fb), ());
 }
 
 fn bench_insert_range(c: &mut Criterion) {

From 2b9a26e3d372509510d4469ad3b231f66c3645ae Mon Sep 17 00:00:00 2001
From: Vinzent Steinberg <Vinzent.Steinberg@gmail.com>
Date: Thu, 22 Feb 2018 18:30:58 +0100
Subject: [PATCH 4/4] Benchmark for iterating over random bitset

---
 Cargo.toml         |  2 ++
 benches/benches.rs | 20 +++++++++++++++++++-
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/Cargo.toml b/Cargo.toml
index 6235a58..596f27f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -17,4 +17,6 @@ name = "benches"
 harness = false
 
 [dev-dependencies]
+byteorder = "1"
 criterion = "0.2"
+rand = "0.4"
diff --git a/benches/benches.rs b/benches/benches.rs
index c56bb9a..6b24638 100644
--- a/benches/benches.rs
+++ b/benches/benches.rs
@@ -1,10 +1,13 @@
+extern crate byteorder;
 #[macro_use]
 extern crate criterion;
+extern crate rand;
 extern crate fixedbitset;
 
+use std::mem::size_of;
 use criterion::{Criterion, Fun};
+use rand::Rng;
 use fixedbitset::{FixedBitSet};
-use std::mem::size_of;
 
 #[inline]
 fn iter_ones_using_contains<F: FnMut(usize)>(fb: &FixedBitSet, f: &mut F) {
@@ -74,6 +77,20 @@ fn bench_iter_ones_all_ones(c: &mut Criterion) {
     c.bench_functions("iter ones: all ones", make_bench_iter_ones(&fb), ());
 }
 
+fn bench_iter_ones_random(c: &mut Criterion) {
+    const N: usize = 15625 * 2 * 32;
+    let mut fb = FixedBitSet::with_capacity(N);
+    let mut rng = rand::thread_rng();
+    {
+        let p = fb.as_mut_slice();
+        for w in p {
+            *w = rng.next_u32();
+        }
+    }
+    assert!(fb.count_ones(..) > 10);
+    c.bench_functions("iter ones: random", make_bench_iter_ones(&fb), ());
+}
+
 fn bench_insert_range(c: &mut Criterion) {
     const N: usize = 1_000_000;
     let fb = FixedBitSet::with_capacity(N);
@@ -98,6 +115,7 @@ fn bench_insert_range(c: &mut Criterion) {
 criterion_group!(benches,
     bench_iter_ones_all_zeros,
     bench_iter_ones_all_ones,
+    bench_iter_ones_random,
     bench_insert_range
 );
 criterion_main!(benches);