use rand instead of file

Cargo.lock

@@ -71,12 +71,24 @@ version = "0.3.55"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2"
 
+[[package]]
+name = "getrandom"
+version = "0.2.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
+dependencies = [
+ "cfg-if 1.0.0",
+ "libc",
+ "wasi",
+]
+
 [[package]]
 name = "hash_test"
 version = "0.1.0"
 dependencies = [
  "fasthash",
  "hashbrown",
+ "rand 0.8.5",
 ]
 
 [[package]]
@@ -110,6 +122,12 @@ version = "1.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
 
+[[package]]
+name = "ppv-lite86"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
+
 [[package]]
 name = "rand"
 version = "0.4.6"
@@ -123,6 +141,27 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "rand"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
+dependencies = [
+ "libc",
+ "rand_chacha",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.6.4",
+]
+
 [[package]]
 name = "rand_core"
 version = "0.3.1"
@@ -138,6 +177,15 @@ version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
 
+[[package]]
+name = "rand_core"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
+dependencies = [
+ "getrandom",
+]
+
 [[package]]
 name = "rdrand"
 version = "0.4.0"
@@ -159,6 +207,12 @@ version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
+[[package]]
+name = "wasi"
+version = "0.11.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+
 [[package]]
 name = "winapi"
 version = "0.3.9"
@@ -187,5 +241,5 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e0eeda34baec49c4f1eb2c04d59b761582fd6330010f9330ca696ca1a355dfcd"
 dependencies = [
- "rand",
+ "rand 0.4.6",
 ]

Cargo.toml

@@ -9,6 +9,7 @@ edition = "2018"
 [dependencies]
 hashbrown = "0.14"
 fasthash = "0.4.0"
+rand = "0.8.5"
 
 [profile.release]
 opt-level = 3

src/main.rs

@@ -1,19 +1,23 @@
 use fasthash::spooky;
 use hashbrown::HashSet;
-use std::fs::File;
+use rand::prelude::*;
-use std::io::{BufReader, BufRead};
 use std::time::Instant;
 
+const CAP: i32 = 1000000;
+
 fn main() {
-    let arg: Vec<String> = std::env::args().collect();
+    let mut v: Vec<i32> = Vec::with_capacity(CAP as usize);
-    if arg.len() != 2 {
+    let mut rng = thread_rng();
-        eprintln!("{} numbers.txt", std::env::args().nth(0).unwrap());
+
-        std::process::exit(1);
+    let mut i = 0i32;
+    loop {
+        if i == CAP {
+            break;
+        }
+        i += 1;
+
+        v.push(rng.gen_range(-CAP..CAP));
     }
-    let name = &arg[1];
-    let f = File::open(name).expect("file can't open");
-    let reader = BufReader::new(f);
-    let v: Vec<i32> = reader.lines().map(|x| x.unwrap().parse::<i32>().unwrap()).collect();
 
     {
         let mut h = HashSet::with_hasher(spooky::Hash64);
@@ -28,7 +32,6 @@ fn main() {
         bench_std(&v, &mut h, true);
         bench_std(&v, &mut h, false);
     }
-
 }
 
 fn bench_swiss(v: &Vec<i32>, h: &mut HashSet<i32, spooky::Hash64>, ins: bool) {
@@ -45,7 +48,11 @@ fn bench_swiss(v: &Vec<i32>, h: &mut HashSet<i32, spooky::Hash64>, ins: bool) {
     }
 
     let elapsed = now.elapsed().as_millis();
-    println!("swiss {} => {}ms", if ins { "insert" } else { "search" }, elapsed);
+    println!(
+        "swiss {} => {}ms",
+        if ins { "insert" } else { "search" },
+        elapsed
+    );
 }
 
 fn bench_std(v: &Vec<i32>, h: &mut std::collections::HashSet<i32, spooky::Hash64>, ins: bool) {
@@ -62,5 +69,9 @@ fn bench_std(v: &Vec<i32>, h: &mut std::collections::HashSet<i32, spooky::Hash64
     }
 
     let elapsed = now.elapsed().as_millis();
-    println!("std  {} => {}ms", if ins { "insert" } else { "search" }, elapsed);
+    println!(
+        "std  {} => {}ms",
+        if ins { "insert" } else { "search" },
+        elapsed
+    );
 }

swisstable -- a joke.md

@@ -0,0 +1,155 @@
+## swisstable -- a joke
+
+`swisstable`是`abseil-cpp`中`flat_hash_set/map`中使用的hash表实现，使用的是二次探测法，号称性能优于STL，下面讲讲这个`swisstable`是怎么回事
+
+
+
+这个hash表使用了一个`group`的概念，将一块连续的内存分为若干个`group`，每个`group`有16个元素包括`meta`和`slot`两部分，其中`meta`存放64位的`hash`值的7位（称为h2)，`slot`用于存放Key，Key通过`hash`值的57位（称为h1）来索引，使用SSE2同时比较128位的ctrl值，快速找到Key在`group`中的偏移，这也是为什么是16个`meta`是一组（128 / 8 = 16），而`meta`又有三个特殊值`-1`、`-2`和`-128`分别对应，sentinel、invalid（delete）和empty，一开始所有`meta`被初始化位`empty`
+
+
+
+查找过程：
+
+1. 计算hash，得到h1和h2
+
+2. 通过h2得到起始的`group`
+
+3. 使用h2和这一`group`的`meta`匹配 （使用SSE2）
+
+4. 若匹配，返回到起始位置的offset
+
+5. 若匹配到 `empty` 则返回不存在
+
+代码如下
+
+```c++
+auto pattern = _mm_set1_epi8(H2(hash)); // 1
+prober seq { H1(hash), groups_ - 1 }; // 1
+matcher m { 0 };
+
+while (true) {
+    group g { ctrl_ + seq.offset() }; // 2
+    m = g.match(pattern); // 3
+
+    while (m) {
+        offset = seq.offset(m.index());
+        assert(offset < cap_);
+        if (likely(slot_[offset] == key)) // 4
+            return true;
+        ++m;
+    }
+    if (likely(g.match_empty())) // 5
+        return false;
+    seq.next();
+}
+```
+
+这个`swisstable`宣传查找比STL要快，但不管是使用`absl::flat_hash_set`的官方实现，还是本人的实现，结果都不如STL（少数时候能比STL快一点点）
+
+```shell
+abby@Serenity ~/s/build> ./bench2
+----------- insert ------------
+  unordered_set => 183.605811ms
+  flat_hash_set => 41.932002ms
+----------- search ------------
+  unordered_set => 4.378100ms
+  flat_hash_set => 22.928702ms
+unordered_set cap 1056323 size 787268 load_factor 0.745291
+flat_hash_set cap 1048575 size 787268 load_factor 0.750798
+abby@Serenity ~/s/build> ./bench2
+----------- insert ------------
+  unordered_set => 182.890869ms
+  flat_hash_set => 39.940193ms
+----------- search ------------
+  unordered_set => 4.542800ms
+  flat_hash_set => 21.498796ms
+unordered_set cap 1056323 size 786505 load_factor 0.744569
+flat_hash_set cap 1048575 size 786505 load_factor 0.750070
+abby@Serenity ~/s/build> ./bench2
+----------- insert ------------
+  unordered_set => 183.804469ms
+  flat_hash_set => 38.301994ms
+----------- search ------------
+  unordered_set => 5.001399ms
+  flat_hash_set => 22.470496ms
+unordered_set cap 1056323 size 787110 load_factor 0.745141
+flat_hash_set cap 1048575 size 787110 load_factor 0.750647
+abby@Serenity ~/s/build> ./bench
+----------- insert ------------
+  unordered_set => 187.271974ms
+          swiss => 22.850596ms
+----------- search ------------
+  unordered_set => 4.307000ms
+          swiss => 4.016499ms
+unordered_set cap 1056323 size 786815 load_factor 0.744862
+swiss         cap 1048576 size 786815 load_factor 0.750365
+abby@Serenity ~/s/build> ./bench
+----------- insert ------------
+  unordered_set => 197.831773ms
+          swiss => 20.787797ms
+----------- search ------------
+  unordered_set => 4.811299ms
+          swiss => 4.107399ms
+unordered_set cap 1056323 size 787381 load_factor 0.745398
+swiss         cap 1048576 size 787381 load_factor 0.750905
+```
+
+
+
+据说rust也有使用了swisstable实现，实际测试如下
+
+```shell
+abby@Serenity ~/hash_test> cargo run --release ~/numbers.txt
+   Compiling libc v0.2.148
+   Compiling gcc v0.3.55
+   Compiling autocfg v1.1.0
+   Compiling version_check v0.9.4
+   Compiling ahash v0.8.3
+   Compiling num-traits v0.2.16
+   Compiling cfg-if v1.0.0
+   Compiling once_cell v1.18.0
+   Compiling fasthash-sys v0.3.2
+   Compiling cfg-if v0.1.10
+   Compiling rand v0.4.6
+   Compiling seahash v3.0.7
+   Compiling allocator-api2 v0.2.16
+   Compiling xoroshiro128 v0.3.0
+   Compiling hashbrown v0.14.1
+   Compiling fasthash v0.4.0
+   Compiling hash_test v0.1.0 (/home/abby/hash_test)
+    Finished release [optimized] target(s) in 12.99s
+     Running `target/release/hash_test /home/abby/numbers.txt`
+swiss insert => 73ms
+swiss search => 57ms
+std  insert => 76ms
+std  search => 69ms
+abby@Serenity ~/hash_test> cargo run --release ~/numbers.txt
+    Finished release [optimized] target(s) in 0.01s
+     Running `target/release/hash_test /home/abby/numbers.txt`
+swiss insert => 71ms
+swiss search => 56ms
+std  insert => 71ms
+std  search => 55ms
+abby@Serenity ~/hash_test> cargo run --release ~/numbers.txt
+    Finished release [optimized] target(s) in 0.02s
+     Running `target/release/hash_test /home/abby/numbers.txt`
+swiss insert => 73ms
+swiss search => 78ms
+std  insert => 72ms
+std  search => 61ms
+abby@Serenity ~/hash_test>
+```
+
+结论：
+
+为什么，swisstable实现会比STL的慢，可能时因为这里面的实现`meta`和`slot`是分开存放的，类似
+
+```
+| meta              |  slot                                  |
++-------------------+----------------------------------------+
+```
+
+导致在查找时需要在`meta`和`slot`之间来回跳转，对缓存不友好
+
+如果将`meta`和`slot`做到和 L1 cache对齐，效果应该要好一些
+