Compare commits
2 Commits
4156be9a7a
...
9c782b4391
| Author | SHA1 | Date | |
|---|---|---|---|
9c782b4391
|
|||
| 22f12c27d1 |
56
Cargo.lock
generated
56
Cargo.lock
generated
@ -71,12 +71,24 @@ version = "0.3.55"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2"
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"libc",
|
||||
"wasi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hash_test"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"fasthash",
|
||||
"hashbrown",
|
||||
"rand 0.8.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -110,6 +122,12 @@ version = "1.18.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.4.6"
|
||||
@ -123,6 +141,27 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rand_chacha",
|
||||
"rand_core 0.6.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core 0.6.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.3.1"
|
||||
@ -138,6 +177,15 @@ version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rdrand"
|
||||
version = "0.4.0"
|
||||
@ -159,6 +207,12 @@ version = "0.9.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.11.0+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
@ -187,5 +241,5 @@ version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e0eeda34baec49c4f1eb2c04d59b761582fd6330010f9330ca696ca1a355dfcd"
|
||||
dependencies = [
|
||||
"rand",
|
||||
"rand 0.4.6",
|
||||
]
|
||||
|
||||
@ -9,6 +9,7 @@ edition = "2018"
|
||||
[dependencies]
|
||||
hashbrown = "0.14"
|
||||
fasthash = "0.4.0"
|
||||
rand = "0.8.5"
|
||||
|
||||
[profile.release]
|
||||
opt-level = 3
|
||||
|
||||
45
src/main.rs
45
src/main.rs
@ -1,19 +1,23 @@
|
||||
use fasthash::spooky;
|
||||
use hashbrown::HashSet;
|
||||
use std::fs::File;
|
||||
use std::io::{BufReader, BufRead};
|
||||
use rand::prelude::*;
|
||||
use std::time::Instant;
|
||||
|
||||
const CAP: i32 = 1000000;
|
||||
|
||||
fn main() {
|
||||
let arg: Vec<String> = std::env::args().collect();
|
||||
if arg.len() != 2 {
|
||||
eprintln!("{} numbers.txt", std::env::args().nth(0).unwrap());
|
||||
std::process::exit(1);
|
||||
let mut v: Vec<i32> = Vec::with_capacity(CAP as usize);
|
||||
let mut rng = thread_rng();
|
||||
|
||||
let mut i = 0i32;
|
||||
loop {
|
||||
if i == CAP {
|
||||
break;
|
||||
}
|
||||
i += 1;
|
||||
|
||||
v.push(rng.gen_range(-CAP..CAP));
|
||||
}
|
||||
let name = &arg[1];
|
||||
let f = File::open(name).expect("file can't open");
|
||||
let reader = BufReader::new(f);
|
||||
let v: Vec<i32> = reader.lines().map(|x| x.unwrap().parse::<i32>().unwrap()).collect();
|
||||
|
||||
{
|
||||
let mut h = HashSet::with_hasher(spooky::Hash64);
|
||||
@ -28,7 +32,6 @@ fn main() {
|
||||
bench_std(&v, &mut h, true);
|
||||
bench_std(&v, &mut h, false);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
fn bench_swiss(v: &Vec<i32>, h: &mut HashSet<i32, spooky::Hash64>, ins: bool) {
|
||||
@ -36,16 +39,20 @@ fn bench_swiss(v: &Vec<i32>, h: &mut HashSet<i32, spooky::Hash64>, ins: bool) {
|
||||
|
||||
if ins {
|
||||
for i in v {
|
||||
h.insert(*i);
|
||||
h.insert(*i);
|
||||
}
|
||||
} else {
|
||||
for i in v {
|
||||
h.contains(i);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
let elapsed = now.elapsed().as_millis();
|
||||
println!("swiss {} => {}ms", if ins { "insert" } else { "search" }, elapsed);
|
||||
println!(
|
||||
"swiss {} => {}ms",
|
||||
if ins { "insert" } else { "search" },
|
||||
elapsed
|
||||
);
|
||||
}
|
||||
|
||||
fn bench_std(v: &Vec<i32>, h: &mut std::collections::HashSet<i32, spooky::Hash64>, ins: bool) {
|
||||
@ -53,14 +60,18 @@ fn bench_std(v: &Vec<i32>, h: &mut std::collections::HashSet<i32, spooky::Hash64
|
||||
|
||||
if ins {
|
||||
for i in v {
|
||||
h.insert(*i);
|
||||
h.insert(*i);
|
||||
}
|
||||
} else {
|
||||
for i in v {
|
||||
h.contains(i);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
let elapsed = now.elapsed().as_millis();
|
||||
println!("std {} => {}ms", if ins { "insert" } else { "search" }, elapsed);
|
||||
println!(
|
||||
"std {} => {}ms",
|
||||
if ins { "insert" } else { "search" },
|
||||
elapsed
|
||||
);
|
||||
}
|
||||
|
||||
155
swisstable -- a joke.md
Executable file
155
swisstable -- a joke.md
Executable file
@ -0,0 +1,155 @@
|
||||
## swisstable -- a joke
|
||||
|
||||
`swisstable`是`abseil-cpp`中`flat_hash_set/map`中使用的hash表实现，使用的是二次探测法，号称性能优于STL，下面讲讲这个`swisstable`是怎么回事
|
||||
|
||||
|
||||
|
||||
这个hash表使用了一个`group`的概念，将一块连续的内存分为若干个`group`，每个`group`有16个元素包括`meta`和`slot`两部分，其中`meta`存放64位的`hash`值的7位（称为h2)，`slot`用于存放Key，Key通过`hash`值的57位（称为h1）来索引，使用SSE2同时比较128位的ctrl值，快速找到Key在`group`中的偏移，这也是为什么是16个`meta`是一组（128 / 8 = 16），而`meta`又有三个特殊值`-1`、`-2`和`-128`分别对应，sentinel、invalid（delete）和empty，一开始所有`meta`被初始化位`empty`
|
||||
|
||||
|
||||
|
||||
查找过程：
|
||||
|
||||
1. 计算hash，得到h1和h2
|
||||
|
||||
2. 通过h2得到起始的`group`
|
||||
|
||||
3. 使用h2和这一`group`的`meta`匹配 （使用SSE2）
|
||||
|
||||
4. 若匹配，返回到起始位置的offset
|
||||
|
||||
5. 若匹配到 `empty` 则返回不存在
|
||||
|
||||
代码如下
|
||||
|
||||
```c++
|
||||
auto pattern = _mm_set1_epi8(H2(hash)); // 1
|
||||
prober seq { H1(hash), groups_ - 1 }; // 1
|
||||
matcher m { 0 };
|
||||
|
||||
while (true) {
|
||||
group g { ctrl_ + seq.offset() }; // 2
|
||||
m = g.match(pattern); // 3
|
||||
|
||||
while (m) {
|
||||
offset = seq.offset(m.index());
|
||||
assert(offset < cap_);
|
||||
if (likely(slot_[offset] == key)) // 4
|
||||
return true;
|
||||
++m;
|
||||
}
|
||||
if (likely(g.match_empty())) // 5
|
||||
return false;
|
||||
seq.next();
|
||||
}
|
||||
```
|
||||
|
||||
这个`swisstable`宣传查找比STL要快，但不管是使用`absl::flat_hash_set`的官方实现，还是本人的实现，结果都不如STL（少数时候能比STL快一点点）
|
||||
|
||||
```shell
|
||||
abby@Serenity ~/s/build> ./bench2
|
||||
----------- insert ------------
|
||||
unordered_set => 183.605811ms
|
||||
flat_hash_set => 41.932002ms
|
||||
----------- search ------------
|
||||
unordered_set => 4.378100ms
|
||||
flat_hash_set => 22.928702ms
|
||||
unordered_set cap 1056323 size 787268 load_factor 0.745291
|
||||
flat_hash_set cap 1048575 size 787268 load_factor 0.750798
|
||||
abby@Serenity ~/s/build> ./bench2
|
||||
----------- insert ------------
|
||||
unordered_set => 182.890869ms
|
||||
flat_hash_set => 39.940193ms
|
||||
----------- search ------------
|
||||
unordered_set => 4.542800ms
|
||||
flat_hash_set => 21.498796ms
|
||||
unordered_set cap 1056323 size 786505 load_factor 0.744569
|
||||
flat_hash_set cap 1048575 size 786505 load_factor 0.750070
|
||||
abby@Serenity ~/s/build> ./bench2
|
||||
----------- insert ------------
|
||||
unordered_set => 183.804469ms
|
||||
flat_hash_set => 38.301994ms
|
||||
----------- search ------------
|
||||
unordered_set => 5.001399ms
|
||||
flat_hash_set => 22.470496ms
|
||||
unordered_set cap 1056323 size 787110 load_factor 0.745141
|
||||
flat_hash_set cap 1048575 size 787110 load_factor 0.750647
|
||||
abby@Serenity ~/s/build> ./bench
|
||||
----------- insert ------------
|
||||
unordered_set => 187.271974ms
|
||||
swiss => 22.850596ms
|
||||
----------- search ------------
|
||||
unordered_set => 4.307000ms
|
||||
swiss => 4.016499ms
|
||||
unordered_set cap 1056323 size 786815 load_factor 0.744862
|
||||
swiss cap 1048576 size 786815 load_factor 0.750365
|
||||
abby@Serenity ~/s/build> ./bench
|
||||
----------- insert ------------
|
||||
unordered_set => 197.831773ms
|
||||
swiss => 20.787797ms
|
||||
----------- search ------------
|
||||
unordered_set => 4.811299ms
|
||||
swiss => 4.107399ms
|
||||
unordered_set cap 1056323 size 787381 load_factor 0.745398
|
||||
swiss cap 1048576 size 787381 load_factor 0.750905
|
||||
```
|
||||
|
||||
|
||||
|
||||
据说rust也有使用了swisstable实现，实际测试如下
|
||||
|
||||
```shell
|
||||
abby@Serenity ~/hash_test> cargo run --release ~/numbers.txt
|
||||
Compiling libc v0.2.148
|
||||
Compiling gcc v0.3.55
|
||||
Compiling autocfg v1.1.0
|
||||
Compiling version_check v0.9.4
|
||||
Compiling ahash v0.8.3
|
||||
Compiling num-traits v0.2.16
|
||||
Compiling cfg-if v1.0.0
|
||||
Compiling once_cell v1.18.0
|
||||
Compiling fasthash-sys v0.3.2
|
||||
Compiling cfg-if v0.1.10
|
||||
Compiling rand v0.4.6
|
||||
Compiling seahash v3.0.7
|
||||
Compiling allocator-api2 v0.2.16
|
||||
Compiling xoroshiro128 v0.3.0
|
||||
Compiling hashbrown v0.14.1
|
||||
Compiling fasthash v0.4.0
|
||||
Compiling hash_test v0.1.0 (/home/abby/hash_test)
|
||||
Finished release [optimized] target(s) in 12.99s
|
||||
Running `target/release/hash_test /home/abby/numbers.txt`
|
||||
swiss insert => 73ms
|
||||
swiss search => 57ms
|
||||
std insert => 76ms
|
||||
std search => 69ms
|
||||
abby@Serenity ~/hash_test> cargo run --release ~/numbers.txt
|
||||
Finished release [optimized] target(s) in 0.01s
|
||||
Running `target/release/hash_test /home/abby/numbers.txt`
|
||||
swiss insert => 71ms
|
||||
swiss search => 56ms
|
||||
std insert => 71ms
|
||||
std search => 55ms
|
||||
abby@Serenity ~/hash_test> cargo run --release ~/numbers.txt
|
||||
Finished release [optimized] target(s) in 0.02s
|
||||
Running `target/release/hash_test /home/abby/numbers.txt`
|
||||
swiss insert => 73ms
|
||||
swiss search => 78ms
|
||||
std insert => 72ms
|
||||
std search => 61ms
|
||||
abby@Serenity ~/hash_test>
|
||||
```
|
||||
|
||||
结论：
|
||||
|
||||
为什么，swisstable实现会比STL的慢，可能时因为这里面的实现`meta`和`slot`是分开存放的，类似
|
||||
|
||||
```
|
||||
| meta | slot |
|
||||
+-------------------+----------------------------------------+
|
||||
```
|
||||
|
||||
导致在查找时需要在`meta`和`slot`之间来回跳转，对缓存不友好
|
||||
|
||||
如果将`meta`和`slot`做到和 L1 cache对齐，效果应该要好一些
|
||||
|
||||
Loading…
Reference in New Issue
Block a user