adpat rocksdb
This commit is contained in:
parent
5796c5aa3e
commit
c9b915cd49
4
Cargo.lock
generated
4
Cargo.lock
generated
@ -192,7 +192,6 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
|||||||
[[package]]
|
[[package]]
|
||||||
name = "io"
|
name = "io"
|
||||||
version = "0.0.2"
|
version = "0.0.2"
|
||||||
source = "git+https://git.o2c.fun/abby/mace.git?branch=task-64#0a5ebdbdca9e1097da83e600a788288bfa4f6b85"
|
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
@ -236,8 +235,7 @@ checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mace"
|
name = "mace"
|
||||||
version = "0.0.9"
|
version = "0.0.10"
|
||||||
source = "git+https://git.o2c.fun/abby/mace.git?branch=task-64#0a5ebdbdca9e1097da83e600a788288bfa4f6b85"
|
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"crc32c",
|
"crc32c",
|
||||||
"crossbeam-epoch",
|
"crossbeam-epoch",
|
||||||
|
|||||||
@ -4,10 +4,17 @@ version = "0.1.0"
|
|||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
mace = { git = "https://git.o2c.fun/abby/mace.git", branch = "task-64" }
|
# mace = { git = "https://git.o2c.fun/abby/mace.git", branch = "task-64-1" }
|
||||||
|
mace = { path = "/home/workspace/gits/github/mace"}
|
||||||
clap = { version = "4.5.42", features = ["derive"] }
|
clap = { version = "4.5.42", features = ["derive"] }
|
||||||
rand = "0.9.2"
|
rand = "0.9.2"
|
||||||
|
|
||||||
[profile.release]
|
[profile.release]
|
||||||
lto = true
|
lto = true
|
||||||
opt-level = 3
|
opt-level = 3
|
||||||
|
|
||||||
|
[profile.release-with-symbol]
|
||||||
|
inherits = "release"
|
||||||
|
# debug = true
|
||||||
|
debug = "line-tables-only"
|
||||||
|
panic = "abort"
|
||||||
|
|||||||
58
rocksdb/.clang-format
Normal file
58
rocksdb/.clang-format
Normal file
@ -0,0 +1,58 @@
|
|||||||
|
# Generated from CLion C/C++ Code Style settings
|
||||||
|
---
|
||||||
|
Language: Cpp
|
||||||
|
BasedOnStyle: LLVM
|
||||||
|
AccessModifierOffset: -4
|
||||||
|
AlignConsecutiveAssignments: false
|
||||||
|
AlignConsecutiveDeclarations: false
|
||||||
|
AlignOperands: true
|
||||||
|
AlignTrailingComments: false
|
||||||
|
AlwaysBreakTemplateDeclarations: Yes
|
||||||
|
BraceWrapping:
|
||||||
|
AfterCaseLabel: false
|
||||||
|
AfterClass: false
|
||||||
|
AfterControlStatement: false
|
||||||
|
AfterEnum: false
|
||||||
|
AfterFunction: false
|
||||||
|
AfterNamespace: false
|
||||||
|
AfterStruct: false
|
||||||
|
AfterUnion: false
|
||||||
|
AfterExternBlock: false
|
||||||
|
BeforeCatch: false
|
||||||
|
BeforeElse: false
|
||||||
|
BeforeLambdaBody: false
|
||||||
|
BeforeWhile: false
|
||||||
|
SplitEmptyFunction: true
|
||||||
|
SplitEmptyRecord: true
|
||||||
|
SplitEmptyNamespace: true
|
||||||
|
BreakBeforeBraces: Custom
|
||||||
|
BreakConstructorInitializers: AfterColon
|
||||||
|
BreakConstructorInitializersBeforeComma: false
|
||||||
|
ColumnLimit: 120
|
||||||
|
ConstructorInitializerAllOnOneLineOrOnePerLine: false
|
||||||
|
ContinuationIndentWidth: 8
|
||||||
|
IncludeCategories:
|
||||||
|
- Regex: '^<.*'
|
||||||
|
Priority: 1
|
||||||
|
- Regex: '^".*'
|
||||||
|
Priority: 2
|
||||||
|
- Regex: '.*'
|
||||||
|
Priority: 3
|
||||||
|
IncludeIsMainRegex: '([-_](test|unittest))?$'
|
||||||
|
IndentCaseLabels: true
|
||||||
|
IndentWidth: 4
|
||||||
|
InsertNewlineAtEOF: true
|
||||||
|
MacroBlockBegin: ''
|
||||||
|
MacroBlockEnd: ''
|
||||||
|
MaxEmptyLinesToKeep: 2
|
||||||
|
NamespaceIndentation: All
|
||||||
|
SpaceAfterCStyleCast: true
|
||||||
|
SpaceAfterTemplateKeyword: false
|
||||||
|
SpaceBeforeRangeBasedForLoopColon: false
|
||||||
|
SpaceInEmptyParentheses: false
|
||||||
|
SpacesInAngles: false
|
||||||
|
SpacesInConditionalStatement: false
|
||||||
|
SpacesInCStyleCastParentheses: false
|
||||||
|
SpacesInParentheses: false
|
||||||
|
TabWidth: 4
|
||||||
|
...
|
||||||
14
rocksdb/.clang-tidy
Normal file
14
rocksdb/.clang-tidy
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
Checks: >
|
||||||
|
-*,
|
||||||
|
clang-diagnostic-*,
|
||||||
|
-llvmlibc-restrict-system-libc-headers,
|
||||||
|
llvm-*,
|
||||||
|
misc-*,
|
||||||
|
-misc-const-correctness,
|
||||||
|
-misc-unused-parameters,
|
||||||
|
-misc-non-private-member-variables-in-classes,
|
||||||
|
-misc-no-recursion,
|
||||||
|
-misc-use-anonymous-namespace,
|
||||||
|
readability-identifier-naming,
|
||||||
|
-llvm-header-guard,
|
||||||
|
-llvm-include-order,
|
||||||
3
rocksdb/.gitignore
vendored
Normal file
3
rocksdb/.gitignore
vendored
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
build/
|
||||||
|
.cache
|
||||||
|
compile_commands.json
|
||||||
@ -1,10 +1,30 @@
|
|||||||
cmake_minimum_required(VERSION 4.0)
|
cmake_minimum_required(VERSION 3.20)
|
||||||
project(rockseb_bench)
|
project(rocksdb_bench CXX)
|
||||||
|
|
||||||
set(CMAKE_CXX_STANDARD 23)
|
set(CMAKE_CXX_STANDARD 23)
|
||||||
|
|
||||||
add_executable(rockseb_bench main.cpp
|
if (CMAKE_BUILD_TYPE MATCHES Release)
|
||||||
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3")
|
||||||
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto")
|
||||||
|
endif()
|
||||||
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g")
|
||||||
|
|
||||||
|
find_package(CLI11 CONFIG REQUIRED)
|
||||||
|
|
||||||
|
add_executable(rocksdb_bench main.cpp
|
||||||
instant.h)
|
instant.h)
|
||||||
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
|
|
||||||
target_compile_options(rockseb_bench PRIVATE "-flto")
|
target_include_directories(rocksdb_bench PRIVATE ${CMAKE_CURRENT_LIST_DIR})
|
||||||
target_link_libraries(rockseb_bench rocksdb)
|
target_link_libraries(rocksdb_bench rocksdb CLI11::CLI11)
|
||||||
|
|
||||||
|
file(GLOB ALL_SOURCE_FILES *.cpp *.h)
|
||||||
|
add_custom_target(format
|
||||||
|
COMMAND clang-format --style=file -i ${ALL_SOURCE_FILES}
|
||||||
|
VERBATIM
|
||||||
|
)
|
||||||
|
|
||||||
|
add_custom_target(tidy
|
||||||
|
COMMAND clang-tidy -p=${CMAKE_BINARY_DIR} --config-file=${CMAKE_SOURCE_DIR}/.clang-tidy ${ALL_SOURCE_FILES} -extra-arg='--std=c++23'
|
||||||
|
COMMENT "Running Clang-Tidy"
|
||||||
|
VERBATIM
|
||||||
|
)
|
||||||
|
|||||||
40
rocksdb/CMakePresets.json
Normal file
40
rocksdb/CMakePresets.json
Normal file
@ -0,0 +1,40 @@
|
|||||||
|
{
|
||||||
|
"version": 3,
|
||||||
|
"configurePresets":[
|
||||||
|
{
|
||||||
|
"name": "basic",
|
||||||
|
"hidden": true,
|
||||||
|
"cacheVariables": {
|
||||||
|
"CMAKE_C_COMPILER": "gcc",
|
||||||
|
"CMAKe_CXX_COMPILER": "g++",
|
||||||
|
"CMAKE_TOOLCHAIN_FILE": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake",
|
||||||
|
"CMAKE_EXPORT_COMPILE_COMMANDS": "ON",
|
||||||
|
"ENABLE_PROFILING": "ON"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "debug",
|
||||||
|
"inherits": "basic",
|
||||||
|
"binaryDir": "${sourceDir}/build/debug",
|
||||||
|
"cacheVariables": {
|
||||||
|
"CMAKE_BUILD_TYPE": "Debug",
|
||||||
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/dist/debug"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "release",
|
||||||
|
"inherits": "basic",
|
||||||
|
"binaryDir": "${sourceDir}/build/release",
|
||||||
|
"cacheVariables": {
|
||||||
|
"CMAKE_BUILD_TYPE": "Release",
|
||||||
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/dist/release"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"buildPresets": [
|
||||||
|
{
|
||||||
|
"name": "release",
|
||||||
|
"configurePreset": "release"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@ -3,53 +3,29 @@
|
|||||||
|
|
||||||
#include <chrono>
|
#include <chrono>
|
||||||
|
|
||||||
namespace nm
|
namespace nm {
|
||||||
{
|
class Instant {
|
||||||
class Instant {
|
public:
|
||||||
public:
|
static Instant now() { return {std::chrono::steady_clock::now()}; }
|
||||||
static Instant now()
|
|
||||||
{
|
|
||||||
return { std::chrono::steady_clock::now() };
|
|
||||||
}
|
|
||||||
|
|
||||||
auto elapse_usec()
|
auto elapse_usec() { return diff().count() / 1e3; }
|
||||||
{
|
|
||||||
return diff().count() / 1e3;
|
|
||||||
}
|
|
||||||
|
|
||||||
auto elapse_ms()
|
auto elapse_ms() { return diff().count() / 1e6; }
|
||||||
{
|
|
||||||
return diff().count() / 1e6;
|
|
||||||
}
|
|
||||||
|
|
||||||
auto elapse_sec()
|
auto elapse_sec() { return diff().count() / 1e9; }
|
||||||
{
|
|
||||||
return diff().count() / 1e9;
|
|
||||||
}
|
|
||||||
|
|
||||||
auto elapse_min()
|
auto elapse_min() { return elapse_sec() / 60.0; }
|
||||||
{
|
|
||||||
return elapse_sec() / 60.0;
|
|
||||||
}
|
|
||||||
|
|
||||||
void reset()
|
void reset() { *this = now(); }
|
||||||
{
|
|
||||||
*this = now();
|
|
||||||
}
|
|
||||||
|
|
||||||
private:
|
private:
|
||||||
using timepoint = std::chrono::steady_clock::time_point;
|
using timepoint = std::chrono::steady_clock::time_point;
|
||||||
Instant(timepoint now) : tp_ { now }
|
Instant(timepoint now) : tp_{now} {}
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
auto diff() -> std::chrono::duration<double, std::nano>
|
auto diff() -> std::chrono::duration<double, std::nano> { return std::chrono::steady_clock::now() - tp_; }
|
||||||
{
|
|
||||||
return std::chrono::steady_clock::now() - tp_;
|
|
||||||
}
|
|
||||||
|
|
||||||
timepoint tp_;
|
timepoint tp_;
|
||||||
};
|
};
|
||||||
} // namespace nm
|
} // namespace nm
|
||||||
|
|
||||||
#endif // INSTANT_20231013102058
|
#endif // INSTANT_20231013102058
|
||||||
|
|||||||
198
rocksdb/main.cpp
198
rocksdb/main.cpp
@ -1,3 +1,7 @@
|
|||||||
|
#include <algorithm>
|
||||||
|
#include <atomic>
|
||||||
|
#include <print>
|
||||||
|
#include <random>
|
||||||
#include <rocksdb/cache.h>
|
#include <rocksdb/cache.h>
|
||||||
#include <rocksdb/db.h>
|
#include <rocksdb/db.h>
|
||||||
#include <rocksdb/options.h>
|
#include <rocksdb/options.h>
|
||||||
@ -5,20 +9,73 @@
|
|||||||
#include <rocksdb/utilities/transaction.h>
|
#include <rocksdb/utilities/transaction.h>
|
||||||
#include <rocksdb/utilities/transaction_db.h>
|
#include <rocksdb/utilities/transaction_db.h>
|
||||||
|
|
||||||
#include <algorithm>
|
|
||||||
#include <barrier>
|
#include <barrier>
|
||||||
#include <filesystem>
|
#include <filesystem>
|
||||||
#include <format>
|
#include <format>
|
||||||
#include <iostream>
|
|
||||||
#include <random>
|
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
|
#include "CLI/CLI.hpp"
|
||||||
#include "instant.h"
|
#include "instant.h"
|
||||||
|
|
||||||
int main()
|
struct Args {
|
||||||
{
|
size_t threads;
|
||||||
std::string db_root = "/home/abby/rocksdb_tmp";
|
size_t iterations;
|
||||||
std::filesystem::remove_all(db_root);
|
size_t key_size;
|
||||||
|
size_t value_size;
|
||||||
|
size_t insert_ratio;
|
||||||
|
bool random;
|
||||||
|
std::string mode;
|
||||||
|
std::string path;
|
||||||
|
};
|
||||||
|
|
||||||
|
int main(int argc, char *argv[]) {
|
||||||
|
CLI::App app{"rocksdb bench"};
|
||||||
|
Args args{
|
||||||
|
.threads = 4,
|
||||||
|
.iterations = 100000,
|
||||||
|
.key_size = 16,
|
||||||
|
.value_size = 1024,
|
||||||
|
.insert_ratio = 30,
|
||||||
|
.mode = "insert",
|
||||||
|
.path = "/tmp/rocksdb_tmp",
|
||||||
|
};
|
||||||
|
|
||||||
|
app.add_option("-m,--mode", args.mode, "Mode: insert, get, mixed");
|
||||||
|
app.add_option("-t,--threads", args.threads, "Threads");
|
||||||
|
app.add_option("-k,--key-size", args.key_size, "Key Size");
|
||||||
|
app.add_option("-v,--value-size", args.value_size, "Value Size");
|
||||||
|
app.add_option("-i,--iterations", args.iterations, "Iterations");
|
||||||
|
app.add_option("-r,--insert-ratio", args.insert_ratio, "Insert Ratio for mixed mode");
|
||||||
|
app.add_option("-p,--path", args.path, "DataBase Home");
|
||||||
|
app.add_option("--random", args.random, "Shuffle insert keys");
|
||||||
|
|
||||||
|
CLI11_PARSE(app, argc, argv);
|
||||||
|
|
||||||
|
if (args.path.empty()) {
|
||||||
|
std::println("path is empty");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (std::filesystem::exists(args.path)) {
|
||||||
|
std::println("path `{}` already exists", args.path);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (args.mode != "insert" && args.mode != "get" && args.mode != "mixed") {
|
||||||
|
std::println("Error: Invalid mode");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (args.key_size < 16 || args.value_size < 16) {
|
||||||
|
std::println("Error: key_size or value_size too small, must >= 16");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (args.insert_ratio > 100) {
|
||||||
|
std::println("Error: Insert ratio must be between 0 and 100");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
rocksdb::Options options;
|
rocksdb::Options options;
|
||||||
options.disable_auto_compactions = true;
|
options.disable_auto_compactions = true;
|
||||||
options.create_if_missing = true;
|
options.create_if_missing = true;
|
||||||
@ -27,93 +84,102 @@ int main()
|
|||||||
options.write_buffer_size = 64 << 20;
|
options.write_buffer_size = 64 << 20;
|
||||||
options.level0_file_num_compaction_trigger = 500;
|
options.level0_file_num_compaction_trigger = 500;
|
||||||
options.max_bytes_for_level_base = 2 << 30;
|
options.max_bytes_for_level_base = 2 << 30;
|
||||||
constexpr size_t count = 100000;
|
|
||||||
constexpr size_t workers = 4;
|
|
||||||
auto ropt = rocksdb::ReadOptions();
|
auto ropt = rocksdb::ReadOptions();
|
||||||
auto wopt = rocksdb::WriteOptions();
|
auto wopt = rocksdb::WriteOptions();
|
||||||
std::vector<std::thread> wg;
|
std::vector<std::thread> wg;
|
||||||
std::vector<std::vector<std::string>> keys {};
|
std::vector<std::vector<std::string>> keys{};
|
||||||
|
std::atomic<uint64_t> total_op{0};
|
||||||
rocksdb::OptimisticTransactionDB *db;
|
rocksdb::OptimisticTransactionDB *db;
|
||||||
std::cout << "db_root " << db_root << '\n';
|
auto b = nm::Instant::now();
|
||||||
auto s = rocksdb::OptimisticTransactionDB::Open(options, db_root, &db);
|
std::mutex mtx{};
|
||||||
|
auto s = rocksdb::OptimisticTransactionDB::Open(options, args.path, &db);
|
||||||
assert(s.ok());
|
assert(s.ok());
|
||||||
std::string val(1024, 'x');
|
std::barrier barrier{static_cast<ptrdiff_t>(args.threads)};
|
||||||
|
|
||||||
for (size_t tid = 0; tid < workers; ++tid) {
|
std::random_device rd{};
|
||||||
std::vector<std::string> key {};
|
std::mt19937 gen(rd());
|
||||||
|
std::uniform_int_distribution<int> dist(0, 100);
|
||||||
|
|
||||||
for (size_t i = 0; i < count; ++i) {
|
|
||||||
|
std::string val(args.value_size, 'x');
|
||||||
|
for (size_t tid = 0; tid < args.threads; ++tid) {
|
||||||
|
std::vector<std::string> key{};
|
||||||
|
for (size_t i = 0; i < args.iterations; ++i) {
|
||||||
auto tmp = std::format("key_{}_{}", tid, i);
|
auto tmp = std::format("key_{}_{}", tid, i);
|
||||||
tmp.resize(1024, 'x');
|
tmp.resize(args.key_size, 'x');
|
||||||
key.push_back(std::move(tmp));
|
key.emplace_back(std::move(tmp));
|
||||||
|
}
|
||||||
|
if (args.random) {
|
||||||
|
std::shuffle(keys.begin(), keys.end(), gen);
|
||||||
}
|
}
|
||||||
keys.emplace_back(std::move(key));
|
keys.emplace_back(std::move(key));
|
||||||
}
|
}
|
||||||
|
|
||||||
for (size_t tid = 0; tid < workers; ++tid) {
|
|
||||||
auto ks = &keys[tid];
|
if (args.mode == "get") {
|
||||||
wg.emplace_back(
|
auto *kv = db->BeginTransaction(wopt);
|
||||||
[&]
|
for (size_t tid = 0; tid < args.threads; ++tid) {
|
||||||
{
|
auto *tk = &keys[tid];
|
||||||
for (auto &x : *ks) {
|
for (auto &key: *tk) {
|
||||||
auto kv = db->BeginTransaction(wopt);
|
kv->Put(key, val);
|
||||||
kv->Put(x, x);
|
}
|
||||||
|
}
|
||||||
kv->Commit();
|
kv->Commit();
|
||||||
delete kv;
|
delete kv;
|
||||||
}
|
}
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
for (auto &w : wg) {
|
for (size_t tid = 0; tid < args.threads; ++tid) {
|
||||||
w.join();
|
auto *tk = &keys[tid];
|
||||||
}
|
wg.emplace_back([&] {
|
||||||
|
std::string rval(args.value_size, '0');
|
||||||
delete db;
|
barrier.arrive_and_wait();
|
||||||
wg.clear();
|
|
||||||
|
|
||||||
auto sts =
|
|
||||||
rocksdb::OptimisticTransactionDB::Open(options, db_root, &db);
|
|
||||||
assert(sts.ok());
|
|
||||||
std::barrier g { workers };
|
|
||||||
auto b = nm::Instant::now();
|
|
||||||
std::mutex mtx {};
|
|
||||||
std::atomic<uint64_t> operation { 0 };
|
|
||||||
|
|
||||||
for (size_t tid = 0; tid < workers; ++tid) {
|
|
||||||
auto ks = &keys[tid];
|
|
||||||
wg.emplace_back(
|
|
||||||
[&]
|
|
||||||
{
|
|
||||||
g.arrive_and_wait();
|
|
||||||
|
|
||||||
std::string val;
|
|
||||||
if (mtx.try_lock()) {
|
if (mtx.try_lock()) {
|
||||||
b = nm::Instant::now();
|
b = nm::Instant::now();
|
||||||
mtx.unlock();
|
mtx.unlock();
|
||||||
}
|
}
|
||||||
|
|
||||||
for (auto &x : *ks) {
|
if (args.mode == "insert") {
|
||||||
auto kv = db->BeginTransaction(wopt);
|
for (auto &key: *tk) {
|
||||||
kv->Get(ropt, x, &val);
|
auto *kv = db->BeginTransaction(wopt);
|
||||||
|
kv->Put(key, val);
|
||||||
kv->Commit();
|
kv->Commit();
|
||||||
delete kv;
|
delete kv;
|
||||||
}
|
}
|
||||||
operation.fetch_add(count,
|
|
||||||
std::memory_order_relaxed);
|
} else if (args.mode == "get") {
|
||||||
|
for (auto &key: *tk) {
|
||||||
|
auto *kv = db->BeginTransaction(wopt);
|
||||||
|
kv->Get(ropt, key, &rval);
|
||||||
|
kv->Commit();
|
||||||
|
delete kv;
|
||||||
|
}
|
||||||
|
} else if (args.mode == "mixed") {
|
||||||
|
for (auto &key: *tk) {
|
||||||
|
auto is_insert = dist(gen) < args.insert_ratio;
|
||||||
|
auto *kv = db->BeginTransaction(wopt);
|
||||||
|
if (is_insert) {
|
||||||
|
kv->Put(key, val);
|
||||||
|
} else {
|
||||||
|
kv->Get(ropt, key, &rval); // not found
|
||||||
|
}
|
||||||
|
kv->Commit();
|
||||||
|
delete kv;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
total_op.fetch_add(args.iterations, std::memory_order::relaxed);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
for (auto &w : wg) {
|
for (auto &w: wg) {
|
||||||
w.join();
|
w.join();
|
||||||
}
|
}
|
||||||
|
size_t ratio = [&args] -> size_t {
|
||||||
printf("thread %ld\niterations %ld\nkey_size %ld\nvalue_size %ld\nops: "
|
if (args.mode == "mixed")
|
||||||
"%.2f\n",
|
return args.insert_ratio;
|
||||||
workers,
|
return args.mode == "insert" ? 100 : 0;
|
||||||
count,
|
}();
|
||||||
keys[0][0].size(),
|
double ops = static_cast<double>(total_op.load(std::memory_order_relaxed)) / b.elapse_sec();
|
||||||
val.size(),
|
std::println("{},{},{},{},{},{:.2f}", args.mode, args.threads, args.key_size, args.value_size, ratio, ops);
|
||||||
static_cast<double>(operation.load(std::memory_order_relaxed)) /
|
|
||||||
b.elapse_sec());
|
|
||||||
delete db;
|
delete db;
|
||||||
|
std::filesystem::remove_all(args.path);
|
||||||
}
|
}
|
||||||
|
|||||||
5
rocksdb/vcpkg.json
Normal file
5
rocksdb/vcpkg.json
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
{
|
||||||
|
"dependencies": [
|
||||||
|
"cli11"
|
||||||
|
]
|
||||||
|
}
|
||||||
@ -21,20 +21,20 @@ function samples() {
|
|||||||
./target/release/kv_bench --path /home/abby/mace_bench --threads $i --iterations 100000 --mode get --key-size ${kv_sz[j]} --value-size ${kv_sz[j+1]}
|
./target/release/kv_bench --path /home/abby/mace_bench --threads $i --iterations 100000 --mode get --key-size ${kv_sz[j]} --value-size ${kv_sz[j+1]}
|
||||||
if test $? -ne 0
|
if test $? -ne 0
|
||||||
then
|
then
|
||||||
echo "insert threads $i ksz ${kv_sz[j]} vsz ${kv_sz[j+1]} fail"
|
echo "get threads $i ksz ${kv_sz[j]} vsz ${kv_sz[j+1]} fail"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
./target/release/kv_bench --path /home/abby/mace_bench --threads $i --iterations 100000 --mode mixed --key-size ${kv_sz[j]} --value-size ${kv_sz[j+1]} --insert-ratio 30
|
./target/release/kv_bench --path /home/abby/mace_bench --threads $i --iterations 100000 --mode mixed --key-size ${kv_sz[j]} --value-size ${kv_sz[j+1]} --insert-ratio 30
|
||||||
if test $? -ne 0
|
if test $? -ne 0
|
||||||
then
|
then
|
||||||
echo "insert threads $i ksz ${kv_sz[j]} vsz ${kv_sz[j+1]} fail"
|
echo "mixed threads $i ksz ${kv_sz[j]} vsz ${kv_sz[j+1]} fail"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
echo mode,threads,key_size,value_size,insert_ratio,ops > scripts/x.csv
|
echo mode,threads,key_size,value_size,insert_ratio,ops > scripts/mace.csv
|
||||||
samples 2>> scripts/x.csv
|
samples 2>> scripts/mace.csv
|
||||||
popd
|
popd
|
||||||
./bin/python plot.py
|
./bin/python plot.py mace.csv
|
||||||
@ -1,14 +1,17 @@
|
|||||||
import pandas as pd
|
import pandas as pd
|
||||||
import matplotlib.pyplot as plt
|
import matplotlib.pyplot as plt
|
||||||
from adjustText import adjust_text
|
from adjustText import adjust_text
|
||||||
|
import sys
|
||||||
|
|
||||||
def real_mode(m):
|
def real_mode(m):
|
||||||
if m == "mixed":
|
if m == "mixed":
|
||||||
return "mixed (70% read, 30% write)"
|
return "MIXED (70% Get, 30% Put)"
|
||||||
return m
|
return m.upper()
|
||||||
|
|
||||||
|
name = sys.argv[1]
|
||||||
|
prefix = name.split(".")[0]
|
||||||
# 读取数据
|
# 读取数据
|
||||||
df = pd.read_csv("./x.csv")
|
df = pd.read_csv(f"./{name}")
|
||||||
|
|
||||||
# 按 mode 分组
|
# 按 mode 分组
|
||||||
modes = df["mode"].unique()
|
modes = df["mode"].unique()
|
||||||
@ -40,11 +43,11 @@ for mode in modes:
|
|||||||
adjust_text(texts, arrowprops=dict(arrowstyle="->", color='gray'))
|
adjust_text(texts, arrowprops=dict(arrowstyle="->", color='gray'))
|
||||||
|
|
||||||
# 设置图表样式
|
# 设置图表样式
|
||||||
plt.title(f"Performance: {real_mode(mode).upper()}", fontsize=16)
|
plt.title(f"{prefix.upper()}: {real_mode(mode)}", fontsize=16)
|
||||||
plt.xlabel("Threads", fontsize=14)
|
plt.xlabel("Threads", fontsize=14)
|
||||||
plt.ylabel("OPS", fontsize=14)
|
plt.ylabel("OPS", fontsize=14)
|
||||||
plt.grid(True, linestyle="--", alpha=0.6)
|
plt.grid(True, linestyle="--", alpha=0.6)
|
||||||
plt.legend()
|
plt.legend()
|
||||||
plt.tight_layout()
|
plt.tight_layout()
|
||||||
plt.savefig(f"{mode}.png")
|
plt.savefig(f"{prefix}_{mode}.png")
|
||||||
plt.close()
|
plt.close()
|
||||||
41
scripts/rocksdb.sh
Executable file
41
scripts/rocksdb.sh
Executable file
@ -0,0 +1,41 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
pushd .
|
||||||
|
cd ../rocksdb
|
||||||
|
cmake --preset release 1>/dev/null 2>/dev/null
|
||||||
|
cmake --build --preset release 1>/dev/null 2>/dev/null
|
||||||
|
|
||||||
|
function samples() {
|
||||||
|
kv_sz=(16 16 100 1024 1024 1024)
|
||||||
|
# set -x
|
||||||
|
|
||||||
|
for ((i = 1; i <= $(nproc); i *= 2))
|
||||||
|
do
|
||||||
|
for ((j = 0; j < ${#kv_sz[@]}; j += 2))
|
||||||
|
do
|
||||||
|
./build/release/rocksdb_bench --path /home/abby/rocksdb_tmp --threads $i --iterations 100000 --mode insert --key-size ${kv_sz[j]} --value-size ${kv_sz[j+1]}
|
||||||
|
if test $? -ne 0
|
||||||
|
then
|
||||||
|
echo "insert threads $i ksz ${kv_sz[j]} vsz ${kv_sz[j+1]} fail"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
./build/release/rocksdb_bench --path /home/abby/rocksdb_tmp --threads $i --iterations 100000 --mode get --key-size ${kv_sz[j]} --value-size ${kv_sz[j+1]}
|
||||||
|
if test $? -ne 0
|
||||||
|
then
|
||||||
|
echo "get threads $i ksz ${kv_sz[j]} vsz ${kv_sz[j+1]} fail"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
./build/release/rocksdb_bench --path /home/abby/rocksdb_tmp --threads $i --iterations 100000 --mode mixed --key-size ${kv_sz[j]} --value-size ${kv_sz[j+1]} --insert-ratio 30
|
||||||
|
if test $? -ne 0
|
||||||
|
then
|
||||||
|
echo "mixed threads $i ksz ${kv_sz[j]} vsz ${kv_sz[j+1]} fail"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
echo mode,threads,key_size,value_size,insert_ratio,ops > ../scripts/rocksdb.csv
|
||||||
|
samples 1>> ../scripts/rocksdb.csv
|
||||||
|
popd
|
||||||
|
./bin/python plot.py rocksdb.csv
|
||||||
51
src/main.rs
51
src/main.rs
@ -59,27 +59,37 @@ fn main() {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let mut keys: Vec<Vec<Vec<u8>>> = Vec::with_capacity(args.threads);
|
||||||
let mut opt = Options::new(path);
|
let mut opt = Options::new(path);
|
||||||
opt.sync_on_write = false;
|
opt.sync_on_write = false;
|
||||||
opt.tmp_store = true;
|
opt.tmp_store = true;
|
||||||
// make sure there's no remote indirection
|
|
||||||
opt.max_inline_size = 4096;
|
|
||||||
// opt.cache_capacity = 3 << 30; // this is very important for large key-value store
|
// opt.cache_capacity = 3 << 30; // this is very important for large key-value store
|
||||||
let db = Mace::new(opt.validate().unwrap()).unwrap();
|
let db = Mace::new(opt.validate().unwrap()).unwrap();
|
||||||
|
|
||||||
|
let mut rng = rand::rng();
|
||||||
let value = Arc::new(vec![b'0'; args.value_size]);
|
let value = Arc::new(vec![b'0'; args.value_size]);
|
||||||
if args.mode == "get" {
|
|
||||||
for tid in 0..args.threads {
|
for tid in 0..args.threads {
|
||||||
|
let mut tk = Vec::with_capacity(args.iterations);
|
||||||
for i in 0..args.iterations {
|
for i in 0..args.iterations {
|
||||||
let key = format!("key_{tid}_{i}");
|
let mut key = format!("key_{tid}_{i}").into_bytes();
|
||||||
let mut tmp = key.into_bytes();
|
key.resize(args.key_size, b'x');
|
||||||
tmp.resize(args.key_size, b'x');
|
tk.push(key);
|
||||||
|
}
|
||||||
|
if args.random {
|
||||||
|
tk.shuffle(&mut rng);
|
||||||
|
}
|
||||||
|
keys.push(tk);
|
||||||
|
}
|
||||||
|
|
||||||
|
if args.mode == "get" {
|
||||||
let pre_tx = db.begin().unwrap();
|
let pre_tx = db.begin().unwrap();
|
||||||
pre_tx.put(&tmp, &*value).unwrap();
|
(0..args.threads).for_each(|tid| {
|
||||||
|
for i in 0..args.iterations {
|
||||||
|
pre_tx.put(&keys[tid][i], &*value).unwrap();
|
||||||
|
}
|
||||||
|
});
|
||||||
pre_tx.commit().unwrap();
|
pre_tx.commit().unwrap();
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let barrier = Arc::new(std::sync::Barrier::new(args.threads));
|
let barrier = Arc::new(std::sync::Barrier::new(args.threads));
|
||||||
let total_ops = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
let total_ops = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||||
@ -88,6 +98,7 @@ fn main() {
|
|||||||
let h: Vec<JoinHandle<()>> = (0..args.threads)
|
let h: Vec<JoinHandle<()>> = (0..args.threads)
|
||||||
.map(|tid| {
|
.map(|tid| {
|
||||||
let db = db.clone();
|
let db = db.clone();
|
||||||
|
let tk: &Vec<Vec<u8>> = unsafe { std::mem::transmute(&keys[tid]) };
|
||||||
let total_ops = total_ops.clone();
|
let total_ops = total_ops.clone();
|
||||||
let barrier = Arc::clone(&barrier);
|
let barrier = Arc::clone(&barrier);
|
||||||
let mode = args.mode.clone();
|
let mode = args.mode.clone();
|
||||||
@ -96,18 +107,6 @@ fn main() {
|
|||||||
let val = value.clone();
|
let val = value.clone();
|
||||||
|
|
||||||
std::thread::spawn(move || {
|
std::thread::spawn(move || {
|
||||||
let mut keys: Vec<Vec<u8>> = Vec::with_capacity(args.iterations);
|
|
||||||
for i in 0..args.iterations {
|
|
||||||
let key = format!("key_{tid}_{i}");
|
|
||||||
let mut tmp = key.into_bytes();
|
|
||||||
tmp.resize(args.key_size, b'x');
|
|
||||||
keys.push(tmp);
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut rng = rand::rng();
|
|
||||||
if args.random {
|
|
||||||
keys.shuffle(&mut rng);
|
|
||||||
}
|
|
||||||
barrier.wait();
|
barrier.wait();
|
||||||
|
|
||||||
{
|
{
|
||||||
@ -118,21 +117,21 @@ fn main() {
|
|||||||
|
|
||||||
match mode.as_str() {
|
match mode.as_str() {
|
||||||
"insert" => {
|
"insert" => {
|
||||||
for key in &keys {
|
for key in tk {
|
||||||
let tx = db.begin().unwrap();
|
let tx = db.begin().unwrap();
|
||||||
tx.put(key.as_slice(), val.as_slice()).unwrap();
|
tx.put(key.as_slice(), val.as_slice()).unwrap();
|
||||||
tx.commit().unwrap();
|
tx.commit().unwrap();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
"get" => {
|
"get" => {
|
||||||
for key in &keys {
|
for key in tk {
|
||||||
let tx = db.view().unwrap();
|
let tx = db.view().unwrap();
|
||||||
tx.get(key).unwrap();
|
tx.get(key).unwrap();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
"mixed" => {
|
"mixed" => {
|
||||||
for key in &keys {
|
for key in tk {
|
||||||
let is_insert = rng.random_range(0..100) < insert_ratio;
|
let is_insert = rand::random_range(0..100) < insert_ratio;
|
||||||
|
|
||||||
if is_insert {
|
if is_insert {
|
||||||
let tx = db.begin().unwrap();
|
let tx = db.begin().unwrap();
|
||||||
@ -140,7 +139,7 @@ fn main() {
|
|||||||
tx.commit().unwrap();
|
tx.commit().unwrap();
|
||||||
} else {
|
} else {
|
||||||
let tx = db.view().unwrap();
|
let tx = db.view().unwrap();
|
||||||
let _ = tx.get(key); // may not insert
|
let _ = tx.get(key); // not found
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user