From 6088ede2b5a8f580f78d4422162d423a7a11f520 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Mon, 2 Jun 2025 19:54:15 -0400 Subject: updates --- CMakeLists.txt | 15 +++ benchmarks/tail-latency/btree_insert_dist.cpp | 2 +- benchmarks/tail-latency/fixed_shard_count.cpp | 19 +++- benchmarks/tail-latency/insert_query_threads.cpp | 4 +- benchmarks/tail-latency/isam_insert_dist.cpp | 107 ---------------------- benchmarks/tail-latency/knn_query_parm_sweep.cpp | 14 +-- benchmarks/tail-latency/query_parm_sweep.cpp | 8 +- benchmarks/tail-latency/selectivity_sweep.cpp | 4 +- benchmarks/tail-latency/standard_latency_dist.cpp | 18 ++-- external/psudb-common | 2 +- 10 files changed, 55 insertions(+), 138 deletions(-) delete mode 100644 benchmarks/tail-latency/isam_insert_dist.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a7bcad6..688935e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -297,6 +297,11 @@ if (tail_bench) target_include_directories(fixed_shard_count PRIVATE include external external/m-tree/cpp external/PGM-index/include external/PLEX/include benchmarks/include external/psudb-common/cpp/include) target_link_options(fixed_shard_count PUBLIC -mcx16) + add_executable(fixed_shard_count_dist ${CMAKE_CURRENT_SOURCE_DIR}/benchmarks/tail-latency/fixed_shard_count_dist.cpp) + target_link_libraries(fixed_shard_count_dist PUBLIC gsl pthread atomic) + target_include_directories(fixed_shard_count_dist PRIVATE include external external/m-tree/cpp external/PGM-index/include external/PLEX/include benchmarks/include external/psudb-common/cpp/include) + target_link_options(fixed_shard_count_dist PUBLIC -mcx16) + add_executable(standard_latency_dist ${CMAKE_CURRENT_SOURCE_DIR}/benchmarks/tail-latency/standard_latency_dist.cpp) target_link_libraries(standard_latency_dist PUBLIC gsl pthread atomic) target_include_directories(standard_latency_dist PRIVATE include external external/m-tree/cpp external/PGM-index/include external/PLEX/include benchmarks/include external/psudb-common/cpp/include) @@ -307,6 +312,11 @@ if (tail_bench) target_include_directories(btree_insert_dist PRIVATE include external external/m-tree/cpp external/PGM-index/include external/PLEX/include benchmarks/include external/psudb-common/cpp/include) target_link_options(btree_insert_dist PUBLIC -mcx16) + add_executable(btree_tput ${CMAKE_CURRENT_SOURCE_DIR}/benchmarks/tail-latency/btree_tput.cpp) + target_link_libraries(btree_tput PUBLIC gsl pthread atomic) + target_include_directories(btree_tput PRIVATE include external external/m-tree/cpp external/PGM-index/include external/PLEX/include benchmarks/include external/psudb-common/cpp/include) + target_link_options(btree_tput PUBLIC -mcx16) + add_executable(mixed_workload ${CMAKE_CURRENT_SOURCE_DIR}/benchmarks/tail-latency/mixed_workload.cpp) target_link_libraries(mixed_workload PUBLIC gsl pthread atomic) target_include_directories(mixed_workload PRIVATE include external external/m-tree/cpp external/PGM-index/include external/PLEX/include benchmarks/include external/psudb-common/cpp/include) @@ -357,6 +367,11 @@ if (tail_bench) target_link_libraries(knn_selectivity_sweep PUBLIC gsl pthread atomic) target_include_directories(knn_selectivity_sweep PRIVATE include external external/m-tree/cpp external/PGM-index/include external/PLEX/include benchmarks/include external/psudb-common/cpp/include) target_link_options(knn_selectivity_sweep PUBLIC -mcx16) + + add_executable(isam_latency_dist ${CMAKE_CURRENT_SOURCE_DIR}/benchmarks/tail-latency/isam_latency_dist.cpp) + target_link_libraries(isam_latency_dist PUBLIC gsl pthread atomic) + target_include_directories(isam_latency_dist PRIVATE include external external/m-tree/cpp external/PGM-index/include external/PLEX/include benchmarks/include external/psudb-common/cpp/include) + target_link_options(isam_latency_dist PUBLIC -mcx16) endif() if (bench) diff --git a/benchmarks/tail-latency/btree_insert_dist.cpp b/benchmarks/tail-latency/btree_insert_dist.cpp index af60819..e1d171d 100644 --- a/benchmarks/tail-latency/btree_insert_dist.cpp +++ b/benchmarks/tail-latency/btree_insert_dist.cpp @@ -45,7 +45,7 @@ int main(int argc, char **argv) { auto queries = read_range_queries(q_fname, .0001); /* warmup structure w/ 10% of records */ - size_t warmup = .1 * n; + size_t warmup = .3 * n; for (size_t i=0; i Rec; -typedef de::TrieSpline Shard; +typedef de::ISAMTree Shard; typedef de::rc::Query Q; -typedef de::DynamicExtension Ext; +typedef de::DynamicExtension Ext; typedef Q::Parameters QP; +typedef de::DEConfiguration Conf; void usage(char *progname) { fprintf(stderr, "%s reccnt datafile queryfile\n", progname); @@ -52,10 +55,16 @@ int main(int argc, char **argv) { for (size_t i=0; i(shard_counts[i], buffer_size, 4, n); - auto extension = new Ext(std::move(policy)); + auto config = Conf(std::move(policy)); + config.recon_enable_maint_on_flush = false; + config.recon_maint_disabled = true; + config.buffer_flush_trigger = 4000; + config.maximum_threads = 8; + + auto extension = new Ext(std::move(config)); /* warmup structure w/ 10% of records */ - size_t warmup = .1 * n; + size_t warmup = .3 * n; for (size_t i=0; iinsert(data[i])) { usleep(1); diff --git a/benchmarks/tail-latency/insert_query_threads.cpp b/benchmarks/tail-latency/insert_query_threads.cpp index 3f78f16..cb4b81b 100644 --- a/benchmarks/tail-latency/insert_query_threads.cpp +++ b/benchmarks/tail-latency/insert_query_threads.cpp @@ -106,11 +106,11 @@ int main(int argc, char **argv) { auto queries = read_sosd_point_lookups(q_fname, 100); size_t buffer_size = 8000; - std::vector policies = {6}; + std::vector policies = {0}; std::vector thread_counts = {32}; std::vector modifiers = {0}; std::vector scale_factors = {6}; - std::vector rate_limits = {1, 0.9999, 0.999, 0.99, 0.9, 0.85, 0.8}; + std::vector rate_limits = {1}; size_t insert_threads = 1; size_t query_threads = 1; diff --git a/benchmarks/tail-latency/isam_insert_dist.cpp b/benchmarks/tail-latency/isam_insert_dist.cpp deleted file mode 100644 index 88d37c5..0000000 --- a/benchmarks/tail-latency/isam_insert_dist.cpp +++ /dev/null @@ -1,107 +0,0 @@ -/* - * - */ - -#include "framework/scheduling/FIFOScheduler.h" -#define ENABLE_TIMER -#define TS_TEST - -#include - -#include "framework/DynamicExtension.h" -#include "shard/ISAMTree.h" -#include "query/rangecount.h" -#include "framework/interface/Record.h" -#include "file_util.h" -#include "standard_benchmarks.h" - -#include - -#include "psu-util/timer.h" - - -typedef de::Record Rec; -typedef de::ISAMTreeShard; -typedef de::rc::Query Q; -typedef de::DynamicExtension Ext; -typedef Q::Parameters QP; - -void usage(char *progname) { - fprintf(stderr, "%s reccnt datafile queryfile\n", progname); -} - -int main(int argc, char **argv) { - - if (argc < 4) { - usage(argv[0]); - exit(EXIT_FAILURE); - } - - size_t n = atol(argv[1]); - std::string d_fname = std::string(argv[2]); - std::string q_fname = std::string(argv[3]); - - - auto data = read_sosd_file(d_fname, n); - auto queries = read_range_queries(q_fname, .0001); - - std::vector policies = {3}; - std::vector buffers = {8000, 16000, 32000}; - std::vector sfs = {8}; - - for (size_t l=0; l(sfs[k], buffers[j], policies[l]); - auto extension = new Ext(policy, buffers[j]/4, buffers[j]); - - /* warmup structure w/ 10% of records */ - size_t warmup = .1 * n; - for (size_t i=0; iinsert(data[i])) { - usleep(1); - } - } - - extension->await_next_epoch(); - - TIMER_INIT(); - - for (size_t i=warmup; iinsert(data[i])) { - usleep(1); - } - TIMER_STOP(); - - //fprintf(stdout, "%ld\t%ld\t%d\t%ld\n", sfs[k], buffers[j], policies[l], TIMER_RESULT()); - } - - extension->await_next_epoch(); - - /* repeat the queries a bunch of times */ - for (size_t l=0; l<10; l++) { - for (size_t i=0; iquery(std::move(q)); - res.get(); - TIMER_STOP(); - - fprintf(stdout, "%ld\t%ld\t%d\t%ld\n", sfs[k], buffers[j], policies[l], TIMER_RESULT()); - } - } - - - QP p = {0, 10000}; - auto res =extension->query(std::move(p)); - - fprintf(stderr, "%ld\n", res.get()); - extension->await_next_epoch(); - delete extension; - }}} - - - fflush(stderr); -} - diff --git a/benchmarks/tail-latency/knn_query_parm_sweep.cpp b/benchmarks/tail-latency/knn_query_parm_sweep.cpp index dfd713d..1766c74 100644 --- a/benchmarks/tail-latency/knn_query_parm_sweep.cpp +++ b/benchmarks/tail-latency/knn_query_parm_sweep.cpp @@ -105,14 +105,14 @@ int main(int argc, char **argv) { //auto queries =read_sosd_point_lookups(q_fname, 1); auto data = read_vector_file(d_fname, n); - auto queries = read_knn_queries(q_fname, 15, 1); + auto queries = read_knn_queries(q_fname, 100, 1); - size_t buffer_size = 8000; - std::vector policies = {0}; + size_t buffer_size = 1000; + std::vector policies = {0, 1}; std::vector thread_counts = {8}; std::vector modifiers = {0}; - std::vector scale_factors = {2, 4, 6, 8, 10}; + std::vector scale_factors = {2, 4, 6, 8, 16, 32, 128}; size_t insert_threads = 1; size_t query_threads = 1; @@ -171,14 +171,14 @@ int main(int argc, char **argv) { extension->await_version(); /* run some queries to "warm up" the cache */ - for (size_t i=0; iquery(std::move(q)).get(); total_res.fetch_add(res.size()); } - total_query_count.store(100000); + total_query_count.store(5000); TIMER_INIT(); TIMER_START(); for (size_t i=0; iget_height(), extension->get_shard_count(), insert_tput, query_lat); - extension->print_scheduler_statistics(); + //extension->print_scheduler_statistics(); //extension->print_scheduler_query_data(); //extension->print_structure(); fflush(stdout); diff --git a/benchmarks/tail-latency/query_parm_sweep.cpp b/benchmarks/tail-latency/query_parm_sweep.cpp index 36665a6..f2453aa 100644 --- a/benchmarks/tail-latency/query_parm_sweep.cpp +++ b/benchmarks/tail-latency/query_parm_sweep.cpp @@ -103,11 +103,11 @@ int main(int argc, char **argv) { auto queries =read_sosd_point_lookups(q_fname, 1); size_t buffer_size = 8000; - std::vector policies = {1}; + std::vector policies = {0, 1}; std::vector thread_counts = {8}; std::vector modifiers = {0}; - std::vector scale_factors = {4, 4, 4, 4}; + std::vector scale_factors = {2, 4, 6, 8, 16, 32, 64}; size_t insert_threads = 1; size_t query_threads = 1; @@ -195,8 +195,8 @@ int main(int argc, char **argv) { fprintf(stdout, "%ld\t%ld\t%ld\t%lf\t%ld\t%ld\t%ld\t%ld\n", internal_thread_cnt, pol, sf, mod, extension->get_height(), extension->get_shard_count(), insert_tput, query_lat); - extension->print_scheduler_statistics(); - extension->print_scheduler_query_data(); + //extension->print_scheduler_statistics(); + //extension->print_scheduler_query_data(); //extension->print_structure(); fflush(stdout); diff --git a/benchmarks/tail-latency/selectivity_sweep.cpp b/benchmarks/tail-latency/selectivity_sweep.cpp index 77c088b..0fc0f42 100644 --- a/benchmarks/tail-latency/selectivity_sweep.cpp +++ b/benchmarks/tail-latency/selectivity_sweep.cpp @@ -49,13 +49,13 @@ int main(int argc, char **argv) { auto data = read_sosd_file(d_fname, n); std::vector> query_sets; - std::vector selectivities = {0, 0.00000001, 0.0000001, 0.000001, 0.00001, 0.0001, 0.001, 0.01, 0.1}; + std::vector selectivities = {0.0000001}; //, 0.000001, 0.00001, 0.0001, 0.001, 0.01, 0.1, .25}; for (auto sel: selectivities) { query_sets.push_back(generate_uniform_range_queries(100, n, sel)); } - std::vector sfs = {2, 4, 8}; //, 4, 8, 16, 32, 64, 128, 256, 512, 1024}; + std::vector sfs = {2, 4, 8, 16, 32, 64, 128}; size_t buffer_size = 8000; std::vector policies = {0, 1}; diff --git a/benchmarks/tail-latency/standard_latency_dist.cpp b/benchmarks/tail-latency/standard_latency_dist.cpp index 8f1594b..dca02bf 100644 --- a/benchmarks/tail-latency/standard_latency_dist.cpp +++ b/benchmarks/tail-latency/standard_latency_dist.cpp @@ -2,17 +2,18 @@ * */ +#define ENABLE_TIMER +#define TS_TEST + #include "framework/scheduling/SerialScheduler.h" #include "framework/util/Configuration.h" #include "util/types.h" -#define ENABLE_TIMER -#define TS_TEST #include #include "framework/DynamicExtension.h" #include "framework/scheduling/FIFOScheduler.h" -#include "shard/TrieSpline.h" +#include "shard/ISAMTree.h" #include "query/rangecount.h" #include "framework/interface/Record.h" #include "file_util.h" @@ -26,11 +27,11 @@ typedef de::Record Rec; -typedef de::TrieSpline Shard; +typedef de::ISAMTree Shard; typedef de::rc::Query Q; -typedef de::DynamicExtension Ext; +typedef de::DynamicExtension Ext; typedef Q::Parameters QP; -typedef de::DEConfiguration Conf; +typedef de::DEConfiguration Conf; void usage(char *progname) { fprintf(stderr, "%s reccnt datafile queryfile\n", progname); @@ -50,9 +51,9 @@ int main(int argc, char **argv) { auto data = read_sosd_file(d_fname, n); auto queries = read_range_queries(q_fname, .0001); - std::vector sfs = {2, 3, 4, 5, 6, 7, 8}; //, 4, 8, 16, 32, 64, 128, 256, 512, 1024}; + std::vector sfs = {4, 8}; //, 4, 8, 16, 32, 64, 128, 256, 512, 1024}; size_t buffer_size = 8000; - std::vector policies = {0, 1, 2}; + std::vector policies = {0, 1}; for (auto pol: policies) { for (size_t i=0; iinsert(data[j])) { usleep(1); - fprintf(stderr, "insert blocked %ld\r", j); } } TIMER_STOP(); diff --git a/external/psudb-common b/external/psudb-common index 67ec5f5..ce3b373 160000 --- a/external/psudb-common +++ b/external/psudb-common @@ -1 +1 @@ -Subproject commit 67ec5f53de89c0edd6c46cd85f4ad05d380c16b4 +Subproject commit ce3b373b75c28098df83ec95234a90cb4f4d364f -- cgit v1.2.3