From 0e8d1ef77ad8db2b2491ae92874eb21b7986d59b Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Tue, 13 Jun 2023 14:48:03 -0400 Subject: Benchmark refactoring/cleanup --- benchmarks/alias_wss_bench.cpp | 57 ++++++++++++++++++++++++++ benchmarks/include/bench.h | 43 ++++++++++++++++++++ benchmarks/include/bench_utility.h | 47 +++++++++++++++++----- benchmarks/sampling_tput.cpp | 82 -------------------------------------- 4 files changed, 136 insertions(+), 93 deletions(-) create mode 100644 benchmarks/alias_wss_bench.cpp delete mode 100644 benchmarks/sampling_tput.cpp (limited to 'benchmarks') diff --git a/benchmarks/alias_wss_bench.cpp b/benchmarks/alias_wss_bench.cpp new file mode 100644 index 0000000..a3a43f2 --- /dev/null +++ b/benchmarks/alias_wss_bench.cpp @@ -0,0 +1,57 @@ +/* + * benchmarks/alias_wss_bench.cpp + * + * Copyright (C) 2023 Douglas Rumbaugh + * + * All rights reserved. Published under the Modified BSD License. + * + */ +#include "include/bench.h" + +int main(int argc, char **argv) +{ + if (argc < 4) { + fprintf(stderr, "Usage: sampling_tput [osm_data]\n"); + exit(EXIT_FAILURE); + } + + std::string filename = std::string(argv[1]); + size_t record_count = atol(argv[2]); + size_t buffer_cap = 12000; + size_t scale_factor = 6; + double delete_prop = atof(argv[3]); + double max_delete_prop = (delete_prop > 0) ? delete_prop : 1; + bool use_osm = (argc == 5) ? atoi(argv[4]) : 0; + + double insert_batch = 0.1; + + init_bench_env(record_count, true, use_osm); + + auto de_wss = ExtendedWSS(buffer_cap, scale_factor, max_delete_prop); + + std::fstream datafile; + datafile.open(filename, std::ios::in); + + std::vector to_delete; + + // warm up the tree with initial_insertions number of initially inserted + // records + size_t warmup_cnt = insert_batch * record_count; + warmup(datafile, de_wss, warmup_cnt, delete_prop, to_delete); + + size_t insert_cnt = record_count - warmup_cnt; + + std::vector> queries(1); + queries[0].rng = g_rng; + queries[0].sample_size = 1000; + + insert_tput_bench(de_wss, datafile, insert_cnt, delete_prop, to_delete); + query_latency_bench>(de_wss, queries, 1000); + fprintf(stdout, "\n"); + + delete_bench_env(); + fflush(stdout); + fflush(stderr); + + exit(EXIT_SUCCESS); +} diff --git a/benchmarks/include/bench.h b/benchmarks/include/bench.h index d82da48..6a9c263 100644 --- a/benchmarks/include/bench.h +++ b/benchmarks/include/bench.h @@ -1,3 +1,13 @@ +/* + * benchmarks/include/bench.h + * + * Copyright (C) 2023 Douglas Rumbaugh + * + * All rights reserved. Published under the Modified BSD License. + * + */ +#pragma once + #include "bench_utility.h" template @@ -64,4 +74,37 @@ static bool insert_tput_bench(DE &de_index, std::fstream &file, size_t insert_cn return continue_benchmark; } +template +static bool query_latency_bench(DE &de_index, std::vector queries, size_t trial_cnt=100) { + char progbuf[25]; + if constexpr (PROGRESS) { + sprintf(progbuf, "querying:"); + } + + size_t total_time = 0; + size_t total_results = 0; + + for (size_t i=0; i(stop - start).count(); + } + + progress_update(1.0, progbuf); + + size_t query_latency = total_time / (trial_cnt * queries.size()); + fprintf(stdout, "%ld\t", query_latency); + fflush(stdout); + + return true; +} diff --git a/benchmarks/include/bench_utility.h b/benchmarks/include/bench_utility.h index 78d6415..f208487 100644 --- a/benchmarks/include/bench_utility.h +++ b/benchmarks/include/bench_utility.h @@ -1,5 +1,13 @@ -#ifndef H_BENCH -#define H_BENCH +/* + * benchmarks/include/bench_utility.h + * + * Copyright (C) 2023 Douglas Rumbaugh + * + * All rights reserved. Published under the Modified BSD License. + * + */ +#pragma once + #include "framework/DynamicExtension.h" #include "shard/WSS.h" #include "shard/MemISAM.h" @@ -29,7 +37,7 @@ typedef de::WeightedRecord WRec; typedef de::Record Rec; typedef de::DynamicExtension, de::WSSQuery> ExtendedWSS; -typedef de::DynamicExtension, de::TrieSplineRangeQuery> ExtendedTS; +typedef de::DynamicExtension, de::TrieSplineRangeQuery> ExtendedTSRQ; typedef de::DynamicExtension, de::PGMRangeQuery> ExtendedPGM; static gsl_rng *g_rng; @@ -82,6 +90,31 @@ static void delete_bench_env() delete g_to_delete; } +/* + * NOTE: The QP type must have lower_bound and upper_bound attributes, which + * this function will initialize. Any other query parameter attributes must + * be manually initialized after the call. + */ +template +static std::vector read_range_queries(std::string fname, double selectivity) { + std::vector queries; + + FILE *qf = fopen(fname.c_str(), "r"); + size_t start, stop; + double sel; + while (fscanf(qf, "%zu%zu%lf\n", &start, &stop, &sel) != EOF) { + if (start < stop && std::abs(sel - selectivity) < 0.1) { + QP q; + q.lower_bound = start; + q.upper_bound = stop; + queries.push_back(q); + } + } + fclose(qf); + + return queries; +} + template static bool next_record(std::fstream &file, R &record) { @@ -213,12 +246,6 @@ static bool warmup(std::fstream &file, DE &extended_index, size_t count, } } - /* - if (progress) { - progress_update(1, "warming up:"); - } - */ - return true; } @@ -232,5 +259,3 @@ static void reset_de_perf_metrics() { RESET_IO_CNT(); } - -#endif // H_BENCH diff --git a/benchmarks/sampling_tput.cpp b/benchmarks/sampling_tput.cpp deleted file mode 100644 index 70ff0cb..0000000 --- a/benchmarks/sampling_tput.cpp +++ /dev/null @@ -1,82 +0,0 @@ -#include "include/bench.h" - -static void sample_benchmark(ExtendedWSS &de_wss, size_t k, size_t trial_cnt) -{ - char progbuf[25]; - sprintf(progbuf, "sampling (%ld):", k); - - size_t batch_size = 100; - size_t batches = trial_cnt / batch_size; - size_t total_time = 0; - - WRec sample_set[k]; - - size_t total_samples = 0; - - de::wss_query_parms parms; - parms.rng = g_rng; - parms.sample_size = k; - - for (int i=0; i(stop - start).count(); - } - - progress_update(1.0, progbuf); - - size_t throughput = (((double)(total_samples) / (double) total_time) * 1e9); - - fprintf(stdout, "%ld\n", throughput); - fflush(stdout); -} - - -int main(int argc, char **argv) -{ - if (argc < 7) { - fprintf(stderr, "Usage: sampling_tput [osm_data]\n"); - exit(EXIT_FAILURE); - } - - std::string filename = std::string(argv[1]); - size_t record_count = atol(argv[2]); - size_t buffer_cap = atol(argv[3]); - size_t scale_factor = atol(argv[4]); - double delete_prop = atof(argv[5]); - double max_delete_prop = atof(argv[6]); - bool use_osm = (argc == 8) ? atoi(argv[7]) : 0; - - double insert_batch = 0.1; - - init_bench_env(record_count, true, use_osm); - - auto de_wss = ExtendedWSS(buffer_cap, scale_factor, max_delete_prop); - - std::fstream datafile; - datafile.open(filename, std::ios::in); - - std::vector to_delete; - - // warm up the tree with initial_insertions number of initially inserted - // records - size_t warmup_cnt = insert_batch * record_count; - warmup(datafile, de_wss, warmup_cnt, delete_prop, to_delete); - - size_t insert_cnt = record_count - warmup_cnt; - - insert_tput_bench(de_wss, datafile, insert_cnt, delete_prop, to_delete); - sample_benchmark(de_wss, 1000, 10000); - - delete_bench_env(); - fflush(stdout); - fflush(stderr); - - exit(EXIT_SUCCESS); -} -- cgit v1.2.3