Benchmark refactoring/cleanup

5 files changed, 140 insertions, 97 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b7b5740..864a3f4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -87,10 +87,10 @@ endif()
 if (bench)
     set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/bin/benchmarks")
 
-    add_executable(sampling_tput ${CMAKE_CURRENT_SOURCE_DIR}/benchmarks/sampling_tput.cpp)
-    target_link_libraries(sampling_tput PUBLIC gsl pthread gomp)
-    target_include_directories(sampling_tput PRIVATE include external/PGM-index/include external/PLEX/include bench/include)
-    target_compile_options(sampling_tput PUBLIC -fopenmp)
+    add_executable(alias_wss_bench ${CMAKE_CURRENT_SOURCE_DIR}/benchmarks/alias_wss_bench.cpp)
+    target_link_libraries(alias_wss_bench PUBLIC gsl pthread gomp)
+    target_include_directories(alias_wss_bench PRIVATE include external/PGM-index/include external/PLEX/include bench/include)
+    target_compile_options(alias_wss_bench PUBLIC -fopenmp)
 
 
 endif()
diff --git a/benchmarks/alias_wss_bench.cpp b/benchmarks/alias_wss_bench.cpp
new file mode 100644
index 0000000..a3a43f2
--- /dev/null
+++ b/benchmarks/alias_wss_bench.cpp
@@ -0,0 +1,57 @@
+/*
+ * benchmarks/alias_wss_bench.cpp
+ *
+ * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> 
+ *
+ * All rights reserved. Published under the Modified BSD License.
+ *
+ */
+#include "include/bench.h"
+
+int main(int argc, char **argv)
+{
+    if (argc < 4) {
+        fprintf(stderr, "Usage: sampling_tput <filename> <record_count> <delete_proportion> [osm_data]\n");
+        exit(EXIT_FAILURE);
+    }
+
+    std::string filename = std::string(argv[1]);
+    size_t record_count = atol(argv[2]);
+    size_t buffer_cap = 12000;
+    size_t scale_factor = 6;
+    double delete_prop = atof(argv[3]);
+    double max_delete_prop = (delete_prop > 0) ? delete_prop : 1;
+    bool use_osm = (argc == 5) ? atoi(argv[4]) : 0;
+
+    double insert_batch = 0.1; 
+
+    init_bench_env(record_count, true, use_osm);
+
+    auto de_wss = ExtendedWSS(buffer_cap, scale_factor, max_delete_prop);
+
+    std::fstream datafile;
+    datafile.open(filename, std::ios::in);
+
+    std::vector<WRec> to_delete;
+
+    // warm up the tree with initial_insertions number of initially inserted
+    // records
+    size_t warmup_cnt = insert_batch * record_count;
+    warmup<ExtendedWSS, WRec>(datafile, de_wss, warmup_cnt, delete_prop, to_delete);
+
+    size_t insert_cnt = record_count - warmup_cnt;
+
+    std::vector<de::wss_query_parms<WRec>> queries(1);
+    queries[0].rng = g_rng;
+    queries[0].sample_size = 1000;
+
+    insert_tput_bench<ExtendedWSS, WRec>(de_wss, datafile, insert_cnt, delete_prop, to_delete);
+    query_latency_bench<ExtendedWSS, WRec, de::wss_query_parms<WRec>>(de_wss, queries, 1000);
+    fprintf(stdout, "\n");
+
+    delete_bench_env();
+    fflush(stdout);
+    fflush(stderr);
+
+    exit(EXIT_SUCCESS);
+}
diff --git a/benchmarks/include/bench.h b/benchmarks/include/bench.h
index d82da48..6a9c263 100644
--- a/benchmarks/include/bench.h
+++ b/benchmarks/include/bench.h
@@ -1,3 +1,13 @@
+/*
+ * benchmarks/include/bench.h
+ *
+ * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> 
+ *
+ * All rights reserved. Published under the Modified BSD License.
+ *
+ */
+#pragma once
+
 #include "bench_utility.h"
 
 template <typename DE, de::RecordInterface R, bool PROGRESS=true, size_t BATCH=1000>
@@ -64,4 +74,37 @@ static bool insert_tput_bench(DE &de_index, std::fstream &file, size_t insert_cn
     return continue_benchmark;
 }
 
+template <typename DE, de::RecordInterface R, typename QP, bool PROGRESS=true>
+static bool query_latency_bench(DE &de_index, std::vector<QP> queries, size_t trial_cnt=100) {
+    char progbuf[25];
+    if constexpr (PROGRESS) {
+        sprintf(progbuf, "querying:");
+    }
+
+    size_t total_time = 0;
+    size_t total_results = 0;
+
+    for (size_t i=0; i<trial_cnt; i++) {
+        if constexpr (PROGRESS) {
+            progress_update((double) (i) / (double) trial_cnt, progbuf);
+        }
+
+        auto start = std::chrono::high_resolution_clock::now();
+        for (size_t j=0; j<queries.size(); j++) {
+            auto res = de_index.query(&queries[j]);
+            total_results += res.size();
+        }
+        auto stop = std::chrono::high_resolution_clock::now();
+
+        total_time += std::chrono::duration_cast<std::chrono::nanoseconds>(stop - start).count();
+    }
+
+    progress_update(1.0, progbuf);
+
+    size_t query_latency = total_time / (trial_cnt * queries.size());
 
+    fprintf(stdout, "%ld\t", query_latency);
+    fflush(stdout);
+
+    return true;
+}
diff --git a/benchmarks/include/bench_utility.h b/benchmarks/include/bench_utility.h
index 78d6415..f208487 100644
--- a/benchmarks/include/bench_utility.h
+++ b/benchmarks/include/bench_utility.h
@@ -1,5 +1,13 @@
-#ifndef H_BENCH
-#define H_BENCH
+/*
+ * benchmarks/include/bench_utility.h
+ *
+ * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> 
+ *
+ * All rights reserved. Published under the Modified BSD License.
+ *
+ */
+#pragma once
+
 #include "framework/DynamicExtension.h"
 #include "shard/WSS.h"
 #include "shard/MemISAM.h"
@@ -29,7 +37,7 @@ typedef de::WeightedRecord<key_type, value_type, weight_type> WRec;
 typedef de::Record<key_type, value_type> Rec;
 
 typedef de::DynamicExtension<WRec, de::WSS<WRec>, de::WSSQuery<WRec>> ExtendedWSS;
-typedef de::DynamicExtension<Rec, de::TrieSpline<Rec>, de::TrieSplineRangeQuery<Rec>> ExtendedTS;
+typedef de::DynamicExtension<Rec, de::TrieSpline<Rec>, de::TrieSplineRangeQuery<Rec>> ExtendedTSRQ;
 typedef de::DynamicExtension<Rec, de::PGM<Rec>, de::PGMRangeQuery<Rec>> ExtendedPGM;
 
 static gsl_rng *g_rng;
@@ -82,6 +90,31 @@ static void delete_bench_env()
     delete g_to_delete;
 }
 
+/*
+ * NOTE: The QP type must have lower_bound and upper_bound attributes, which
+ * this function will initialize. Any other query parameter attributes must
+ * be manually initialized after the call.
+ */
+template <typename QP>
+static std::vector<QP> read_range_queries(std::string fname, double selectivity) {
+    std::vector<QP> queries;
+
+    FILE *qf = fopen(fname.c_str(), "r");
+    size_t start, stop;
+    double sel;
+    while (fscanf(qf, "%zu%zu%lf\n", &start, &stop, &sel) != EOF) {
+        if (start < stop && std::abs(sel - selectivity) < 0.1) {
+            QP q;
+            q.lower_bound = start;
+            q.upper_bound = stop;
+            queries.push_back(q);
+        }
+    }
+    fclose(qf);
+
+    return queries;
+}
+
 template <de::RecordInterface R>
 static bool next_record(std::fstream &file, R &record)
 {
@@ -213,12 +246,6 @@ static bool warmup(std::fstream &file, DE &extended_index, size_t count,
         }
     }
 
-    /*
-    if (progress) {
-        progress_update(1, "warming up:");
-    }
-    */
-
     return true;
 }
 
@@ -232,5 +259,3 @@ static void reset_de_perf_metrics() {
 
     RESET_IO_CNT(); 
 }
-
-#endif // H_BENCH
diff --git a/benchmarks/sampling_tput.cpp b/benchmarks/sampling_tput.cpp
deleted file mode 100644
index 70ff0cb..0000000
--- a/benchmarks/sampling_tput.cpp
+++ /dev/null
@@ -1,82 +0,0 @@
-#include "include/bench.h"
-
-static void sample_benchmark(ExtendedWSS &de_wss, size_t k, size_t trial_cnt)
-{
-    char progbuf[25];
-    sprintf(progbuf, "sampling (%ld):", k);
-
-    size_t batch_size = 100;
-    size_t batches = trial_cnt / batch_size;
-    size_t total_time = 0;
-
-    WRec sample_set[k];
-
-    size_t total_samples = 0;
-
-    de::wss_query_parms<WRec> parms;
-    parms.rng = g_rng;
-    parms.sample_size = k;
-
-    for (int i=0; i<batches; i++) {
-        progress_update((double) (i * batch_size) / (double) trial_cnt, progbuf);
-        auto start = std::chrono::high_resolution_clock::now();
-        for (int j=0; j < batch_size; j++) {
-            auto res = de_wss.query(&parms);
-            total_samples += res.size();
-        }
-        auto stop = std::chrono::high_resolution_clock::now();
-
-        total_time += std::chrono::duration_cast<std::chrono::nanoseconds>(stop - start).count();
-    }
-
-    progress_update(1.0, progbuf);
-
-    size_t throughput = (((double)(total_samples) / (double) total_time) * 1e9);
-
-    fprintf(stdout, "%ld\n", throughput);
-    fflush(stdout);
-}
-
-
-int main(int argc, char **argv)
-{
-    if (argc < 7) {
-        fprintf(stderr, "Usage: sampling_tput <filename> <record_count> <buffer_cap> <scale_factor> <delete_proportion> <max_delete_proportion> [osm_data]\n");
-        exit(EXIT_FAILURE);
-    }
-
-    std::string filename = std::string(argv[1]);
-    size_t record_count = atol(argv[2]);
-    size_t buffer_cap = atol(argv[3]);
-    size_t scale_factor = atol(argv[4]);
-    double delete_prop = atof(argv[5]);
-    double max_delete_prop = atof(argv[6]);
-    bool use_osm = (argc == 8) ? atoi(argv[7]) : 0;
-
-    double insert_batch = 0.1; 
-
-    init_bench_env(record_count, true, use_osm);
-
-    auto de_wss = ExtendedWSS(buffer_cap, scale_factor, max_delete_prop);
-
-    std::fstream datafile;
-    datafile.open(filename, std::ios::in);
-
-    std::vector<WRec> to_delete;
-
-    // warm up the tree with initial_insertions number of initially inserted
-    // records
-    size_t warmup_cnt = insert_batch * record_count;
-    warmup<ExtendedWSS, WRec>(datafile, de_wss, warmup_cnt, delete_prop, to_delete);
-
-    size_t insert_cnt = record_count - warmup_cnt;
-
-    insert_tput_bench<ExtendedWSS, WRec>(de_wss, datafile, insert_cnt, delete_prop, to_delete);
-    sample_benchmark(de_wss, 1000, 10000);
-
-    delete_bench_env();
-    fflush(stdout);
-    fflush(stderr);
-
-    exit(EXIT_SUCCESS);
-}