Additional layout policies + more flexibility in buffer flushing

4 files changed, 128 insertions, 29 deletions

diff --git a/benchmarks/include/standard_benchmarks.h b/benchmarks/include/standard_benchmarks.h
index dfa6513..76423ab 100644
--- a/benchmarks/include/standard_benchmarks.h
+++ b/benchmarks/include/standard_benchmarks.h
@@ -22,6 +22,7 @@
 #include "framework/reconstruction/LevelingPolicy.h"
 #include "framework/reconstruction/TieringPolicy.h"
 #include "framework/reconstruction/BSMPolicy.h"
+#include "framework/reconstruction/FloodL0Policy.h"
 
 constexpr double delete_proportion = 0.05;
 static size_t g_deleted_records = 0;
@@ -38,7 +39,9 @@ de::ReconstructionPolicy<S, Q> *get_policy(size_t scale_factor, size_t buffer_si
         recon = new de::LevelingPolicy<S, Q>(scale_factor, buffer_size);
     } else if (policy == 2) {
         recon = new de::BSMPolicy<S, Q>(buffer_size);
-    } 
+    } else if (policy == 3) {
+        recon = new de::FloodL0Policy<S, Q>(buffer_size);
+    }
 
     return recon;
 }
diff --git a/benchmarks/tail-latency/config_sweep.cpp b/benchmarks/tail-latency/config_sweep.cpp
index ef84aa7..d973ee5 100644
--- a/benchmarks/tail-latency/config_sweep.cpp
+++ b/benchmarks/tail-latency/config_sweep.cpp
@@ -2,6 +2,7 @@
  *
  */
 
+#include "framework/scheduling/FIFOScheduler.h"
 #define ENABLE_TIMER
 #define TS_TEST
 
@@ -22,7 +23,7 @@
 typedef de::Record<uint64_t, uint64_t> Rec;
 typedef de::TrieSpline<Rec> Shard;
 typedef de::rc::Query<Shard> Q;
-typedef de::DynamicExtension<Shard, Q, de::DeletePolicy::TOMBSTONE, de::SerialScheduler> Ext;
+typedef de::DynamicExtension<Shard, Q, de::DeletePolicy::TOMBSTONE, de::FIFOScheduler> Ext;
 typedef Q::Parameters QP;
 
 void usage(char *progname) {
@@ -44,15 +45,15 @@ int main(int argc, char **argv) {
     auto data = read_sosd_file<Rec>(d_fname, n);
     auto queries = read_range_queries<QP>(q_fname, .0001);
 
-    std::vector<int> policies = {0, 1};
-    std::vector<size_t> buffers = {4000, 8000, 12000, 16000, 20000};
-    std::vector<size_t> sfs = {2, 4, 6, 8, 12};
+    std::vector<int> policies = {3};
+    std::vector<size_t> buffers = {8000, 16000, 32000};
+    std::vector<size_t> sfs = {8};
 
     for (size_t l=0; l<policies.size(); l++) {
     for (size_t j=0; j<buffers.size(); j++) {
     for (size_t k=0; k<sfs.size(); k++) {
         auto policy = get_policy<Shard, Q>(sfs[k], buffers[j], policies[l]);
-        auto extension = new Ext(policy, 8000);
+        auto extension = new Ext(policy, buffers[j]/4, buffers[j]);
 
         /* warmup structure w/ 10% of records */
         size_t warmup = .1 * n;
@@ -73,8 +74,23 @@ int main(int argc, char **argv) {
             }
             TIMER_STOP();
 
+            //fprintf(stdout, "%ld\t%ld\t%d\t%ld\n", sfs[k], buffers[j], policies[l], TIMER_RESULT());
+        }
+
+        extension->await_next_epoch();
+        
+        /* repeat the queries a bunch of times */
+        for (size_t l=0; l<10; l++) {
+        for (size_t i=0; i<queries.size(); i++) {
+            TIMER_START();
+            auto q = queries[i];
+            auto res = extension->query(std::move(q));
+            res.get();
+            TIMER_STOP();
+
             fprintf(stdout, "%ld\t%ld\t%d\t%ld\n", sfs[k], buffers[j], policies[l], TIMER_RESULT());
         }
+        }
 
 
         QP p = {0, 10000};
@@ -85,29 +101,6 @@ int main(int argc, char **argv) {
         delete extension;
     }}}
 
-    /*
-    std::vector<int64_t> query_latencies;
-    query_latencies.reserve(queries.size());
-    for (size_t i=warmup; i<data.size(); i++) {
-        TIMER_START();
-        auto q = queries[i];
-        auto res = extension->query(std::move(q));
-        res.get();
-        TIMER_STOP();
-
-        query_latencies.push_back(TIMER_RESULT());
-    }
-
-    printf("here\n");
-
-    for (size_t i=0; i<insert_latencies.size(); i++) {
-        fprintf(stdout, "I\t%ld\n", insert_latencies[i]);
-    }
-
-    for (size_t i=0; i<query_latencies.size(); i++) {
-        fprintf(stdout, "Q\t%ld\n", query_latencies[i]);
-    }
-    */
 
     fflush(stderr);
 }
diff --git a/benchmarks/tail-latency/fixed_shard_count.cpp b/benchmarks/tail-latency/fixed_shard_count.cpp
new file mode 100644
index 0000000..fecf599
--- /dev/null
+++ b/benchmarks/tail-latency/fixed_shard_count.cpp
@@ -0,0 +1,101 @@
+/*
+ *
+ */
+
+#define ENABLE_TIMER
+#define TS_TEST
+
+#include <thread>
+
+#include "framework/DynamicExtension.h"
+#include "framework/scheduling/FIFOScheduler.h"
+#include "shard/TrieSpline.h"
+#include "query/rangecount.h"
+#include "framework/interface/Record.h"
+#include "file_util.h"
+#include "standard_benchmarks.h"
+
+#include "framework/reconstruction/FixedShardCountPolicy.h"
+
+#include <gsl/gsl_rng.h>
+
+#include "psu-util/timer.h"
+
+
+typedef de::Record<uint64_t, uint64_t> Rec;
+typedef de::TrieSpline<Rec> Shard;
+typedef de::rc::Query<Shard> Q;
+typedef de::DynamicExtension<Shard, Q, de::DeletePolicy::TOMBSTONE, de::FIFOScheduler> Ext;
+typedef Q::Parameters QP;
+
+void usage(char *progname) {
+    fprintf(stderr, "%s reccnt datafile queryfile\n", progname);
+}
+
+int main(int argc, char **argv) {
+
+    if (argc < 4) {
+        usage(argv[0]);
+        exit(EXIT_FAILURE);
+    }
+
+    size_t n = atol(argv[1]);
+    std::string d_fname = std::string(argv[2]);
+    std::string q_fname = std::string(argv[3]);
+
+    
+    auto data = read_sosd_file<Rec>(d_fname, n);
+    auto queries = read_range_queries<QP>(q_fname, .0001);
+
+    std::vector<size_t> shard_counts = {4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 4096*2};
+    size_t buffer_size = 8000;
+
+    for (size_t i=0; i<shard_counts.size(); i++) {
+        auto policy = new de::FixedShardCountPolicy<Shard,Q>(buffer_size, shard_counts[i], n);
+        auto extension = new Ext(policy, buffer_size / 4, buffer_size);
+
+        /* warmup structure w/ 10% of records */
+        size_t warmup = .1 * n;
+        for (size_t i=0; i<warmup; i++) {
+            while (!extension->insert(data[i])) {
+                usleep(1);
+            }
+        }
+
+        extension->await_next_epoch();
+
+        TIMER_INIT();
+
+        TIMER_START();
+        for (size_t i=warmup; i<data.size(); i++) {
+            while (!extension->insert(data[i])) {
+                usleep(1);
+            }
+        }
+        TIMER_STOP();
+
+        auto insert_tput = (size_t) ((double) (n - warmup) / (double) TIMER_RESULT() *1.0e9);
+
+        extension->await_next_epoch();
+        
+        /* repeat the queries a bunch of times */
+        TIMER_START();
+        for (size_t l=0; l<10; l++) {
+        for (size_t i=0; i<queries.size(); i++) {
+            auto q = queries[i];
+            auto res = extension->query(std::move(q));
+            res.get();
+        }
+        }
+        TIMER_STOP();
+
+        auto query_lat = TIMER_RESULT() / 10*queries.size();
+
+        fprintf(stdout, "%ld\t%ld\t%ld\t%ld\n", shard_counts[i], extension->get_shard_count(), insert_tput, query_lat);
+
+        delete extension;
+    }
+
+    fflush(stderr);
+}
+
diff --git a/benchmarks/vldb/ts_bench.cpp b/benchmarks/vldb/ts_bench.cpp
index 8b0ee35..1182376 100644
--- a/benchmarks/vldb/ts_bench.cpp
+++ b/benchmarks/vldb/ts_bench.cpp
@@ -89,6 +89,8 @@ int main(int argc, char **argv) {
 
     fprintf(stdout, "%ld\t%ld\t%ld\t%ld\t%ld\n", insert_throughput, query_latency, ext_size, static_latency, static_size);
 
+    fprintf(stdout, "%ld\n", extension->get_height());
+
     gsl_rng_free(rng);
     delete extension;
     fflush(stderr);