Record format generalization

Currently, tombstone counting is bugged. But the rest of it appears to
be working.

6 files changed, 213 insertions, 175 deletions

diff --git a/tests/dynamic_extension_tests.cpp b/tests/dynamic_extension_tests.cpp
index ce2abdb..3638b76 100644
--- a/tests/dynamic_extension_tests.cpp
+++ b/tests/dynamic_extension_tests.cpp
@@ -19,11 +19,9 @@
 #include <check.h>
 using namespace de;
 
-typedef DynamicExtension<uint64_t, uint32_t, uint64_t> DE_WIRS;
-
 START_TEST(t_create)
 {
-    auto ext_wirs = new DE_WIRS(100, 100, 2, 1, 1, g_rng);
+    auto ext_wirs = new DynamicExtension<WRec>(100, 100, 2, 1, 1, g_rng);
 
 
     ck_assert_ptr_nonnull(ext_wirs);
@@ -37,12 +35,13 @@ END_TEST
 
 START_TEST(t_append)
 {
-    auto ext_wirs = new DE_WIRS(100, 100, 2, 1, 1, g_rng);
+    auto ext_wirs = new DynamicExtension<WRec>(100, 100, 2, 1, 1, g_rng);
 
     uint64_t key = 0;
     uint32_t val = 0;
     for (size_t i=0; i<100; i++) {
-        ck_assert_int_eq(ext_wirs->append(key, val, 1, false, g_rng), 1);
+        WRec r = {key, val, 1};
+        ck_assert_int_eq(ext_wirs->append(r, g_rng), 1);
         key++;
         val++;
     }
@@ -57,12 +56,13 @@ END_TEST
 
 START_TEST(t_append_with_mem_merges)
 {
-    auto ext_wirs = new DE_WIRS(100, 100, 2, 1, 1, g_rng);
+    auto ext_wirs = new DynamicExtension<WRec>(100, 100, 2, 1, 1, g_rng);
 
     uint64_t key = 0;
     uint32_t val = 0;
     for (size_t i=0; i<300; i++) {
-        ck_assert_int_eq(ext_wirs->append(key, val, 1, false, g_rng), 1);
+        WRec r = {key, val, 1};
+        ck_assert_int_eq(ext_wirs->append(r, g_rng), 1);
         key++;
         val++;
     }
@@ -77,12 +77,13 @@ END_TEST
 
 START_TEST(t_range_sample_memtable)
 {
-    auto ext_wirs = new DE_WIRS(100, 100, 2, 1, 1, g_rng);
+    auto ext_wirs = new DynamicExtension<WRec>(100, 100, 2, 1, 1, g_rng);
 
     uint64_t key = 0;
     uint32_t val = 0;
     for (size_t i=0; i<100; i++) {
-        ck_assert_int_eq(ext_wirs->append(key, val, 1, false, g_rng), 1);
+        WRec r = {key, val, 1};
+        ck_assert_int_eq(ext_wirs->append(r, g_rng), 1);
         key++;
         val++;
     }
@@ -92,7 +93,7 @@ START_TEST(t_range_sample_memtable)
 
     char *buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE);
     char *util_buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE);
-    WeightedRec sample_set[100];
+    WRec sample_set[100];
 
     ext_wirs->range_sample(sample_set, lower_bound, upper_bound, 100, g_rng);
 
@@ -111,12 +112,13 @@ END_TEST
 
 START_TEST(t_range_sample_memlevels)
 {
-    auto ext_wirs = new DE_WIRS(100, 100, 2, 1, 1, g_rng);
+    auto ext_wirs = new DynamicExtension<WRec>(100, 100, 2, 1, 1, g_rng);
 
     uint64_t key = 0;
     uint32_t val = 0;
     for (size_t i=0; i<300; i++) {
-        ck_assert_int_eq(ext_wirs->append(key, val, 1, false, g_rng), 1);
+        WRec r = {key, val, 1};
+        ck_assert_int_eq(ext_wirs->append(r, g_rng), 1);
         key++;
         val++;
     }
@@ -127,7 +129,7 @@ START_TEST(t_range_sample_memlevels)
     char *buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE);
     char *util_buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE);
 
-    WeightedRec sample_set[100];
+    WRec sample_set[100];
     ext_wirs->range_sample(sample_set, lower_bound, upper_bound, 100, g_rng);
 
     for(size_t i=0; i<100; i++) {
@@ -144,7 +146,7 @@ END_TEST
 
 START_TEST(t_range_sample_weighted)
 {
-    auto ext_wirs = new DE_WIRS(100, 100, 2, 1, 1, g_rng);
+    auto ext_wirs = new DynamicExtension<WRec>(100, 100, 2, 1, 1, g_rng);
     size_t n = 10000;
 
     std::vector<uint64_t> keys;
@@ -171,22 +173,23 @@ START_TEST(t_range_sample_weighted)
     std::shuffle(keys.begin(), keys.end(), gen);
 
     for (size_t i=0; i<keys.size(); i++) {
-        double weight;
+        uint64_t weight;
         if (keys[i] == 1)  {
-            weight = 2.0;
+            weight = 2;
         } else if (keys[i] == 2) {
-            weight = 4.0;
+            weight = 4;
         } else {
-            weight = 8.0;
+            weight = 8;
         }
 
-        ext_wirs->append(keys[i], i, weight, false, g_rng);
+        WRec r = {keys[i], (uint32_t) i, weight, 0};
+        ext_wirs->append(r, g_rng);
     }
     size_t k = 1000;
     uint64_t lower_key = 0;
     uint64_t upper_key = 5;
 
-    WeightedRec* buffer = new WeightedRec[k]();
+    WRec* buffer = new WRec[k]();
     char *buffer1 = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE);
     char *buffer2 = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE);
 
@@ -214,7 +217,7 @@ END_TEST
 START_TEST(t_tombstone_merging_01)
 {
     size_t reccnt = 100000;
-    auto ext_wirs = new DE_WIRS(100, 100, 2, .01, 1, g_rng);
+    auto ext_wirs = new DynamicExtension<WRec>(100, 100, 2, .01, 1, g_rng);
 
     std::set<std::pair<uint64_t, uint32_t>> records; 
     std::set<std::pair<uint64_t, uint32_t>> to_delete;
@@ -232,14 +235,17 @@ START_TEST(t_tombstone_merging_01)
     size_t deletes = 0;
     size_t cnt=0;
     for (auto rec : records) {
-        ck_assert_int_eq(ext_wirs->append(rec.first, rec.second, 1, false, g_rng), 1);
+        WRec r = {rec.first, rec.second, 1, 0};
+        ck_assert_int_eq(ext_wirs->append(r, g_rng), 1);
 
          if (gsl_rng_uniform(g_rng) < 0.05 && !to_delete.empty()) {
             std::vector<std::pair<uint64_t, uint32_t>> del_vec;
             std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()});
 
             for (size_t i=0; i<del_vec.size(); i++) {
-                ext_wirs->append(del_vec[i].first, del_vec[i].second, 1, true, g_rng);
+                WRec dr = {del_vec[i].first, del_vec[i].second, 1};
+                dr.set_tombstone();
+                ext_wirs->append(dr, g_rng);
                 deletes++;
                 to_delete.erase(del_vec[i]);
                 deleted.insert(del_vec[i]);
@@ -259,12 +265,12 @@ START_TEST(t_tombstone_merging_01)
 }
 END_TEST
 
-DE_WIRS *create_test_tree(size_t reccnt, size_t memlevel_cnt) {
-    auto ext_wirs = new DE_WIRS(1000, 1000, 2, 1, 1, g_rng);
+DynamicExtension<WRec> *create_test_tree(size_t reccnt, size_t memlevel_cnt) {
+    auto ext_wirs = new DynamicExtension<WRec>(1000, 1000, 2, 1, 1, g_rng);
 
-    std::set<std::pair<uint64_t, uint32_t>> records; 
-    std::set<std::pair<uint64_t, uint32_t>> to_delete;
-    std::set<std::pair<uint64_t, uint32_t>> deleted;
+    std::set<WRec> records; 
+    std::set<WRec> to_delete;
+    std::set<WRec> deleted;
 
     while (records.size() < reccnt) {
         uint64_t key = rand();
@@ -277,14 +283,15 @@ DE_WIRS *create_test_tree(size_t reccnt, size_t memlevel_cnt) {
 
     size_t deletes = 0;
     for (auto rec : records) {
-        ck_assert_int_eq(ext_wirs->append(rec.first, rec.second, 1, 0, g_rng), 1);
+        ck_assert_int_eq(ext_wirs->append(rec, g_rng), 1);
 
          if (gsl_rng_uniform(g_rng) < 0.05 && !to_delete.empty()) {
-            std::vector<std::pair<uint64_t, uint32_t>> del_vec;
+            std::vector<WRec> del_vec;
             std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()});
 
             for (size_t i=0; i<del_vec.size(); i++) {
-                ext_wirs->append(del_vec[i].first, del_vec[i].second, 1, true, g_rng);
+                del_vec[i].set_tombstone();
+                ext_wirs->append(del_vec[i], g_rng);
                 deletes++;
                 to_delete.erase(del_vec[i]);
                 deleted.insert(del_vec[i]);
@@ -302,7 +309,7 @@ DE_WIRS *create_test_tree(size_t reccnt, size_t memlevel_cnt) {
 START_TEST(t_sorted_array)
 {
     size_t reccnt = 100000;
-    auto ext_wirs = new DE_WIRS(100, 100, 2, 1, 1, g_rng);
+    auto ext_wirs = new DynamicExtension<WRec>(100, 100, 2, 1, 1, g_rng);
 
     std::set<std::pair<uint64_t, uint32_t>> records; 
     std::set<std::pair<uint64_t, uint32_t>> to_delete;
@@ -319,14 +326,18 @@ START_TEST(t_sorted_array)
 
     size_t deletes = 0;
     for (auto rec : records) {
-        ck_assert_int_eq(ext_wirs->append(rec.first, rec.second, 1, 0, g_rng), 1);
+        WRec r = {rec.first, rec.second, 1};
+        ck_assert_int_eq(ext_wirs->append(r, g_rng), 1);
 
          if (gsl_rng_uniform(g_rng) < 0.05 && !to_delete.empty()) {
             std::vector<std::pair<uint64_t, uint32_t>> del_vec;
             std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()});
 
             for (size_t i=0; i<del_vec.size(); i++) {
-                ext_wirs->append(del_vec[i].first, del_vec[i].second, 1, true, g_rng);
+                WRec dr = {del_vec[i].first, del_vec[i].second, 1};
+                dr.set_tombstone();
+
+                ext_wirs->append(dr , g_rng);
                 deletes++;
                 to_delete.erase(del_vec[i]);
                 deleted.insert(del_vec[i]);
diff --git a/tests/internal_level_tests.cpp b/tests/internal_level_tests.cpp
index 7e542e6..7842b01 100644
--- a/tests/internal_level_tests.cpp
+++ b/tests/internal_level_tests.cpp
@@ -20,19 +20,19 @@ using namespace de;
 
 START_TEST(t_memlevel_merge)
 {
-    auto tbl1 = create_test_mbuffer<uint64_t, uint32_t, uint64_t>(100);
-    auto tbl2 = create_test_mbuffer<uint64_t, uint32_t, uint64_t>(100);
+    auto tbl1 = create_test_mbuffer<WRec>(100);
+    auto tbl2 = create_test_mbuffer<WRec>(100);
 
-    auto base_level = new WeightedLevel(1, 1, false);
+    auto base_level = new InternalLevel<WRec>(1, 1, false);
     base_level->append_mem_table(tbl1, g_rng);
     ck_assert_int_eq(base_level->get_record_cnt(), 100);
 
-    auto merging_level = new WeightedLevel(0, 1, false);
+    auto merging_level = new InternalLevel<WRec>(0, 1, false);
     merging_level->append_mem_table(tbl2, g_rng);
     ck_assert_int_eq(merging_level->get_record_cnt(), 100);
 
     auto old_level = base_level;
-    base_level = WeightedLevel::merge_levels(old_level, merging_level, false, g_rng);
+    base_level = InternalLevel<WRec>::merge_levels(old_level, merging_level, false, g_rng);
 
     delete old_level;
     delete merging_level;
@@ -44,11 +44,11 @@ START_TEST(t_memlevel_merge)
 }
 
 
-WeightedLevel *create_test_memlevel(size_t reccnt) {
-    auto tbl1 = create_test_mbuffer<uint64_t, uint32_t, uint64_t>(reccnt/2);
-    auto tbl2 = create_test_mbuffer<uint64_t, uint32_t, uint64_t>(reccnt/2);
+InternalLevel<WRec> *create_test_memlevel(size_t reccnt) {
+    auto tbl1 = create_test_mbuffer<WRec>(reccnt/2);
+    auto tbl2 = create_test_mbuffer<WRec>(reccnt/2);
 
-    auto base_level = new WeightedLevel(1, 2, false);
+    auto base_level = new InternalLevel<WRec>(1, 2, false);
     base_level->append_mem_table(tbl1, g_rng);
     base_level->append_mem_table(tbl2, g_rng);
 
diff --git a/tests/memisam_tests.cpp b/tests/memisam_tests.cpp
index f3b80b3..1609edf 100644
--- a/tests/memisam_tests.cpp
+++ b/tests/memisam_tests.cpp
@@ -8,24 +8,25 @@
 
 using namespace de;
 
-typedef MemISAM<uint64_t, uint32_t> M_ISAM;
+typedef MemISAM<Rec> M_ISAM;
 
 START_TEST(t_memtable_init)
 {
-    auto buffer = new UnweightedMBuffer(1024, true, 512, g_rng);
+    auto buffer = new MutableBuffer<Rec>(1024, true, 512, g_rng);
     for (uint64_t i = 512; i > 0; i--) {
-        uint32_t v = i;
-        buffer->append(i, v);
+        buffer->append(Rec {i, (uint32_t)i});
     }
     
+    Rec r;
+    r.set_tombstone();
     for (uint64_t i = 1; i <= 256; ++i) {
-        uint32_t v = i;
-        buffer->append(i, v, true);
+        r.key = i;
+        r.value = i;
+        buffer->append(r);
     }
 
     for (uint64_t i = 257; i <= 512; ++i) {
-        uint32_t v = i + 1;
-        buffer->append(i, v);
+        buffer->append({i, (uint32_t) i+1});
     }
 
     BloomFilter* bf = new BloomFilter(BF_FPR, buffer->get_tombstone_count(), BF_HASH_FUNCS, g_rng);
@@ -40,9 +41,9 @@ START_TEST(t_memtable_init)
 START_TEST(t_inmemrun_init)
 {
     size_t n = 512;
-    auto memtable1 = create_test_mbuffer<uint64_t, uint32_t>(n);
-    auto memtable2 = create_test_mbuffer<uint64_t, uint32_t>(n);
-    auto memtable3 = create_test_mbuffer<uint64_t, uint32_t>(n);
+    auto memtable1 = create_test_mbuffer<Rec>(n);
+    auto memtable2 = create_test_mbuffer<Rec>(n);
+    auto memtable3 = create_test_mbuffer<Rec>(n);
 
     BloomFilter* bf1 = new BloomFilter(100, BF_HASH_FUNCS, g_rng);
     BloomFilter* bf2 = new BloomFilter(100, BF_HASH_FUNCS, g_rng);
@@ -70,11 +71,11 @@ START_TEST(t_inmemrun_init)
 
         auto cur_rec = run4->get_record_at(i);
 
-        if (run1_idx < n && cur_rec->match(rec1)) {
+        if (run1_idx < n && cur_rec == rec1) {
             ++run1_idx;
-        } else if (run2_idx < n && cur_rec->match(rec2)) {
+        } else if (run2_idx < n && cur_rec == rec2) {
             ++run2_idx;
-        } else if (run3_idx < n && cur_rec->match(rec3)) {
+        } else if (run3_idx < n && cur_rec == rec3) {
             ++run3_idx;
         } else {
            assert(false);
@@ -98,7 +99,7 @@ START_TEST(t_inmemrun_init)
 START_TEST(t_get_lower_bound_index)
 {
     size_t n = 10000;
-    auto memtable = create_double_seq_mbuffer<uint64_t, uint32_t>(n);
+    auto memtable = create_double_seq_mbuffer<Rec>(n);
 
     ck_assert_ptr_nonnull(memtable);
     BloomFilter* bf = new BloomFilter(100, BF_HASH_FUNCS, g_rng);
@@ -109,7 +110,7 @@ START_TEST(t_get_lower_bound_index)
 
     auto tbl_records = memtable->sorted_output();
     for (size_t i=0; i<n; i++) {
-        const UnweightedRec *tbl_rec = memtable->get_record_at(i);
+        const auto *tbl_rec = memtable->get_record_at(i);
         auto pos = run->get_lower_bound(tbl_rec->key);
         ck_assert_int_eq(run->get_record_at(pos)->key, tbl_rec->key);
         ck_assert_int_le(pos, i);
@@ -123,7 +124,7 @@ START_TEST(t_get_lower_bound_index)
 START_TEST(t_get_upper_bound_index)
 {
     size_t n = 10000;
-    auto memtable = create_double_seq_mbuffer<uint64_t, uint32_t>(n);
+    auto memtable = create_double_seq_mbuffer<Rec>(n);
 
     ck_assert_ptr_nonnull(memtable);
     BloomFilter* bf = new BloomFilter(100, BF_HASH_FUNCS, g_rng);
@@ -134,7 +135,7 @@ START_TEST(t_get_upper_bound_index)
 
     auto tbl_records = memtable->sorted_output();
     for (size_t i=0; i<n; i++) {
-        const UnweightedRec *tbl_rec = memtable->get_record_at(i);
+        const auto *tbl_rec = memtable->get_record_at(i);
         auto pos = run->get_upper_bound(tbl_rec->key);
         ck_assert(pos == run->get_record_count() ||
                   run->get_record_at(pos)->key > tbl_rec->key);
@@ -150,8 +151,8 @@ START_TEST(t_get_upper_bound_index)
 START_TEST(t_full_cancelation)
 {
     size_t n = 100;
-    auto mtable = create_double_seq_mbuffer<uint64_t, uint32_t>(n, false);
-    auto mtable_ts = create_double_seq_mbuffer<uint64_t, uint32_t>(n, true);
+    auto mtable = create_double_seq_mbuffer<Rec>(n, false);
+    auto mtable_ts = create_double_seq_mbuffer<Rec>(n, true);
     BloomFilter* bf1 = new BloomFilter(100, BF_HASH_FUNCS, g_rng);
     BloomFilter* bf2 = new BloomFilter(100, BF_HASH_FUNCS, g_rng);
     BloomFilter* bf3 = new BloomFilter(100, BF_HASH_FUNCS, g_rng);
diff --git a/tests/mutable_buffer_tests.cpp b/tests/mutable_buffer_tests.cpp
index 2112036..75cbeec 100644
--- a/tests/mutable_buffer_tests.cpp
+++ b/tests/mutable_buffer_tests.cpp
@@ -20,12 +20,14 @@
 
 #include <check.h>
 
+#define DE_MT_TEST 0
+
 using namespace de;
 
 START_TEST(t_create)
 {
     auto rng = gsl_rng_alloc(gsl_rng_mt19937);
-    auto buffer = new WeightedMBuffer(100, true, 50, rng);
+    auto buffer = new MutableBuffer<Rec>(100, true, 50, rng);
 
     ck_assert_ptr_nonnull(buffer);
     ck_assert_int_eq(buffer->get_capacity(), 100);
@@ -44,30 +46,32 @@ END_TEST
 START_TEST(t_insert)
 {
     auto rng = gsl_rng_alloc(gsl_rng_mt19937);
-    auto buffer = new WeightedMBuffer(100, true, 50, rng);
+    auto buffer = new MutableBuffer<WRec>(100, true, 50, rng);
 
     uint64_t key = 0;
     uint32_t val = 5;
 
+    WRec rec = {0, 5, 1};
+
     for (size_t i=0; i<99; i++) {
-        ck_assert_int_eq(buffer->append(key, val, 1, false), 1);
-        ck_assert_int_eq(buffer->check_tombstone(key, val), 0);
+        ck_assert_int_eq(buffer->append(rec), 1);
+        ck_assert_int_eq(buffer->check_tombstone(rec), 0);
 
-        key++;
-        val++;
+        rec.key++;
+        rec.value++;
 
         ck_assert_int_eq(buffer->get_record_count(), i+1);
         ck_assert_int_eq(buffer->get_tombstone_count(), 0);
         ck_assert_int_eq(buffer->is_full(), 0);
     }
 
-    ck_assert_int_eq(buffer->append(key, val, 1.0, false), 1);
+    ck_assert_int_eq(buffer->append(rec), 1);
 
-    key++;
-    val++;
+    rec.key++;
+    rec.value++;
 
     ck_assert_int_eq(buffer->is_full(), 1);
-    ck_assert_int_eq(buffer->append(key, val, 1.0, false), 0);
+    ck_assert_int_eq(buffer->append(rec), 0);
 
     delete buffer;
     gsl_rng_free(rng);
@@ -79,12 +83,12 @@ END_TEST
 START_TEST(t_insert_tombstones)
 {
     auto rng = gsl_rng_alloc(gsl_rng_mt19937);
-    auto buffer = new WeightedMBuffer(100, true, 50, rng);
+    auto buffer = new MutableBuffer<Rec>(100, true, 50, rng);
 
-    uint64_t key = 0;
-    uint32_t val = 5;
     size_t ts_cnt = 0;
 
+    Rec rec = {0, 5};
+
     for (size_t i=0; i<99; i++) {
         bool ts = false;
         if (i % 2 == 0) {
@@ -92,11 +96,13 @@ START_TEST(t_insert_tombstones)
             ts=true;
         }
 
-        ck_assert_int_eq(buffer->append(key, val, 1.0, ts), 1);
-        ck_assert_int_eq(buffer->check_tombstone(key, val), ts);
+        rec.set_tombstone(ts);
 
-        key++;
-        val++;
+        ck_assert_int_eq(buffer->append(rec), 1);
+        ck_assert_int_eq(buffer->check_tombstone(rec), ts);
+
+        rec.key++;
+        rec.value++;
 
         ck_assert_int_eq(buffer->get_record_count(), i+1);
         ck_assert_int_eq(buffer->get_tombstone_count(), ts_cnt);
@@ -104,16 +110,18 @@ START_TEST(t_insert_tombstones)
     }
 
     // inserting one more tombstone should not be possible
-    ck_assert_int_eq(buffer->append(key, val, 1.0, true), 0);
+    rec.set_tombstone();
+    ck_assert_int_eq(buffer->append(rec), 0);
 
 
-    ck_assert_int_eq(buffer->append(key, val, 1.0, false), 1);
+    rec.set_tombstone(false);
+    ck_assert_int_eq(buffer->append(rec), 1);
 
-    key++;
-    val++;
+    rec.key++;
+    rec.value++;
 
     ck_assert_int_eq(buffer->is_full(), 1);
-    ck_assert_int_eq(buffer->append(key, val, 1.0, false), 0);
+    ck_assert_int_eq(buffer->append(rec), 0);
 
     delete buffer;
     gsl_rng_free(rng);
@@ -124,11 +132,10 @@ END_TEST
 START_TEST(t_truncate)
 {
     auto rng = gsl_rng_alloc(gsl_rng_mt19937);
-    auto buffer = new WeightedMBuffer(100, true, 100, rng);
+    auto buffer = new MutableBuffer<Rec>(100, true, 100, rng);
 
-    uint64_t key = 0;
-    uint32_t val = 5;
     size_t ts_cnt = 0;
+    Rec rec = {0, 5};
 
     for (size_t i=0; i<100; i++) {
         bool ts = false;
@@ -137,25 +144,28 @@ START_TEST(t_truncate)
             ts=true;
         }
 
-        ck_assert_int_eq(buffer->append(key, val, 1.0, ts), 1);
-        ck_assert_int_eq(buffer->check_tombstone(key, val), ts);
+        rec.set_tombstone(ts);
 
-        key++;
-        val++;
+        ck_assert_int_eq(buffer->append(rec), 1);
+        ck_assert_int_eq(buffer->check_tombstone(rec), ts);
+
+        rec.key++;
+        rec.value++;
 
         ck_assert_int_eq(buffer->get_record_count(), i+1);
         ck_assert_int_eq(buffer->get_tombstone_count(), ts_cnt);
     }
 
     ck_assert_int_eq(buffer->is_full(), 1);
-    ck_assert_int_eq(buffer->append(key, val, 1.0, false), 0);
+    rec.set_tombstone(false);
+    ck_assert_int_eq(buffer->append(rec), 0);
 
     ck_assert_int_eq(buffer->truncate(), 1);
 
     ck_assert_int_eq(buffer->is_full(), 0);
     ck_assert_int_eq(buffer->get_record_count(), 0);
     ck_assert_int_eq(buffer->get_tombstone_count(), 0);
-    ck_assert_int_eq(buffer->append(key, val, 1.0, false), 1);
+    ck_assert_int_eq(buffer->append(rec), 1);
 
     delete buffer;
     gsl_rng_free(rng);
@@ -169,7 +179,7 @@ START_TEST(t_sorted_output)
     size_t cnt = 100;
 
     auto rng = gsl_rng_alloc(gsl_rng_mt19937);
-    auto buffer = new WeightedMBuffer(cnt, true, cnt/2, rng);
+    auto buffer = new MutableBuffer<Rec>(cnt, true, cnt/2, rng);
 
 
     std::vector<uint64_t> keys(cnt);
@@ -184,14 +194,19 @@ START_TEST(t_sorted_output)
 
     uint32_t val = 12345;
     for (size_t i=0; i<cnt-2; i++) {
-        buffer->append(keys[i], val, 1.0, false);
+        buffer->append(Rec {keys[i], val});
     }
 
-    buffer->append(keys[cnt-2], val, 1.0, true);
-    buffer->append(keys[cnt-1], val, 1.0, true);
+    Rec r1 = {keys[cnt-2], val};
+    r1.set_tombstone();
+    buffer->append(r1);
+
+    Rec r2 = {keys[cnt-1], val};
+    r2.set_tombstone();
+    buffer->append(r2);
 
 
-    WeightedRec *sorted_records = buffer->sorted_output();
+    auto *sorted_records = buffer->sorted_output();
     std::sort(keys.begin(), keys.end());
 
     for (size_t i=0; i<cnt; i++) {
@@ -204,23 +219,24 @@ START_TEST(t_sorted_output)
 END_TEST
 
 
-void insert_records(std::vector<std::pair<uint64_t, uint32_t>> *values, size_t start, size_t stop, WeightedMBuffer *buffer)
+void insert_records(std::vector<std::pair<uint64_t, uint32_t>> *values, size_t start, size_t stop, MutableBuffer<Rec> *buffer)
 {
     for (size_t i=start; i<stop; i++) {
-        buffer->append((*values)[i].first, (*values)[i].second, 1.0);
+        buffer->append({(*values)[i].first, (*values)[i].second});
     }
 
 }
 
+#if DE_MT_TEST
 START_TEST(t_multithreaded_insert)
 {
     size_t cnt = 10000;
     auto rng = gsl_rng_alloc(gsl_rng_mt19937);
-    auto buffer = new WeightedMBuffer(cnt, true, cnt/2, rng);
+    auto buffer = new MutableBuffer<Rec>(cnt, true, cnt/2, rng);
 
-    std::vector<std::pair<uint64_t, uint32_t>> records(cnt);
+    std::vector<Rec> records(cnt);
     for (size_t i=0; i<cnt; i++) {
-        records[i] = {rand(), rand()};
+        records[i] = Rec {(uint64_t) rand(), (uint32_t) rand()};
     }
 
     // perform a t_multithreaded insertion
@@ -245,15 +261,16 @@ START_TEST(t_multithreaded_insert)
     ck_assert_int_eq(buffer->get_record_count(), cnt);
 
     std::sort(records.begin(), records.end());
-    WeightedRec *sorted_records = buffer->sorted_output();
+    auto *sorted_records = buffer->sorted_output();
     for (size_t i=0; i<cnt; i++) {
-        ck_assert_int_eq(sorted_records[i].key, records[i].first);
+        ck_assert_int_eq(sorted_records[i].key, records[i].key);
     }
 
     delete buffer;
     gsl_rng_free(rng);
 }
 END_TEST
+#endif
 
 
 Suite *unit_testing()
@@ -268,7 +285,9 @@ Suite *unit_testing()
     TCase *append = tcase_create("de::MutableBuffer::append Testing");
     tcase_add_test(append, t_insert);
     tcase_add_test(append, t_insert_tombstones);
-    tcase_add_test(append, t_multithreaded_insert);
+    #if DE_MT_TEST
+        tcase_add_test(append, t_multithreaded_insert);
+    #endif
 
     suite_add_tcase(unit, append);
 
diff --git a/tests/testing.h b/tests/testing.h
index 062e930..eb5c095 100644
--- a/tests/testing.h
+++ b/tests/testing.h
@@ -16,18 +16,14 @@
 #include <unistd.h>
 #include <fcntl.h>
 
+#include "util/Record.h"
 #include "util/types.h"
 #include "util/base.h"
 #include "framework/MutableBuffer.h"
-#include "framework/InternalLevel.h"
+//#include "framework/InternalLevel.h"
 
-typedef de::Record<uint64_t, uint32_t, uint64_t> WeightedRec;
-typedef de::MutableBuffer<uint64_t, uint32_t, uint64_t> WeightedMBuffer;
-typedef de::InternalLevel<uint64_t, uint32_t, uint64_t> WeightedLevel;
-
-typedef de::Record<uint64_t, uint32_t> UnweightedRec;
-typedef de::MutableBuffer<uint64_t, uint32_t> UnweightedMBuffer;
-typedef de::InternalLevel<uint64_t, uint32_t> UnweightedLevel;
+typedef de::WeightedRecord<uint64_t, uint32_t, uint64_t> WRec;
+typedef de::Record<uint64_t, uint32_t> Rec;
 
 static gsl_rng *g_rng = gsl_rng_alloc(gsl_rng_mt19937);
 
@@ -72,90 +68,101 @@ static bool roughly_equal(int n1, int n2, size_t mag, double epsilon) {
     return ((double) std::abs(n1 - n2) / (double) mag) < epsilon;
 }
 
-template <typename K, typename V, typename W=void>
-static de::MutableBuffer<K,V,W> *create_test_mbuffer(size_t cnt)
+template <de::RecordInterface R>
+static de::MutableBuffer<R> *create_test_mbuffer(size_t cnt)
 {
-    auto buffer = new de::MutableBuffer<K,V,W>(cnt, true, cnt, g_rng);
+    auto buffer = new de::MutableBuffer<R>(cnt, true, cnt, g_rng);
 
+    R rec;
     for (size_t i = 0; i < cnt; i++) {
-        uint64_t key = rand();
-        uint32_t val = rand();
+        rec.key = rand();
+        rec.value = rand();
+
+        if constexpr (de::WeightedRecordInterface<R>) {
+            rec.weight = 1;
+        }
 
-        buffer->append(key, val);
+        buffer->append(rec);
     }
 
     return buffer;
 }
 
-template <typename K, typename V, typename W=void>
-static de::MutableBuffer<K,V,W> *create_test_mbuffer_tombstones(size_t cnt, size_t ts_cnt) 
+template <de::RecordInterface R>
+static de::MutableBuffer<R> *create_test_mbuffer_tombstones(size_t cnt, size_t ts_cnt) 
 {
-    auto buffer = new de::MutableBuffer<K,V,W>(cnt, true, ts_cnt, g_rng);
+    auto buffer = new de::MutableBuffer<R>(cnt, true, ts_cnt, g_rng);
 
     std::vector<std::pair<uint64_t, uint32_t>> tombstones;
 
+    R rec;
     for (size_t i = 0; i < cnt; i++) {
-        uint64_t key = rand();
-        uint32_t val = rand();
+        rec.key = rand();
+        rec.value = rand();
+
+        if constexpr (de::WeightedRecordInterface<R>) {
+            rec.weight = 1;
+        }
 
         if (i < ts_cnt) {
-            tombstones.push_back({key, val});
+            tombstones.push_back({rec.key, rec.value});
         }
 
-        buffer->append(key, val);
+        buffer->append(rec);
     }
 
+    rec.set_tombstone();
     for (size_t i=0; i<ts_cnt; i++) {
-        buffer->append(tombstones[i].first, tombstones[i].second, true);
+        buffer->append(rec);
     }
 
     return buffer;
 }
 
-template <typename K, typename V, typename W=void>
-static de::MutableBuffer<K,V,W> *create_weighted_mbuffer(size_t cnt)
+template <de::WeightedRecordInterface R>
+static de::MutableBuffer<R> *create_weighted_mbuffer(size_t cnt)
 {
-    static_assert(!std::is_same<W, void>::value);
-    auto buffer = new de::MutableBuffer<K,V,W>(cnt, true, cnt, g_rng);
+    auto buffer = new de::MutableBuffer<R>(cnt, true, cnt, g_rng);
     
     // Put in half of the count with weight one.
-    uint64_t key = 1;
-    for (size_t i=0; i< cnt / 2; i++) {
-        buffer->append(key, i, 2);
+    for (uint32_t i=0; i< cnt / 2; i++) {
+        buffer->append(R {1, i, 2});
     }
 
-    // put in a quarter of the count with weight two.
-    key = 2;
-    for (size_t i=0; i< cnt / 4; i++) {
-        buffer->append(key, i, 4);
+    // put in a quarter of the count with weight four.
+    for (uint32_t i=0; i< cnt / 4; i++) {
+        buffer->append(R {2, i, 4});
     }
 
-    // the remaining quarter with weight four.
-    key = 3;
-    for (size_t i=0; i< cnt / 4; i++) {
-        buffer->append(key, i, 8);
+    // the remaining quarter with weight eight.
+    for (uint32_t i=0; i< cnt / 4; i++) {
+        buffer->append(R {3, i, 8});
     }
 
     return buffer;
 }
 
-template <typename K, typename V, typename W=void>
-static de::MutableBuffer<K,V,W> *create_double_seq_mbuffer(size_t cnt, bool ts=false) 
+template <de::RecordInterface R>
+static de::MutableBuffer<R> *create_double_seq_mbuffer(size_t cnt, bool ts=false) 
 {
-    auto buffer = new de::MutableBuffer<K,V,W>(cnt, true, cnt, g_rng);
+    auto buffer = new de::MutableBuffer<R>(cnt, true, cnt, g_rng);
 
     for (size_t i = 0; i < cnt / 2; i++) {
-        uint64_t key = i;
-        uint32_t val = i;
+        R rec;
+        rec.key = i;
+        rec.value = i;
+        if (ts) rec.set_tombstone();
 
-        buffer->append(key, val, ts);
+        buffer->append(rec);
     }
 
     for (size_t i = 0; i < cnt / 2; i++) {
-        uint64_t key = i;
-        uint32_t val = i + 1;
+        R rec;
+        rec.key = i;
+        rec.value = i + 1;
+        if (ts) rec.set_tombstone();
 
-        buffer->append(key, val, ts);
+        buffer->append(rec);
     }
 
     return buffer;
diff --git a/tests/wirs_tests.cpp b/tests/wirs_tests.cpp
index ed83d40..673bdca 100644
--- a/tests/wirs_tests.cpp
+++ b/tests/wirs_tests.cpp
@@ -19,24 +19,24 @@
 
 using namespace de;
 
-typedef WIRS<uint64_t, uint32_t, uint64_t> Shard;
+typedef WIRS<WRec> Shard;
 
 START_TEST(t_mbuffer_init)
 {
-    auto mem_table = new WeightedMBuffer(1024, true, 1024, g_rng);
+    auto mem_table = new MutableBuffer<WRec>(1024, true, 1024, g_rng);
     for (uint64_t i = 512; i > 0; i--) {
         uint32_t v = i;
-        mem_table->append(i, v);
+        mem_table->append({i,v, 1});
     }
     
     for (uint64_t i = 1; i <= 256; ++i) {
         uint32_t v = i;
-        mem_table->append(i, v, 1.0, true);
+        mem_table->append({i, v, 1, 1});
     }
 
     for (uint64_t i = 257; i <= 512; ++i) {
         uint32_t v = i + 1;
-        mem_table->append(i, v);
+        mem_table->append({i, v, 1});
     }
 
     BloomFilter* bf = new BloomFilter(BF_FPR, mem_table->get_tombstone_count(), BF_HASH_FUNCS, g_rng);
@@ -51,9 +51,9 @@ START_TEST(t_mbuffer_init)
 START_TEST(t_wirs_init)
 {
     size_t n = 512;
-    auto mbuffer1 = create_test_mbuffer<uint64_t, uint32_t, uint64_t>(n);
-    auto mbuffer2 = create_test_mbuffer<uint64_t, uint32_t, uint64_t>(n);
-    auto mbuffer3 = create_test_mbuffer<uint64_t, uint32_t, uint64_t>(n);
+    auto mbuffer1 = create_test_mbuffer<WRec>(n);
+    auto mbuffer2 = create_test_mbuffer<WRec>(n);
+    auto mbuffer3 = create_test_mbuffer<WRec>(n);
 
     BloomFilter* bf1 = new BloomFilter(100, BF_HASH_FUNCS, g_rng);
     BloomFilter* bf2 = new BloomFilter(100, BF_HASH_FUNCS, g_rng);
@@ -81,11 +81,11 @@ START_TEST(t_wirs_init)
 
         auto cur_rec = shard4->get_record_at(i);
 
-        if (shard1_idx < n && cur_rec->match(rec1)) {
+        if (shard1_idx < n && *cur_rec == *rec1) {
             ++shard1_idx;
-        } else if (shard2_idx < n && cur_rec->match(rec2)) {
+        } else if (shard2_idx < n && *cur_rec == *rec2) {
             ++shard2_idx;
-        } else if (shard3_idx < n && cur_rec->match(rec3)) {
+        } else if (shard3_idx < n && *cur_rec == *rec3) {
             ++shard3_idx;
         } else {
            assert(false);
@@ -109,7 +109,7 @@ START_TEST(t_wirs_init)
 START_TEST(t_get_lower_bound_index)
 {
     size_t n = 10000;
-    auto mbuffer = create_double_seq_mbuffer<uint64_t, uint32_t, uint64_t>(n);
+    auto mbuffer = create_double_seq_mbuffer<WRec>(n);
 
     ck_assert_ptr_nonnull(mbuffer);
     BloomFilter* bf = new BloomFilter(100, BF_HASH_FUNCS, g_rng);
@@ -120,7 +120,7 @@ START_TEST(t_get_lower_bound_index)
 
     auto tbl_records = mbuffer->sorted_output();
     for (size_t i=0; i<n; i++) {
-        const WeightedRec *tbl_rec = mbuffer->get_record_at(i);
+        const WRec *tbl_rec = mbuffer->get_record_at(i);
         auto pos = shard->get_lower_bound(tbl_rec->key);
         ck_assert_int_eq(shard->get_record_at(pos)->key, tbl_rec->key);
         ck_assert_int_le(pos, i);
@@ -135,8 +135,8 @@ START_TEST(t_get_lower_bound_index)
 START_TEST(t_full_cancelation)
 {
     size_t n = 100;
-    auto buffer = create_double_seq_mbuffer<uint64_t, uint32_t, uint64_t>(n, false);
-    auto buffer_ts = create_double_seq_mbuffer<uint64_t, uint32_t, uint64_t>(n, true);
+    auto buffer = create_double_seq_mbuffer<WRec>(n, false);
+    auto buffer_ts = create_double_seq_mbuffer<WRec>(n, true);
     BloomFilter* bf1 = new BloomFilter(100, BF_HASH_FUNCS, g_rng);
     BloomFilter* bf2 = new BloomFilter(100, BF_HASH_FUNCS, g_rng);
     BloomFilter* bf3 = new BloomFilter(100, BF_HASH_FUNCS, g_rng);
@@ -171,7 +171,7 @@ END_TEST
 START_TEST(t_weighted_sampling)
 {
     size_t n=1000;
-    auto buffer = create_weighted_mbuffer<uint64_t, uint32_t, uint64_t>(n);
+    auto buffer = create_weighted_mbuffer<WRec>(n);
 
     BloomFilter* bf = new BloomFilter(100, BF_HASH_FUNCS, g_rng);
     Shard* shard = new Shard(buffer, bf, false);
@@ -181,7 +181,7 @@ START_TEST(t_weighted_sampling)
 
     size_t k = 1000;
 
-    std::vector<WeightedRec> results;
+    std::vector<WRec> results;
     results.reserve(k);
     size_t cnt[3] = {0};
     for (size_t i=0; i<1000; i++) {
@@ -193,7 +193,7 @@ START_TEST(t_weighted_sampling)
             cnt[results[j].key - 1]++;
         }
 
-        WIRS<uint64_t, uint32_t, uint64_t>::delete_state(state);
+        WIRS<WRec>::delete_state(state);
     }
 
     ck_assert(roughly_equal(cnt[0] / 1000, (double) k/4.0, k, .05));
@@ -211,14 +211,14 @@ START_TEST(t_tombstone_check)
 {
     size_t cnt = 1024;
     size_t ts_cnt = 256;
-    auto buffer = new WeightedMBuffer(cnt + ts_cnt, true, ts_cnt, g_rng);
+    auto buffer = new MutableBuffer<WRec>(cnt + ts_cnt, true, ts_cnt, g_rng);
 
     std::vector<std::pair<uint64_t, uint32_t>> tombstones;
 
     uint64_t key = 1000;
     uint32_t val = 101;
     for (size_t i = 0; i < cnt; i++) {
-        buffer->append(key, val);
+        buffer->append({key, val, 1});
         key++;
         val++;
     }
@@ -230,14 +230,14 @@ START_TEST(t_tombstone_check)
     }
 
     for (size_t i=0; i<ts_cnt; i++) {
-        buffer->append(tombstones[i].first, tombstones[i].second, 1.0, true);
+        buffer->append({tombstones[i].first, tombstones[i].second, 1, 1});
     }
 
     BloomFilter* bf1 = new BloomFilter(100, BF_HASH_FUNCS, g_rng);
     auto shard = new Shard(buffer, bf1, false);
 
     for (size_t i=0; i<tombstones.size(); i++) {
-        ck_assert(shard->check_tombstone(tombstones[i].first, tombstones[i].second));
+        ck_assert(shard->check_tombstone({tombstones[i].first, tombstones[i].second}));
         ck_assert_int_eq(shard->get_rejection_count(), i+1);
     }