From c4514c2e62a711189cf3c914297885d97fb51a09 Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Fri, 12 Jan 2024 14:08:33 -0500
Subject: Initial pass at unit test refactoring

Restructured unit tests to be a bit more modular. I have some further
plans to expand on this, particular for the query tests (including both
shard and framework level test functions that can be injected at will).
---
 tests/include/dynamic_extension.h | 341 ++++++++++++++++++++++++++++++++++++++
 tests/include/rangequery.h        | 179 ++++++++++++++++++++
 tests/include/shard_standard.h    | 198 ++++++++++++++++++++++
 tests/include/testing.h           | 218 ++++++++++++++++++++++++
 4 files changed, 936 insertions(+)
 create mode 100644 tests/include/dynamic_extension.h
 create mode 100644 tests/include/rangequery.h
 create mode 100644 tests/include/shard_standard.h
 create mode 100644 tests/include/testing.h

(limited to 'tests/include')

diff --git a/tests/include/dynamic_extension.h b/tests/include/dynamic_extension.h
new file mode 100644
index 0000000..5a08f5a
--- /dev/null
+++ b/tests/include/dynamic_extension.h
@@ -0,0 +1,341 @@
+/*
+ * tests/include/dynamic_extension.h
+ *
+ * Standardized unit tests for DynamicExtension objects
+ *
+ * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> 
+ *
+ * Distributed under the Modified BSD License.
+ *
+ * WARNING: This file must be included in the main unit test set
+ *          after the definition of an appropriate Shard, Query, and Rec
+ *          type. In particular, Rec needs to implement the key-value
+ *          pair interface. For other types of record, you'll need to
+ *          use a different set of unit tests.
+ */
+#pragma once
+
+/*
+ * Uncomment these lines temporarily to remove errors in this file
+ * temporarily for development purposes. They should be removed prior
+ * to building, to ensure no duplicate definitions. These includes/defines
+ * should be included in the source file that includes this one, above the
+ * include statement.
+ */
+//#include "testing.h"
+//#include "framework/DynamicExtension.h"
+//#include "framework/scheduling/SerialScheduler.h"
+//#include "shard/ISAMTree.h"
+//#include "query/rangequery.h"
+//#include <check.h>
+//using namespace de;
+//typedef DynamicExtension<Rec, ISAMTree<Rec>, rq::Query<ISAMTree<Rec>, Rec>, LayoutPolicy::TEIRING, DeletePolicy::TAGGING, SerialScheduler> DE;
+
+
+START_TEST(t_create)
+{
+    auto test_de = new DE(100, 1000, 2);
+
+    ck_assert_ptr_nonnull(test_de);
+    ck_assert_int_eq(test_de->get_record_count(), 0);
+    ck_assert_int_eq(test_de->get_height(), 0);
+
+    delete test_de;
+}
+END_TEST
+
+
+START_TEST(t_insert)
+{
+    auto test_de = new DE(100, 1000, 2);
+
+    uint64_t key = 0;
+    uint32_t val = 0;
+    for (size_t i=0; i<100; i++) {
+        Rec r = {key, val};
+        ck_assert_int_eq(test_de->insert(r), 1);
+        key++;
+        val++;
+    }
+
+    ck_assert_int_eq(test_de->get_height(), 0);
+    ck_assert_int_eq(test_de->get_record_count(), 100);
+
+    delete test_de;
+}
+END_TEST
+
+
+START_TEST(t_debug_insert)
+{
+    auto test_de = new DE(100, 1000, 2);
+
+    uint64_t key = 0;
+    uint32_t val = 0;
+    for (size_t i=0; i<1000; i++) {
+        Rec r = {key, val};
+        ck_assert_int_eq(test_de->insert(r), 1);
+        ck_assert_int_eq(test_de->get_record_count(), i+1);
+        key++;
+        val++;
+    }
+
+    delete test_de;
+}
+END_TEST
+
+
+START_TEST(t_insert_with_mem_merges)
+{
+    auto test_de = new DE(100, 1000, 2);
+
+    uint64_t key = 0;
+    uint32_t val = 0;
+    for (size_t i=0; i<300; i++) {
+        Rec r = {key, val};
+        ck_assert_int_eq(test_de->insert(r), 1);
+        key++;
+        val++;
+    }
+
+    test_de->await_next_epoch();
+
+    ck_assert_int_eq(test_de->get_record_count(), 300);
+    ck_assert_int_eq(test_de->get_height(), 1);
+
+    delete test_de;
+}
+END_TEST
+
+
+START_TEST(t_range_query)
+{
+    auto test_de = new DE(100, 1000, 2);
+    size_t n = 10000;
+
+    std::vector<uint64_t> keys;
+    for (size_t i=0; i<n; i++) {
+        keys.push_back(rand() % 25000);
+    }
+
+    std::random_device rd;
+    std::mt19937 gen{rd()};
+    std::shuffle(keys.begin(), keys.end(), gen);
+
+    for (size_t i=0; i<keys.size(); i++) {
+        Rec r = {keys[i], (uint32_t) i};
+        ck_assert_int_eq(test_de->insert(r), 1);
+    }
+
+    test_de->await_next_epoch();
+
+    std::sort(keys.begin(), keys.end());
+
+    auto idx = rand() % (keys.size() - 250);
+
+    uint64_t lower_key = keys[idx];
+    uint64_t upper_key = keys[idx + 250];
+
+    rq::Parms<Rec> p;
+    p.lower_bound = lower_key;
+    p.upper_bound = upper_key;
+
+    auto result = test_de->query(&p);
+    auto r = result.get();
+    std::sort(r.begin(), r.end());
+    ck_assert_int_eq(r.size(), 251);
+
+    for (size_t i=0; i<r.size(); i++) {
+        ck_assert_int_eq(r[i].key, keys[idx + i]);
+    }
+
+    delete test_de;
+}
+END_TEST
+
+
+START_TEST(t_tombstone_merging_01)
+{
+    size_t reccnt = 100000;
+    auto test_de = new DE(100, 1000, 2);
+
+    auto rng = gsl_rng_alloc(gsl_rng_mt19937);
+
+    std::set<std::pair<uint64_t, uint32_t>> records; 
+    std::set<std::pair<uint64_t, uint32_t>> to_delete;
+    std::set<std::pair<uint64_t, uint32_t>> deleted;
+
+    while (records.size() < reccnt) {
+        uint64_t key = rand();
+        uint32_t val = rand();
+
+        if (records.find({key, val}) != records.end()) continue;
+
+        records.insert({key, val});
+    }
+
+    size_t deletes = 0;
+    size_t cnt=0;
+    for (auto rec : records) {
+        Rec r = {rec.first, rec.second};
+        ck_assert_int_eq(test_de->insert(r), 1);
+
+         if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) {
+            std::vector<std::pair<uint64_t, uint32_t>> del_vec;
+            std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()});
+
+            for (size_t i=0; i<del_vec.size(); i++) {
+                Rec dr = {del_vec[i].first, del_vec[i].second};
+                test_de->erase(dr);
+                deletes++;
+                to_delete.erase(del_vec[i]);
+                deleted.insert(del_vec[i]);
+            }
+        }
+
+        if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) {
+            to_delete.insert(rec);
+        }
+    }
+
+    test_de->await_next_epoch();
+
+    ck_assert(test_de->validate_tombstone_proportion());
+
+    gsl_rng_free(rng);
+    delete test_de;
+}
+END_TEST
+
+DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) {
+    auto rng = gsl_rng_alloc(gsl_rng_mt19937);
+
+    auto test_de = new DE(1000, 10000, 2);
+
+    std::set<Rec> records; 
+    std::set<Rec> to_delete;
+    std::set<Rec> deleted;
+
+    while (records.size() < reccnt) {
+        uint64_t key = rand();
+        uint32_t val = rand();
+
+        if (records.find({key, val}) != records.end()) continue;
+
+        records.insert({key, val});
+    }
+
+    size_t deletes = 0;
+    for (auto rec : records) {
+        ck_assert_int_eq(test_de->insert(rec), 1);
+
+         if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) {
+            std::vector<Rec> del_vec;
+            std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()});
+
+            for (size_t i=0; i<del_vec.size(); i++) {
+                test_de->erase(del_vec[i]);
+                deletes++;
+                to_delete.erase(del_vec[i]);
+                deleted.insert(del_vec[i]);
+            }
+        }
+
+        if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) {
+            to_delete.insert(rec);
+        }
+    }
+
+    gsl_rng_free(rng);
+
+    return test_de;
+}
+
+START_TEST(t_static_structure)
+{
+    auto rng = gsl_rng_alloc(gsl_rng_mt19937);
+
+    size_t reccnt = 100000;
+    auto test_de = new DE(100, 1000, 2);
+
+    std::set<Rec> records; 
+    std::set<Rec> to_delete;
+    std::set<Rec> deleted;
+
+    while (records.size() < reccnt) {
+        uint64_t key = rand();
+        uint32_t val = rand();
+
+        if (records.find({key, val}) != records.end()) continue;
+
+        records.insert({key, val});
+    }
+
+    size_t deletes = 0;
+    size_t t_reccnt = 0;
+    size_t k=0;
+    for (auto rec : records) {
+        k++;
+        ck_assert_int_eq(test_de->insert(rec), 1);
+        t_reccnt++;
+
+         if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) {
+            std::vector<Rec> del_vec;
+            std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()});
+
+            for (size_t i=0; i<del_vec.size(); i++) {
+                ck_assert_int_eq(test_de->erase(del_vec[i]), 1);
+
+                deletes++;
+                to_delete.erase(del_vec[i]);
+                deleted.insert(del_vec[i]);
+            }
+        }
+
+        if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) {
+            to_delete.insert(rec);
+        }
+    }
+
+    auto flat = test_de->create_static_structure();
+    ck_assert_int_eq(flat->get_record_count(), reccnt - deletes);
+
+    uint64_t prev_key = 0;
+    for (size_t i=0; i<flat->get_record_count(); i++) {
+        auto k = flat->get_record_at(i)->rec.key;
+        ck_assert_int_ge(k, prev_key);
+        prev_key = k;
+    }
+
+    gsl_rng_free(rng);
+    delete flat;
+    delete test_de;
+}
+END_TEST
+
+
+static void inject_dynamic_extension_tests(Suite *suite) {
+    TCase *create = tcase_create("de::DynamicExtension::constructor Testing");
+    tcase_add_test(create, t_create);
+    suite_add_tcase(suite, create);
+
+    TCase *insert = tcase_create("de::DynamicExtension::insert Testing");
+    tcase_add_test(insert, t_insert);
+    tcase_add_test(insert, t_insert_with_mem_merges);
+    tcase_add_test(insert, t_debug_insert);
+    suite_add_tcase(suite, insert);
+
+    TCase *query = tcase_create("de::DynamicExtension::range_query Testing");
+    tcase_add_test(query, t_range_query);
+    suite_add_tcase(suite, query);
+
+    TCase *ts = tcase_create("de::DynamicExtension::tombstone_compaction Testing");
+    tcase_add_test(ts, t_tombstone_merging_01);
+    tcase_set_timeout(ts, 500);
+    suite_add_tcase(suite, ts);
+
+    TCase *flat = tcase_create("de::DynamicExtension::create_static_structure Testing");
+    tcase_add_test(flat, t_static_structure);
+    tcase_set_timeout(flat, 500);
+    suite_add_tcase(suite, flat);
+}
diff --git a/tests/include/rangequery.h b/tests/include/rangequery.h
new file mode 100644
index 0000000..3c7e7e0
--- /dev/null
+++ b/tests/include/rangequery.h
@@ -0,0 +1,179 @@
+/*
+ * tests/include/rangequery.h
+ *
+ * Standardized unit tests for range queries against supporting
+ * shard types
+ *
+ * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> 
+ *
+ * Distributed under the Modified BSD License.
+ *
+ * WARNING: This file must be included in the main unit test set
+ *          after the definition of an appropriate Shard and Rec
+ *          type. In particular, Rec needs to implement the key-value
+ *          pair interface and Shard needs to support lower_bound. 
+ *          For other types of record and shard, you'll need to
+ *          use a different set of unit tests.
+ */
+#pragma once
+
+/*
+ * Uncomment these lines temporarily to remove errors in this file
+ * temporarily for development purposes. They should be removed prior
+ * to building, to ensure no duplicate definitions. These includes/defines
+ * should be included in the source file that includes this one, above the
+ * include statement.
+ */
+//#include "shard/ISAMTree.h"
+//#include "query/rangequery.h"
+//#include "testing.h"
+//#include <check.h>
+//using namespace de;
+//typedef ISAMTree<Rec> Shard;
+
+
+START_TEST(t_range_query)
+{
+    auto buffer = create_sequential_mbuffer<Rec>(100, 1000);
+    auto shard = Shard(buffer->get_buffer_view());
+
+    rq::Parms<Rec> parms;
+    parms.lower_bound = 300;
+    parms.upper_bound = 500;
+
+    auto state = rq::Query<Shard, Rec>::get_query_state(&shard, &parms);
+    auto result = rq::Query<Shard, Rec>::query(&shard, state, &parms);
+    rq::Query<Shard, Rec>::delete_query_state(state);
+
+    ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1);
+    for (size_t i=0; i<result.size(); i++) {
+        ck_assert_int_le(result[i].rec.key, parms.upper_bound);
+        ck_assert_int_ge(result[i].rec.key, parms.lower_bound);
+    }
+
+    delete buffer;
+}
+END_TEST
+
+
+START_TEST(t_buffer_range_query)
+{
+    auto buffer = create_sequential_mbuffer<Rec>(100, 1000);
+
+    rq::Parms<Rec> parms;
+    parms.lower_bound = 300;
+    parms.upper_bound = 500;
+
+    auto state = rq::Query<Shard, Rec>::get_buffer_query_state(buffer->get_buffer_view(), &parms);
+    auto result = rq::Query<Shard, Rec>::buffer_query(state, &parms);
+    rq::Query<Shard, Rec>::delete_buffer_query_state(state);
+
+    ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1);
+    for (size_t i=0; i<result.size(); i++) {
+        ck_assert_int_le(result[i].rec.key, parms.upper_bound);
+        ck_assert_int_ge(result[i].rec.key, parms.lower_bound);
+    }
+
+    delete buffer;
+}
+END_TEST
+
+
+START_TEST(t_range_query_merge)
+{    
+    auto buffer1 = create_sequential_mbuffer<Rec>(100, 200);
+    auto buffer2 = create_sequential_mbuffer<Rec>(400, 1000);
+
+    auto shard1 = Shard(buffer1->get_buffer_view());
+    auto shard2 = Shard(buffer2->get_buffer_view());
+
+    rq::Parms<Rec> parms;
+    parms.lower_bound = 150;
+    parms.upper_bound = 500;
+
+    size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200;
+
+    auto state1 = rq::Query<Shard, Rec>::get_query_state(&shard1, &parms);
+    auto state2 = rq::Query<Shard, Rec>::get_query_state(&shard2, &parms);
+
+    std::vector<std::vector<de::Wrapped<Rec>>> results(2);
+    results[0] = rq::Query<Shard, Rec>::query(&shard1, state1, &parms);
+    results[1] = rq::Query<Shard, Rec>::query(&shard2, state2, &parms);
+
+    rq::Query<Shard, Rec>::delete_query_state(state1);
+    rq::Query<Shard, Rec>::delete_query_state(state2);
+
+    ck_assert_int_eq(results[0].size() + results[1].size(), result_size);
+
+    std::vector<std::vector<Wrapped<Rec>>> proc_results;
+
+    for (size_t j=0; j<results.size(); j++) {
+        proc_results.emplace_back(std::vector<Wrapped<Rec>>());
+        for (size_t i=0; i<results[j].size(); i++) {
+            proc_results[j].emplace_back(results[j][i]);
+        }
+    }
+
+    auto result = rq::Query<Shard, Rec>::merge(proc_results, nullptr);
+    std::sort(result.begin(), result.end());
+
+    ck_assert_int_eq(result.size(), result_size);
+    auto key = parms.lower_bound;
+    for (size_t i=0; i<result.size(); i++) {
+        ck_assert_int_eq(key++, result[i].key);
+        if (key == 200) {
+            key = 400;
+        }
+    }
+
+    delete buffer1;
+    delete buffer2;
+}
+END_TEST
+
+
+START_TEST(t_lower_bound)
+{
+    auto buffer1 = create_sequential_mbuffer<Rec>(100, 200);
+    auto buffer2 = create_sequential_mbuffer<Rec>(400, 1000);
+
+    Shard *shards[2];
+
+    auto shard1 = Shard(buffer1->get_buffer_view());
+    auto shard2 = Shard(buffer2->get_buffer_view());
+
+    shards[0] = &shard1;
+    shards[1] = &shard2;
+
+    auto merged = Shard(shards, 2);
+
+    for (size_t i=100; i<1000; i++) {
+        Rec r;
+        r.key = i;
+        r.value = i;
+
+        auto idx = merged.get_lower_bound(i);
+
+        assert(idx < merged.get_record_count());
+
+        auto res = merged.get_record_at(idx);
+
+        if (i >=200 && i <400) {
+            ck_assert_int_lt(res->rec.key, i);
+        } else {
+            ck_assert_int_eq(res->rec.key, i);
+        }
+    }
+
+    delete buffer1;
+    delete buffer2;
+}
+END_TEST
+
+static void inject_rangequery_tests(Suite *suite) {
+    TCase *range_query = tcase_create("Range Query Testing"); 
+    tcase_add_test(range_query, t_range_query); 
+    tcase_add_test(range_query, t_buffer_range_query); 
+    tcase_add_test(range_query, t_range_query_merge); 
+    suite_add_tcase(suite, range_query);
+}
diff --git a/tests/include/shard_standard.h b/tests/include/shard_standard.h
new file mode 100644
index 0000000..047a7b5
--- /dev/null
+++ b/tests/include/shard_standard.h
@@ -0,0 +1,198 @@
+/*
+ * tests/include/shard_standard.h
+ *
+ * Standardized unit tests for Shard objects
+ *
+ * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> 
+ *
+ * Distributed under the Modified BSD License.
+ *
+ * WARNING: This file must be included in the main unit test set
+ *          after the definition of an appropriate Shard and Rec
+ *          type. In particular, Rec needs to implement the key-value
+ *          pair interface. For other types of record, you'll need to
+ *          use a different set of unit tests.
+ */
+#pragma once
+
+/*
+ * Uncomment these lines temporarily to remove errors in this file
+ * temporarily for development purposes. They should be removed prior
+ * to building, to ensure no duplicate definitions. These includes/defines
+ * should be included in the source file that includes this one, above the
+ * include statement.
+ */
+//#include "shard/ISAMTree.h"
+//#include "testing.h"
+//#include <check.h>
+//using namespace de;
+//typedef ISAMTree<Rec> Shard;
+
+START_TEST(t_mbuffer_init)
+{
+    auto buffer = new MutableBuffer<Rec>(512, 1024);
+    for (uint64_t i = 512; i > 0; i--) {
+        uint32_t v = i;
+        buffer->append({i,v, 1});
+    }
+    
+    for (uint64_t i = 1; i <= 256; ++i) {
+        uint32_t v = i;
+        buffer->append({i, v, 1}, true);
+    }
+
+    for (uint64_t i = 257; i <= 512; ++i) {
+        uint32_t v = i + 1;
+        buffer->append({i, v, 1});
+    }
+
+    Shard* shard = new Shard(buffer->get_buffer_view());
+    ck_assert_uint_eq(shard->get_record_count(), 512);
+
+    delete buffer;
+    delete shard;
+}
+
+
+START_TEST(t_shard_init)
+{
+    size_t n = 512;
+    auto mbuffer1 = create_test_mbuffer<Rec>(n);
+    auto mbuffer2 = create_test_mbuffer<Rec>(n);
+    auto mbuffer3 = create_test_mbuffer<Rec>(n);
+
+    auto shard1 = new Shard(mbuffer1->get_buffer_view());
+    auto shard2 = new Shard(mbuffer2->get_buffer_view());
+    auto shard3 = new Shard(mbuffer3->get_buffer_view());
+
+    Shard* shards[3] = {shard1, shard2, shard3};
+    auto shard4 = new Shard(shards, 3);
+
+    ck_assert_int_eq(shard4->get_record_count(), n * 3);
+    ck_assert_int_eq(shard4->get_tombstone_count(), 0);
+
+    size_t total_cnt = 0;
+    size_t shard1_idx = 0;
+    size_t shard2_idx = 0;
+    size_t shard3_idx = 0;
+
+    for (size_t i = 0; i < shard4->get_record_count(); ++i) {
+        auto rec1 = shard1->get_record_at(shard1_idx);
+        auto rec2 = shard2->get_record_at(shard2_idx);
+        auto rec3 = shard3->get_record_at(shard3_idx);
+
+        auto cur_rec = shard4->get_record_at(i);
+
+        if (shard1_idx < n && cur_rec->rec == rec1->rec) {
+            ++shard1_idx;
+        } else if (shard2_idx < n && cur_rec->rec == rec2->rec) {
+            ++shard2_idx;
+        } else if (shard3_idx < n && cur_rec->rec == rec3->rec) {
+            ++shard3_idx;
+        } else {
+           assert(false);
+        }
+    }
+
+    delete mbuffer1;
+    delete mbuffer2;
+    delete mbuffer3;
+
+    delete shard1;
+    delete shard2;
+    delete shard3;
+    delete shard4;
+}
+
+
+START_TEST(t_full_cancelation)
+{
+    size_t n = 100;
+    auto buffer = create_double_seq_mbuffer<Rec>(n, false);
+    auto buffer_ts = create_double_seq_mbuffer<Rec>(n, true);
+
+    Shard* shard = new Shard(buffer->get_buffer_view());
+    Shard* shard_ts = new Shard(buffer_ts->get_buffer_view());
+
+    ck_assert_int_eq(shard->get_record_count(), n);
+    ck_assert_int_eq(shard->get_tombstone_count(), 0);
+    ck_assert_int_eq(shard_ts->get_record_count(), n);
+    ck_assert_int_eq(shard_ts->get_tombstone_count(), n);
+
+    Shard* shards[] = {shard, shard_ts};
+
+    Shard* merged = new Shard(shards, 2);
+
+    ck_assert_int_eq(merged->get_tombstone_count(), 0);
+    ck_assert_int_eq(merged->get_record_count(), 0);
+
+    delete buffer;
+    delete buffer_ts;
+    delete shard;
+    delete shard_ts;
+    delete merged;
+}
+END_TEST
+
+
+START_TEST(t_point_lookup) 
+{
+    size_t n = 10000;
+
+    auto buffer = create_double_seq_mbuffer<Rec>(n, false);
+    auto isam = Shard(buffer->get_buffer_view());
+
+    {
+        auto view = buffer->get_buffer_view();
+
+        for (size_t i=0; i<n; i++) {
+            Rec r;
+            auto rec = view.get(i);
+            r.key = rec->rec.key;
+            r.value = rec->rec.value;
+
+            auto result = isam.point_lookup(r);
+            ck_assert_ptr_nonnull(result);
+            ck_assert_int_eq(result->rec.key, r.key);
+            ck_assert_int_eq(result->rec.value, r.value);
+        }
+    }
+
+    delete buffer;
+}
+END_TEST
+
+
+START_TEST(t_point_lookup_miss) 
+{
+    size_t n = 10000;
+
+    auto buffer = create_double_seq_mbuffer<Rec>(n, false);
+    auto isam = Shard(buffer->get_buffer_view());
+
+    for (size_t i=n + 100; i<2*n; i++) {
+        Rec r;
+        r.key = i;
+        r.value = i;
+
+        auto result = isam.point_lookup(r);
+        ck_assert_ptr_null(result);
+    }
+
+    delete buffer;
+}
+
+static void inject_shard_tests(Suite *suite) {
+    TCase *create = tcase_create("Shard constructor Testing");
+    tcase_add_test(create, t_mbuffer_init);
+    tcase_add_test(create, t_shard_init);
+    tcase_set_timeout(create, 100);
+    suite_add_tcase(suite, create);
+    TCase *tombstone = tcase_create("Shard tombstone cancellation Testing");
+    tcase_add_test(tombstone, t_full_cancelation);
+    suite_add_tcase(suite, tombstone); 
+    TCase *pointlookup = tcase_create("Shard point lookup Testing"); 
+    tcase_add_test(pointlookup, t_point_lookup);
+    tcase_add_test(pointlookup, t_point_lookup_miss); 
+    suite_add_tcase(suite, pointlookup);
+}
diff --git a/tests/include/testing.h b/tests/include/testing.h
new file mode 100644
index 0000000..4e660dd
--- /dev/null
+++ b/tests/include/testing.h
@@ -0,0 +1,218 @@
+/*
+ * tests/testing.h
+ *
+ * Unit test utility functions/definitions
+ *
+ * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> 
+ *                    Dong Xie <dongx@psu.edu>
+ *
+ * Distributed under the Modified BSD License.
+ *
+ */
+#pragma once
+
+#include <string>
+
+#include <unistd.h>
+#include <fcntl.h>
+
+#include "util/types.h"
+#include "psu-util/alignment.h"
+#include "framework/structure/MutableBuffer.h"
+#include "framework/interface/Record.h"
+
+typedef de::WeightedRecord<uint64_t, uint32_t, uint64_t> WRec;
+typedef de::Record<uint64_t, uint32_t> Rec;
+typedef de::EuclidPoint<int64_t> PRec;
+
+template <de::RecordInterface R> 
+std::vector<R> strip_wrapping(std::vector<de::Wrapped<R>> vec) {
+    std::vector<R> out(vec.size());
+    for (size_t i=0; i<vec.size(); i++) {
+        out[i] = vec[i].rec;
+    }
+
+    return out;
+}
+
+static bool initialize_test_file(std::string fname, size_t page_cnt)
+{
+    auto flags = O_RDWR | O_CREAT | O_TRUNC;
+    mode_t mode = 0640;
+    char *page = nullptr;
+
+    int fd = open(fname.c_str(), flags, mode);
+    if (fd == -1) {
+        goto error;
+    }
+
+    page = (char *) aligned_alloc(psudb::SECTOR_SIZE, psudb::PAGE_SIZE);
+    if (!page) {
+        goto error_opened;
+    }
+
+    for (size_t i=0; i<=page_cnt; i++) {
+        *((int *) page) = i;
+        if (write(fd, page, psudb::PAGE_SIZE) == -1) {
+            goto error_alloced;
+        }
+    }
+
+    free(page);
+
+    return 1;
+
+error_alloced:
+    free(page);
+
+error_opened:
+    close(fd);
+
+error:
+    return 0;
+}
+
+static bool roughly_equal(int n1, int n2, size_t mag, double epsilon) {
+    return ((double) std::abs(n1 - n2) / (double) mag) < epsilon;
+}
+
+static de::MutableBuffer<PRec> *create_2d_mbuffer(size_t cnt) {
+    auto buffer = new de::MutableBuffer<PRec>(cnt/2, cnt);
+
+    for (int64_t i=0; i<cnt; i++) {
+        buffer->append({rand(), rand()});
+    }
+
+    return buffer;
+}
+
+static de::MutableBuffer<PRec> *create_2d_sequential_mbuffer(size_t cnt) {
+    auto buffer = new de::MutableBuffer<PRec>(cnt/2, cnt);
+    for (int64_t i=0; i<cnt; i++) {
+        buffer->append({i, i});
+    }
+
+    return buffer;
+}
+
+template <de::KVPInterface R>
+static de::MutableBuffer<R> *create_test_mbuffer(size_t cnt)
+{
+    auto buffer = new de::MutableBuffer<R>(cnt/2, cnt);
+
+    R rec;
+    for (size_t i = 0; i < cnt; i++) {
+        rec.key = rand();
+        rec.value = rand();
+
+        if constexpr (de::WeightedRecordInterface<R>) {
+            rec.weight = 1;
+        }
+
+        buffer->append(rec);
+    }
+
+    return buffer;
+}
+
+template <de::KVPInterface R>
+static de::MutableBuffer<R> *create_sequential_mbuffer(decltype(R::key) start, decltype(R::key) stop)
+{
+    size_t cnt = stop - start;
+    auto buffer = new de::MutableBuffer<R>(cnt/2, cnt);
+
+    for (size_t i=start; i<stop; i++) {
+        R rec;
+        rec.key = i;
+        rec.value = i;
+
+        if constexpr (de::WeightedRecordInterface<R>) {
+            rec.weight = 1;
+        }
+
+        buffer->append(rec);
+    }
+
+    return buffer;
+}
+
+template <de::KVPInterface R>
+static de::MutableBuffer<R> *create_test_mbuffer_tombstones(size_t cnt, size_t ts_cnt) 
+{
+    auto buffer = new de::MutableBuffer<R>(cnt/2, cnt);
+
+    std::vector<std::pair<uint64_t, uint32_t>> tombstones;
+
+    R rec;
+    for (size_t i = 0; i < cnt; i++) {
+        rec.key = rand();
+        rec.value = rand();
+
+        if constexpr (de::WeightedRecordInterface<R>) {
+            rec.weight = 1;
+        }
+
+        if (i < ts_cnt) {
+            tombstones.push_back({rec.key, rec.value});
+        }
+
+        buffer->append(rec);
+    }
+
+    rec.set_tombstone();
+    for (size_t i=0; i<ts_cnt; i++) {
+        buffer->append(rec);
+    }
+
+    return buffer;
+}
+
+template <typename R>
+requires de::WeightedRecordInterface<R> && de::KVPInterface<R>
+static de::MutableBuffer<R> *create_weighted_mbuffer(size_t cnt)
+{
+    auto buffer = new de::MutableBuffer<R>(cnt/2, cnt);
+    
+    // Put in half of the count with weight one.
+    for (uint32_t i=0; i< cnt / 2; i++) {
+        buffer->append(R {1, i, 2});
+    }
+
+    // put in a quarter of the count with weight four.
+    for (uint32_t i=0; i< cnt / 4; i++) {
+        buffer->append(R {2, i, 4});
+    }
+
+    // the remaining quarter with weight eight.
+    for (uint32_t i=0; i< cnt / 4; i++) {
+        buffer->append(R {3, i, 8});
+    }
+
+    return buffer;
+}
+
+template <de::KVPInterface R>
+static de::MutableBuffer<R> *create_double_seq_mbuffer(size_t cnt, bool ts=false) 
+{
+    auto buffer = new de::MutableBuffer<R>(cnt/2, cnt);
+
+    for (size_t i = 0; i < cnt / 2; i++) { 
+        R rec;
+        rec.key = i;
+        rec.value = i;
+
+        buffer->append(rec, ts);
+    }
+
+    for (size_t i = 0; i < cnt / 2; i++) {
+        R rec;
+        rec.key = i;
+        rec.value = i + 1;
+
+        buffer->append(rec, ts);
+    }
+
+    return buffer;
+}
+
+
-- 
cgit v1.2.3


From 2117935e85412f3733ee0bcb1830c7fd0b129b29 Mon Sep 17 00:00:00 2001
From: "Douglas B. Rumbaugh" <doug@douglasrumbaugh.com>
Date: Mon, 15 Jan 2024 17:23:57 -0500
Subject: Concurrency testing and bug fixes

---
 tests/include/concurrent_extension.h | 379 +++++++++++++++++++++++++++++++++++
 1 file changed, 379 insertions(+)
 create mode 100644 tests/include/concurrent_extension.h

(limited to 'tests/include')

diff --git a/tests/include/concurrent_extension.h b/tests/include/concurrent_extension.h
new file mode 100644
index 0000000..86f8e12
--- /dev/null
+++ b/tests/include/concurrent_extension.h
@@ -0,0 +1,379 @@
+/*
+ * tests/include/dynamic_extension.h
+ *
+ * Standardized unit tests for DynamicExtension objects
+ *
+ * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> 
+ *
+ * Distributed under the Modified BSD License.
+ *
+ * WARNING: This file must be included in the main unit test set
+ *          after the definition of an appropriate Shard, Query, and Rec
+ *          type. In particular, Rec needs to implement the key-value
+ *          pair interface. For other types of record, you'll need to
+ *          use a different set of unit tests.
+ */
+#pragma once
+
+/*
+ * Uncomment these lines temporarily to remove errors in this file
+ * temporarily for development purposes. They should be removed prior
+ * to building, to ensure no duplicate definitions. These includes/defines
+ * should be included in the source file that includes this one, above the
+ * include statement.
+ */
+#include "testing.h"
+#include "framework/DynamicExtension.h"
+#include "framework/scheduling/FIFOScheduler.h"
+#include "shard/ISAMTree.h"
+#include "query/rangequery.h"
+#include <check.h>
+using namespace de;
+typedef DynamicExtension<Rec, ISAMTree<Rec>, rq::Query<ISAMTree<Rec>, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE;
+
+
+START_TEST(t_create)
+{
+    auto test_de = new DE(100, 1000, 2);
+
+    ck_assert_ptr_nonnull(test_de);
+    ck_assert_int_eq(test_de->get_record_count(), 0);
+    ck_assert_int_eq(test_de->get_height(), 0);
+
+    delete test_de;
+}
+END_TEST
+
+
+START_TEST(t_insert)
+{
+    auto test_de = new DE(100, 1000, 2);
+
+    uint64_t key = 0;
+    uint32_t val = 0;
+    for (size_t i=0; i<100; i++) {
+        Rec r = {key, val};
+        ck_assert_int_eq(test_de->insert(r), 1);
+        key++;
+        val++;
+    }
+
+    ck_assert_int_eq(test_de->get_height(), 0);
+    ck_assert_int_eq(test_de->get_record_count(), 100);
+
+    delete test_de;
+}
+END_TEST
+
+
+START_TEST(t_debug_insert)
+{
+    auto test_de = new DE(100, 1000, 2);
+
+    uint64_t key = 0;
+    uint32_t val = 0;
+    for (size_t i=0; i<1000; i++) {
+        Rec r = {key, val};
+        ck_assert_int_eq(test_de->insert(r), 1);
+        //ck_assert_int_eq(test_de->get_record_count(), i+1);
+        key++;
+        val++;
+    }
+
+    delete test_de;
+}
+END_TEST
+
+
+START_TEST(t_insert_with_mem_merges)
+{
+    auto test_de = new DE(100, 1000, 2);
+
+    uint64_t key = 0;
+    uint32_t val = 0;
+
+    Rec r = {key, val};
+    for (size_t i=0; i<1000; i++) {
+        ck_assert_int_eq(test_de->insert(r), 1);
+        r.key++;
+        r.value++;
+    }
+
+    ck_assert_int_eq(test_de->get_record_count(), 1000);
+
+    test_de->await_next_epoch();
+
+    ck_assert_int_eq(test_de->get_record_count(), 1000);
+
+    /* 
+     * verify that we can fill past the high water mark, potentially
+     * stalling to allow merges to finish as needed.
+     */
+    size_t cnt = 0;
+    do {
+        if (test_de->insert(r)) {
+            r.key++;
+            r.value++;
+            cnt++;
+        } else {
+            sleep(1);
+        }
+    } while (cnt < 10000);
+
+    test_de->await_next_epoch();
+
+    ck_assert_int_eq(test_de->get_record_count(), 11000);
+
+    delete test_de;
+}
+END_TEST
+
+
+START_TEST(t_range_query)
+{
+    auto test_de = new DE(100, 1000, 2);
+    size_t n = 10000;
+
+    std::vector<uint64_t> keys;
+    for (size_t i=0; i<n; i++) {
+        keys.push_back(rand() % 25000);
+    }
+
+    std::random_device rd;
+    std::mt19937 gen{rd()};
+    std::shuffle(keys.begin(), keys.end(), gen);
+
+    size_t i=0;
+    while ( i < keys.size()) {
+        Rec r = {keys[i], (uint32_t) i};
+        if (test_de->insert(r)) {
+            i++;
+        } else {
+            sleep(1);
+        }
+    }
+
+    test_de->await_next_epoch();
+
+    std::sort(keys.begin(), keys.end());
+
+    auto idx = rand() % (keys.size() - 250);
+
+    uint64_t lower_key = keys[idx];
+    uint64_t upper_key = keys[idx + 250];
+
+    rq::Parms<Rec> p;
+    p.lower_bound = lower_key;
+    p.upper_bound = upper_key;
+
+    auto result = test_de->query(&p);
+    auto r = result.get();
+    std::sort(r.begin(), r.end());
+    ck_assert_int_eq(r.size(), 251);
+
+    for (size_t i=0; i<r.size(); i++) {
+        ck_assert_int_eq(r[i].key, keys[idx + i]);
+    }
+
+    delete test_de;
+}
+END_TEST
+
+
+START_TEST(t_tombstone_merging_01)
+{
+    size_t reccnt = 100000;
+    auto test_de = new DE(100, 1000, 2);
+
+    auto rng = gsl_rng_alloc(gsl_rng_mt19937);
+
+    std::set<std::pair<uint64_t, uint32_t>> records; 
+    std::set<std::pair<uint64_t, uint32_t>> to_delete;
+    std::set<std::pair<uint64_t, uint32_t>> deleted;
+
+    while (records.size() < reccnt) {
+        uint64_t key = rand();
+        uint32_t val = rand();
+
+        if (records.find({key, val}) != records.end()) continue;
+
+        records.insert({key, val});
+    }
+
+    size_t deletes = 0;
+    size_t cnt=0;
+    for (auto rec : records) {
+        Rec r = {rec.first, rec.second};
+        while (!test_de->insert(r)) {
+            sleep(1);
+        }
+
+        if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) {
+            std::vector<std::pair<uint64_t, uint32_t>> del_vec;
+            std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()});
+
+            for (size_t i=0; i<del_vec.size(); i++) {
+                Rec dr = {del_vec[i].first, del_vec[i].second};
+                while (!test_de->erase(dr)) {
+                    sleep(1);
+                }
+                deletes++;
+                to_delete.erase(del_vec[i]);
+                deleted.insert(del_vec[i]);
+            }
+        }
+
+        if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) {
+            to_delete.insert(rec);
+        }
+    }
+
+    test_de->await_next_epoch();
+
+    ck_assert(test_de->validate_tombstone_proportion());
+
+    gsl_rng_free(rng);
+    delete test_de;
+}
+END_TEST
+
+DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) {
+    auto rng = gsl_rng_alloc(gsl_rng_mt19937);
+
+    auto test_de = new DE(1000, 10000, 2);
+
+    std::set<Rec> records; 
+    std::set<Rec> to_delete;
+    std::set<Rec> deleted;
+
+    while (records.size() < reccnt) {
+        uint64_t key = rand();
+        uint32_t val = rand();
+
+        if (records.find({key, val}) != records.end()) continue;
+
+        records.insert({key, val});
+    }
+
+    size_t deletes = 0;
+    for (auto rec : records) {
+        ck_assert_int_eq(test_de->insert(rec), 1);
+
+         if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) {
+            std::vector<Rec> del_vec;
+            std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()});
+
+            for (size_t i=0; i<del_vec.size(); i++) {
+                test_de->erase(del_vec[i]);
+                deletes++;
+                to_delete.erase(del_vec[i]);
+                deleted.insert(del_vec[i]);
+            }
+        }
+
+        if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) {
+            to_delete.insert(rec);
+        }
+    }
+
+    gsl_rng_free(rng);
+
+    return test_de;
+}
+
+START_TEST(t_static_structure)
+{
+    auto rng = gsl_rng_alloc(gsl_rng_mt19937);
+
+    size_t reccnt = 100000;
+    auto test_de = new DE(100, 1000, 2);
+
+    std::set<Rec> records; 
+    std::set<Rec> to_delete;
+    std::set<Rec> deleted;
+
+    while (records.size() < reccnt) {
+        uint64_t key = rand();
+        uint32_t val = rand();
+
+        if (records.find({key, val}) != records.end()) continue;
+
+        records.insert({key, val});
+    }
+
+    size_t deletes = 0;
+    size_t t_reccnt = 0;
+    size_t k=0;
+    for (auto rec : records) {
+        k++;
+        while (!test_de->insert(rec)) {
+            sleep(1);
+        }
+        t_reccnt++;
+
+         if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) {
+            std::vector<Rec> del_vec;
+            std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()});
+
+            for (size_t i=0; i<del_vec.size(); i++) {
+                while (!test_de->erase(del_vec[1])) {
+                    sleep(1);
+                }
+
+                deletes++;
+                to_delete.erase(del_vec[i]);
+                deleted.insert(del_vec[i]);
+            }
+        }
+
+        if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) {
+            to_delete.insert(rec);
+        }
+    }
+
+    auto flat = test_de->create_static_structure();
+    ck_assert_int_eq(flat->get_record_count(), reccnt - deletes);
+
+    uint64_t prev_key = 0;
+    for (size_t i=0; i<flat->get_record_count(); i++) {
+        auto k = flat->get_record_at(i)->rec.key;
+        ck_assert_int_ge(k, prev_key);
+        prev_key = k;
+    }
+
+    gsl_rng_free(rng);
+    delete flat;
+    delete test_de;
+}
+END_TEST
+
+
+static void inject_dynamic_extension_tests(Suite *suite) {
+    TCase *create = tcase_create("de::DynamicExtension::constructor Testing");
+    tcase_add_test(create, t_create);
+    suite_add_tcase(suite, create);
+
+    TCase *insert = tcase_create("de::DynamicExtension::insert Testing");
+    tcase_add_test(insert, t_insert);
+    tcase_add_test(insert, t_insert_with_mem_merges);
+    tcase_add_test(insert, t_debug_insert);
+    tcase_set_timeout(insert, 500);
+    suite_add_tcase(suite, insert);
+
+    /*
+    TCase *query = tcase_create("de::DynamicExtension::range_query Testing");
+    tcase_add_test(query, t_range_query);
+    suite_add_tcase(suite, query);
+
+    
+    TCase *ts = tcase_create("de::DynamicExtension::tombstone_compaction Testing");
+    tcase_add_test(ts, t_tombstone_merging_01);
+    tcase_set_timeout(ts, 500);
+    suite_add_tcase(suite, ts);
+
+    TCase *flat = tcase_create("de::DynamicExtension::create_static_structure Testing");
+    tcase_add_test(flat, t_static_structure);
+    tcase_set_timeout(flat, 500);
+    suite_add_tcase(suite, flat);
+    */
+}
-- 
cgit v1.2.3


From 138c793b0a58577713d98c98bb140cf1d9c79bee Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Wed, 17 Jan 2024 18:22:00 -0500
Subject: Multiple concurrency bug fixes

A poorly organized commit with fixes for a variety of bugs that were
causing missing records. The core problems all appear to be fixed,
though there is an outstanding problem with tombstones not being
completely canceled. A very small number are appearing in the wrong
order during the static structure test.
---
 tests/include/concurrent_extension.h | 54 +++++++++++++++++++++++-------------
 tests/include/rangequery.h           | 25 ++++++++---------
 tests/include/shard_standard.h       |  8 +++---
 3 files changed, 51 insertions(+), 36 deletions(-)

(limited to 'tests/include')

diff --git a/tests/include/concurrent_extension.h b/tests/include/concurrent_extension.h
index 86f8e12..a0e71c9 100644
--- a/tests/include/concurrent_extension.h
+++ b/tests/include/concurrent_extension.h
@@ -28,8 +28,9 @@
 #include "shard/ISAMTree.h"
 #include "query/rangequery.h"
 #include <check.h>
-using namespace de;
-typedef DynamicExtension<Rec, ISAMTree<Rec>, rq::Query<ISAMTree<Rec>, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE;
+
+//using namespace de;
+//typedef DynamicExtension<Rec, ISAMTree<Rec>, rq::Query<ISAMTree<Rec>, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE;
 
 
 START_TEST(t_create)
@@ -75,7 +76,7 @@ START_TEST(t_debug_insert)
     for (size_t i=0; i<1000; i++) {
         Rec r = {key, val};
         ck_assert_int_eq(test_de->insert(r), 1);
-        //ck_assert_int_eq(test_de->get_record_count(), i+1);
+        ck_assert_int_eq(test_de->get_record_count(), i+1);
         key++;
         val++;
     }
@@ -115,14 +116,15 @@ START_TEST(t_insert_with_mem_merges)
             r.key++;
             r.value++;
             cnt++;
+            ck_assert_int_eq(test_de->get_record_count(), cnt + 1000);
         } else {
-            sleep(1);
+            _mm_pause();
         }
-    } while (cnt < 10000);
+    } while (cnt < 100000);
 
     test_de->await_next_epoch();
 
-    ck_assert_int_eq(test_de->get_record_count(), 11000);
+    ck_assert_int_eq(test_de->get_record_count(), 101000);
 
     delete test_de;
 }
@@ -131,12 +133,12 @@ END_TEST
 
 START_TEST(t_range_query)
 {
-    auto test_de = new DE(100, 1000, 2);
-    size_t n = 10000;
+    auto test_de = new DE(1000, 10000, 4);
+    size_t n = 10000000;
 
     std::vector<uint64_t> keys;
     for (size_t i=0; i<n; i++) {
-        keys.push_back(rand() % 25000);
+        keys.push_back(i);
     }
 
     std::random_device rd;
@@ -149,10 +151,11 @@ START_TEST(t_range_query)
         if (test_de->insert(r)) {
             i++;
         } else {
-            sleep(1);
+            _mm_pause();
         }
     }
 
+
     test_de->await_next_epoch();
 
     std::sort(keys.begin(), keys.end());
@@ -166,9 +169,12 @@ START_TEST(t_range_query)
     p.lower_bound = lower_key;
     p.upper_bound = upper_key;
 
+    fprintf(stderr, "query start\n");
     auto result = test_de->query(&p);
     auto r = result.get();
+    fprintf(stderr, "query stop\n");
     std::sort(r.begin(), r.end());
+
     ck_assert_int_eq(r.size(), 251);
 
     for (size_t i=0; i<r.size(); i++) {
@@ -205,7 +211,7 @@ START_TEST(t_tombstone_merging_01)
     for (auto rec : records) {
         Rec r = {rec.first, rec.second};
         while (!test_de->insert(r)) {
-            sleep(1);
+            _mm_pause();
         }
 
         if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) {
@@ -215,7 +221,7 @@ START_TEST(t_tombstone_merging_01)
             for (size_t i=0; i<del_vec.size(); i++) {
                 Rec dr = {del_vec[i].first, del_vec[i].second};
                 while (!test_de->erase(dr)) {
-                    sleep(1);
+                    _mm_pause();
                 }
                 deletes++;
                 to_delete.erase(del_vec[i]);
@@ -307,7 +313,7 @@ START_TEST(t_static_structure)
     for (auto rec : records) {
         k++;
         while (!test_de->insert(rec)) {
-            sleep(1);
+            _mm_pause();
         }
         t_reccnt++;
 
@@ -316,8 +322,8 @@ START_TEST(t_static_structure)
             std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()});
 
             for (size_t i=0; i<del_vec.size(); i++) {
-                while (!test_de->erase(del_vec[1])) {
-                    sleep(1);
+                while (!test_de->erase(del_vec[i])) {
+                    _mm_pause();
                 }
 
                 deletes++;
@@ -331,12 +337,23 @@ START_TEST(t_static_structure)
         }
     }
 
-    auto flat = test_de->create_static_structure();
-    ck_assert_int_eq(flat->get_record_count(), reccnt - deletes);
+
+    //fprintf(stderr, "Tombstones: %ld\tRecords: %ld\n", test_de->get_tombstone_count(), test_de->get_record_count());
+    //fprintf(stderr, "Inserts: %ld\tDeletes:%ld\tNet:%ld\n", reccnt, deletes, reccnt - deletes);
+
+    auto flat = test_de->create_static_structure(true);
+    //fprintf(stderr, "Flat: Tombstones: %ld\tRecords %ld\n", flat->get_tombstone_count(), flat->get_record_count());
+    //ck_assert_int_eq(flat->get_record_count(), reccnt - deletes);
 
     uint64_t prev_key = 0;
     for (size_t i=0; i<flat->get_record_count(); i++) {
         auto k = flat->get_record_at(i)->rec.key;
+        if (flat->get_record_at(i)->is_tombstone()) {
+            fprintf(stderr, "%ld %ld %ld\n", flat->get_record_at(i-1)->rec.key,
+                    flat->get_record_at(i)->rec.key, 
+                    flat->get_record_at(i+1)->rec.key);
+        }
+     //   ck_assert(!flat->get_record_at(i)->is_tombstone());
         ck_assert_int_ge(k, prev_key);
         prev_key = k;
     }
@@ -360,9 +377,9 @@ static void inject_dynamic_extension_tests(Suite *suite) {
     tcase_set_timeout(insert, 500);
     suite_add_tcase(suite, insert);
 
-    /*
     TCase *query = tcase_create("de::DynamicExtension::range_query Testing");
     tcase_add_test(query, t_range_query);
+    tcase_set_timeout(query, 500);
     suite_add_tcase(suite, query);
 
     
@@ -375,5 +392,4 @@ static void inject_dynamic_extension_tests(Suite *suite) {
     tcase_add_test(flat, t_static_structure);
     tcase_set_timeout(flat, 500);
     suite_add_tcase(suite, flat);
-    */
 }
diff --git a/tests/include/rangequery.h b/tests/include/rangequery.h
index 3c7e7e0..e45de57 100644
--- a/tests/include/rangequery.h
+++ b/tests/include/rangequery.h
@@ -24,12 +24,12 @@
  * should be included in the source file that includes this one, above the
  * include statement.
  */
-//#include "shard/ISAMTree.h"
-//#include "query/rangequery.h"
-//#include "testing.h"
-//#include <check.h>
-//using namespace de;
-//typedef ISAMTree<Rec> Shard;
+#include "shard/ISAMTree.h"
+#include "query/rangequery.h"
+#include "testing.h"
+#include <check.h>
+using namespace de;
+typedef ISAMTree<Rec> Shard;
 
 
 START_TEST(t_range_query)
@@ -137,15 +137,12 @@ START_TEST(t_lower_bound)
     auto buffer1 = create_sequential_mbuffer<Rec>(100, 200);
     auto buffer2 = create_sequential_mbuffer<Rec>(400, 1000);
 
-    Shard *shards[2];
+    auto shard1 = new Shard(buffer1->get_buffer_view());
+    auto shard2 = new Shard(buffer2->get_buffer_view());
 
-    auto shard1 = Shard(buffer1->get_buffer_view());
-    auto shard2 = Shard(buffer2->get_buffer_view());
-
-    shards[0] = &shard1;
-    shards[1] = &shard2;
+    std::vector<Shard*> shards = {shard1, shard2};
 
-    auto merged = Shard(shards, 2);
+    auto merged = Shard(shards);
 
     for (size_t i=100; i<1000; i++) {
         Rec r;
@@ -167,6 +164,8 @@ START_TEST(t_lower_bound)
 
     delete buffer1;
     delete buffer2;
+    delete shard1;
+    delete shard2;
 }
 END_TEST
 
diff --git a/tests/include/shard_standard.h b/tests/include/shard_standard.h
index 047a7b5..ddd7614 100644
--- a/tests/include/shard_standard.h
+++ b/tests/include/shard_standard.h
@@ -65,8 +65,8 @@ START_TEST(t_shard_init)
     auto shard2 = new Shard(mbuffer2->get_buffer_view());
     auto shard3 = new Shard(mbuffer3->get_buffer_view());
 
-    Shard* shards[3] = {shard1, shard2, shard3};
-    auto shard4 = new Shard(shards, 3);
+    std::vector<Shard*> shards = {shard1, shard2, shard3};
+    auto shard4 = new Shard(shards);
 
     ck_assert_int_eq(shard4->get_record_count(), n * 3);
     ck_assert_int_eq(shard4->get_tombstone_count(), 0);
@@ -119,9 +119,9 @@ START_TEST(t_full_cancelation)
     ck_assert_int_eq(shard_ts->get_record_count(), n);
     ck_assert_int_eq(shard_ts->get_tombstone_count(), n);
 
-    Shard* shards[] = {shard, shard_ts};
+    std::vector<Shard *> shards = {shard, shard_ts};
 
-    Shard* merged = new Shard(shards, 2);
+    Shard* merged = new Shard(shards);
 
     ck_assert_int_eq(merged->get_tombstone_count(), 0);
     ck_assert_int_eq(merged->get_record_count(), 0);
-- 
cgit v1.2.3


From 4ac2e14d24a1fdd3f9bf777775b16bf6a677f487 Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Mon, 22 Jan 2024 10:14:05 -0500
Subject: Added RangeCount query

---
 tests/include/rangecount.h | 155 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 155 insertions(+)
 create mode 100644 tests/include/rangecount.h

(limited to 'tests/include')

diff --git a/tests/include/rangecount.h b/tests/include/rangecount.h
new file mode 100644
index 0000000..83bf4d4
--- /dev/null
+++ b/tests/include/rangecount.h
@@ -0,0 +1,155 @@
+/*
+ * tests/include/rangecount.h
+ *
+ * Standardized unit tests for range queries against supporting
+ * shard types
+ *
+ * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> 
+ *
+ * Distributed under the Modified BSD License.
+ *
+ * WARNING: This file must be included in the main unit test set
+ *          after the definition of an appropriate Shard and Rec
+ *          type. In particular, Rec needs to implement the key-value
+ *          pair interface and Shard needs to support lower_bound. 
+ *          For other types of record and shard, you'll need to
+ *          use a different set of unit tests.
+ */
+#pragma once
+
+/*
+ * Uncomment these lines temporarily to remove errors in this file
+ * temporarily for development purposes. They should be removed prior
+ * to building, to ensure no duplicate definitions. These includes/defines
+ * should be included in the source file that includes this one, above the
+ * include statement.
+ */
+//#include "shard/ISAMTree.h"
+//#include "query/rangecount.h"
+//#include "testing.h"
+//#include <check.h>
+//using namespace de;
+//typedef ISAMTree<Rec> Shard;
+
+START_TEST(t_range_count)
+{
+    auto buffer = create_sequential_mbuffer<Rec>(100, 1000);
+    auto shard = Shard(buffer->get_buffer_view());
+
+    rc::Parms<Rec> parms;
+    parms.lower_bound = 300;
+    parms.upper_bound = 500;
+
+    auto state = rc::Query<Shard, Rec>::get_query_state(&shard, &parms);
+    auto result = rc::Query<Shard, Rec>::query(&shard, state, &parms);
+    rc::Query<Shard, Rec>::delete_query_state(state);
+
+    ck_assert_int_eq(result.size(), 1);
+    ck_assert_int_eq(result[0].rec.key, parms.upper_bound - parms.lower_bound + 1);
+
+    delete buffer;
+}
+END_TEST
+
+
+START_TEST(t_buffer_range_count)
+{
+    auto buffer = create_sequential_mbuffer<Rec>(100, 1000);
+
+    rc::Parms<Rec> parms;
+    parms.lower_bound = 300;
+    parms.upper_bound = 500;
+
+    auto state = rc::Query<Shard, Rec>::get_buffer_query_state(buffer->get_buffer_view(), &parms);
+    auto result = rc::Query<Shard, Rec>::buffer_query(state, &parms);
+    rc::Query<Shard, Rec>::delete_buffer_query_state(state);
+
+    ck_assert_int_eq(result.size(), 1);
+    ck_assert_int_eq(result[0].rec.key, parms.upper_bound - parms.lower_bound + 1);
+
+    delete buffer;
+}
+END_TEST
+
+
+START_TEST(t_range_count_merge)
+{    
+    auto buffer1 = create_sequential_mbuffer<Rec>(100, 200);
+    auto buffer2 = create_sequential_mbuffer<Rec>(400, 1000);
+
+    auto shard1 = Shard(buffer1->get_buffer_view());
+    auto shard2 = Shard(buffer2->get_buffer_view());
+
+    rc::Parms<Rec> parms;
+    parms.lower_bound = 150;
+    parms.upper_bound = 500;
+
+    size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200;
+
+    auto state1 = rc::Query<Shard, Rec>::get_query_state(&shard1, &parms);
+    auto state2 = rc::Query<Shard, Rec>::get_query_state(&shard2, &parms);
+
+    std::vector<std::vector<de::Wrapped<Rec>>> results(2);
+    results[0] = rc::Query<Shard, Rec>::query(&shard1, state1, &parms);
+    results[1] = rc::Query<Shard, Rec>::query(&shard2, state2, &parms);
+
+    rc::Query<Shard, Rec>::delete_query_state(state1);
+    rc::Query<Shard, Rec>::delete_query_state(state2);
+
+    ck_assert_int_eq(results[0].size(), 1);
+    ck_assert_int_eq(results[1].size(), 1);
+
+    auto result = rc::Query<Shard, Rec>::merge(results, nullptr);
+
+    ck_assert_int_eq(result[0].key, result_size);
+
+    delete buffer1;
+    delete buffer2;
+}
+END_TEST
+
+
+START_TEST(t_lower_bound)
+{
+    auto buffer1 = create_sequential_mbuffer<Rec>(100, 200);
+    auto buffer2 = create_sequential_mbuffer<Rec>(400, 1000);
+
+    auto shard1 = new Shard(buffer1->get_buffer_view());
+    auto shard2 = new Shard(buffer2->get_buffer_view());
+
+    std::vector<Shard*> shards = {shard1, shard2};
+
+    auto merged = Shard(shards);
+
+    for (size_t i=100; i<1000; i++) {
+        Rec r;
+        r.key = i;
+        r.value = i;
+
+        auto idx = merged.get_lower_bound(i);
+
+        assert(idx < merged.get_record_count());
+
+        auto res = merged.get_record_at(idx);
+
+        if (i >=200 && i <400) {
+            ck_assert_int_lt(res->rec.key, i);
+        } else {
+            ck_assert_int_eq(res->rec.key, i);
+        }
+    }
+
+    delete buffer1;
+    delete buffer2;
+    delete shard1;
+    delete shard2;
+}
+END_TEST
+
+static void inject_rangecount_tests(Suite *suite) {
+    TCase *range_count = tcase_create("Range Query Testing"); 
+    tcase_add_test(range_count, t_range_count); 
+    tcase_add_test(range_count, t_buffer_range_count); 
+    tcase_add_test(range_count, t_range_count_merge); 
+    suite_add_tcase(suite, range_count);
+}
-- 
cgit v1.2.3


From 51a85013236f4b2bd596caf179d90e67c848963c Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Tue, 30 Jan 2024 15:31:34 -0500
Subject: TrieSpline + tests

---
 tests/include/rangequery.h | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

(limited to 'tests/include')

diff --git a/tests/include/rangequery.h b/tests/include/rangequery.h
index e45de57..1ac0891 100644
--- a/tests/include/rangequery.h
+++ b/tests/include/rangequery.h
@@ -24,12 +24,12 @@
  * should be included in the source file that includes this one, above the
  * include statement.
  */
-#include "shard/ISAMTree.h"
-#include "query/rangequery.h"
-#include "testing.h"
-#include <check.h>
-using namespace de;
-typedef ISAMTree<Rec> Shard;
+//#include "shard/ISAMTree.h"
+//#include "query/rangequery.h"
+//#include "testing.h"
+//#include <check.h>
+//using namespace de;
+//typedef ISAMTree<Rec> Shard;
 
 
 START_TEST(t_range_query)
-- 
cgit v1.2.3


From db4806d9dd9757273a14e6c3ea92e5a087239145 Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Mon, 5 Feb 2024 15:17:25 -0500
Subject: Set up tombstone deletes properly

---
 tests/include/rangecount.h | 14 +++++++++-----
 tests/include/rangequery.h | 19 +++++++++++--------
 2 files changed, 20 insertions(+), 13 deletions(-)

(limited to 'tests/include')

diff --git a/tests/include/rangecount.h b/tests/include/rangecount.h
index 83bf4d4..e09ab12 100644
--- a/tests/include/rangecount.h
+++ b/tests/include/rangecount.h
@@ -33,6 +33,7 @@
 
 START_TEST(t_range_count)
 {
+    
     auto buffer = create_sequential_mbuffer<Rec>(100, 1000);
     auto shard = Shard(buffer->get_buffer_view());
 
@@ -60,12 +61,15 @@ START_TEST(t_buffer_range_count)
     parms.lower_bound = 300;
     parms.upper_bound = 500;
 
-    auto state = rc::Query<Shard, Rec>::get_buffer_query_state(buffer->get_buffer_view(), &parms);
-    auto result = rc::Query<Shard, Rec>::buffer_query(state, &parms);
-    rc::Query<Shard, Rec>::delete_buffer_query_state(state);
+    {
+        auto view = buffer->get_buffer_view();
+        auto state = rc::Query<Shard, Rec>::get_buffer_query_state(&view, &parms);
+        auto result = rc::Query<Shard, Rec>::buffer_query(state, &parms);
+        rc::Query<Shard, Rec>::delete_buffer_query_state(state);
 
-    ck_assert_int_eq(result.size(), 1);
-    ck_assert_int_eq(result[0].rec.key, parms.upper_bound - parms.lower_bound + 1);
+        ck_assert_int_eq(result.size(), 1);
+        ck_assert_int_eq(result[0].rec.key, parms.upper_bound - parms.lower_bound + 1);
+    }
 
     delete buffer;
 }
diff --git a/tests/include/rangequery.h b/tests/include/rangequery.h
index 1ac0891..b9694a4 100644
--- a/tests/include/rangequery.h
+++ b/tests/include/rangequery.h
@@ -64,14 +64,17 @@ START_TEST(t_buffer_range_query)
     parms.lower_bound = 300;
     parms.upper_bound = 500;
 
-    auto state = rq::Query<Shard, Rec>::get_buffer_query_state(buffer->get_buffer_view(), &parms);
-    auto result = rq::Query<Shard, Rec>::buffer_query(state, &parms);
-    rq::Query<Shard, Rec>::delete_buffer_query_state(state);
-
-    ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1);
-    for (size_t i=0; i<result.size(); i++) {
-        ck_assert_int_le(result[i].rec.key, parms.upper_bound);
-        ck_assert_int_ge(result[i].rec.key, parms.lower_bound);
+    {
+        auto view = buffer->get_buffer_view();
+        auto state = rq::Query<Shard, Rec>::get_buffer_query_state(&view, &parms);
+        auto result = rq::Query<Shard, Rec>::buffer_query(state, &parms);
+        rq::Query<Shard, Rec>::delete_buffer_query_state(state);
+
+        ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1);
+        for (size_t i=0; i<result.size(); i++) {
+            ck_assert_int_le(result[i].rec.key, parms.upper_bound);
+            ck_assert_int_ge(result[i].rec.key, parms.lower_bound);
+        }
     }
 
     delete buffer;
-- 
cgit v1.2.3


From 10b4425e842d10b7fbfa85978969ed4591d6b98e Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Wed, 7 Feb 2024 10:56:52 -0500
Subject: Fully implemented Query concept and adjusted queries to use it

---
 tests/include/concurrent_extension.h | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

(limited to 'tests/include')

diff --git a/tests/include/concurrent_extension.h b/tests/include/concurrent_extension.h
index a0e71c9..24cb2ce 100644
--- a/tests/include/concurrent_extension.h
+++ b/tests/include/concurrent_extension.h
@@ -22,7 +22,7 @@
  * should be included in the source file that includes this one, above the
  * include statement.
  */
-#include "testing.h"
+/*#include "testing.h"
 #include "framework/DynamicExtension.h"
 #include "framework/scheduling/FIFOScheduler.h"
 #include "shard/ISAMTree.h"
@@ -31,6 +31,7 @@
 
 //using namespace de;
 //typedef DynamicExtension<Rec, ISAMTree<Rec>, rq::Query<ISAMTree<Rec>, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE;
+*/
 
 
 START_TEST(t_create)
@@ -169,10 +170,10 @@ START_TEST(t_range_query)
     p.lower_bound = lower_key;
     p.upper_bound = upper_key;
 
-    fprintf(stderr, "query start\n");
+    //fprintf(stderr, "query start\n");
     auto result = test_de->query(&p);
     auto r = result.get();
-    fprintf(stderr, "query stop\n");
+    //fprintf(stderr, "query stop\n");
     std::sort(r.begin(), r.end());
 
     ck_assert_int_eq(r.size(), 251);
-- 
cgit v1.2.3


From 2c5d549b3618b9ea72e6eece4cb4f3da5a6811a8 Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Wed, 7 Feb 2024 13:42:34 -0500
Subject: Fully realized shard concept interface

---
 tests/include/rangecount.h | 26 +++++++++++++-------------
 tests/include/rangequery.h | 26 +++++++++++++-------------
 2 files changed, 26 insertions(+), 26 deletions(-)

(limited to 'tests/include')

diff --git a/tests/include/rangecount.h b/tests/include/rangecount.h
index e09ab12..471af27 100644
--- a/tests/include/rangecount.h
+++ b/tests/include/rangecount.h
@@ -41,9 +41,9 @@ START_TEST(t_range_count)
     parms.lower_bound = 300;
     parms.upper_bound = 500;
 
-    auto state = rc::Query<Shard, Rec>::get_query_state(&shard, &parms);
-    auto result = rc::Query<Shard, Rec>::query(&shard, state, &parms);
-    rc::Query<Shard, Rec>::delete_query_state(state);
+    auto state = rc::Query<Rec, Shard>::get_query_state(&shard, &parms);
+    auto result = rc::Query<Rec, Shard>::query(&shard, state, &parms);
+    rc::Query<Rec, Shard>::delete_query_state(state);
 
     ck_assert_int_eq(result.size(), 1);
     ck_assert_int_eq(result[0].rec.key, parms.upper_bound - parms.lower_bound + 1);
@@ -63,9 +63,9 @@ START_TEST(t_buffer_range_count)
 
     {
         auto view = buffer->get_buffer_view();
-        auto state = rc::Query<Shard, Rec>::get_buffer_query_state(&view, &parms);
-        auto result = rc::Query<Shard, Rec>::buffer_query(state, &parms);
-        rc::Query<Shard, Rec>::delete_buffer_query_state(state);
+        auto state = rc::Query<Rec, Shard>::get_buffer_query_state(&view, &parms);
+        auto result = rc::Query<Rec, Shard>::buffer_query(state, &parms);
+        rc::Query<Rec, Shard>::delete_buffer_query_state(state);
 
         ck_assert_int_eq(result.size(), 1);
         ck_assert_int_eq(result[0].rec.key, parms.upper_bound - parms.lower_bound + 1);
@@ -90,20 +90,20 @@ START_TEST(t_range_count_merge)
 
     size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200;
 
-    auto state1 = rc::Query<Shard, Rec>::get_query_state(&shard1, &parms);
-    auto state2 = rc::Query<Shard, Rec>::get_query_state(&shard2, &parms);
+    auto state1 = rc::Query<Rec, Shard>::get_query_state(&shard1, &parms);
+    auto state2 = rc::Query<Rec, Shard>::get_query_state(&shard2, &parms);
 
     std::vector<std::vector<de::Wrapped<Rec>>> results(2);
-    results[0] = rc::Query<Shard, Rec>::query(&shard1, state1, &parms);
-    results[1] = rc::Query<Shard, Rec>::query(&shard2, state2, &parms);
+    results[0] = rc::Query<Rec, Shard>::query(&shard1, state1, &parms);
+    results[1] = rc::Query<Rec, Shard>::query(&shard2, state2, &parms);
 
-    rc::Query<Shard, Rec>::delete_query_state(state1);
-    rc::Query<Shard, Rec>::delete_query_state(state2);
+    rc::Query<Rec, Shard>::delete_query_state(state1);
+    rc::Query<Rec, Shard>::delete_query_state(state2);
 
     ck_assert_int_eq(results[0].size(), 1);
     ck_assert_int_eq(results[1].size(), 1);
 
-    auto result = rc::Query<Shard, Rec>::merge(results, nullptr);
+    auto result = rc::Query<Rec, Shard>::merge(results, nullptr);
 
     ck_assert_int_eq(result[0].key, result_size);
 
diff --git a/tests/include/rangequery.h b/tests/include/rangequery.h
index b9694a4..dbb71db 100644
--- a/tests/include/rangequery.h
+++ b/tests/include/rangequery.h
@@ -41,9 +41,9 @@ START_TEST(t_range_query)
     parms.lower_bound = 300;
     parms.upper_bound = 500;
 
-    auto state = rq::Query<Shard, Rec>::get_query_state(&shard, &parms);
-    auto result = rq::Query<Shard, Rec>::query(&shard, state, &parms);
-    rq::Query<Shard, Rec>::delete_query_state(state);
+    auto state = rq::Query<Rec, Shard>::get_query_state(&shard, &parms);
+    auto result = rq::Query<Rec, Shard>::query(&shard, state, &parms);
+    rq::Query<Rec, Shard>::delete_query_state(state);
 
     ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1);
     for (size_t i=0; i<result.size(); i++) {
@@ -66,9 +66,9 @@ START_TEST(t_buffer_range_query)
 
     {
         auto view = buffer->get_buffer_view();
-        auto state = rq::Query<Shard, Rec>::get_buffer_query_state(&view, &parms);
-        auto result = rq::Query<Shard, Rec>::buffer_query(state, &parms);
-        rq::Query<Shard, Rec>::delete_buffer_query_state(state);
+        auto state = rq::Query<Rec, Shard>::get_buffer_query_state(&view, &parms);
+        auto result = rq::Query<Rec, Shard>::buffer_query(state, &parms);
+        rq::Query<Rec, Shard>::delete_buffer_query_state(state);
 
         ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1);
         for (size_t i=0; i<result.size(); i++) {
@@ -96,15 +96,15 @@ START_TEST(t_range_query_merge)
 
     size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200;
 
-    auto state1 = rq::Query<Shard, Rec>::get_query_state(&shard1, &parms);
-    auto state2 = rq::Query<Shard, Rec>::get_query_state(&shard2, &parms);
+    auto state1 = rq::Query<Rec, Shard>::get_query_state(&shard1, &parms);
+    auto state2 = rq::Query<Rec, Shard>::get_query_state(&shard2, &parms);
 
     std::vector<std::vector<de::Wrapped<Rec>>> results(2);
-    results[0] = rq::Query<Shard, Rec>::query(&shard1, state1, &parms);
-    results[1] = rq::Query<Shard, Rec>::query(&shard2, state2, &parms);
+    results[0] = rq::Query<Rec, Shard>::query(&shard1, state1, &parms);
+    results[1] = rq::Query<Rec, Shard>::query(&shard2, state2, &parms);
 
-    rq::Query<Shard, Rec>::delete_query_state(state1);
-    rq::Query<Shard, Rec>::delete_query_state(state2);
+    rq::Query<Rec, Shard>::delete_query_state(state1);
+    rq::Query<Rec, Shard>::delete_query_state(state2);
 
     ck_assert_int_eq(results[0].size() + results[1].size(), result_size);
 
@@ -117,7 +117,7 @@ START_TEST(t_range_query_merge)
         }
     }
 
-    auto result = rq::Query<Shard, Rec>::merge(proc_results, nullptr);
+    auto result = rq::Query<Rec, Shard>::merge(proc_results, nullptr);
     std::sort(result.begin(), result.end());
 
     ck_assert_int_eq(result.size(), result_size);
-- 
cgit v1.2.3


From bd74e27b28bd95267ce50d2e4b6f12b51d9b6aae Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Wed, 7 Feb 2024 17:23:23 -0500
Subject: Cleaned up shard files (except VPTree)

Cleaned up shard implementations, fixed a few bugs, and set up some
tests. There's still some work to be done in creating tests for the
weighted sampling operations for the alias and aug btree shards.
---
 tests/include/concurrent_extension.h |  40 ++++----
 tests/include/dynamic_extension.h    |  36 +++----
 tests/include/rangecount.h           |  57 +++++------
 tests/include/rangequery.h           |  60 ++++++------
 tests/include/shard_standard.h       |  26 ++---
 tests/include/wirs.h                 | 181 +++++++++++++++++++++++++++++++++++
 tests/include/wss.h                  | 144 ++++++++++++++++++++++++++++
 7 files changed, 437 insertions(+), 107 deletions(-)
 create mode 100644 tests/include/wirs.h
 create mode 100644 tests/include/wss.h

(limited to 'tests/include')

diff --git a/tests/include/concurrent_extension.h b/tests/include/concurrent_extension.h
index 24cb2ce..0993fac 100644
--- a/tests/include/concurrent_extension.h
+++ b/tests/include/concurrent_extension.h
@@ -8,8 +8,8 @@
  * Distributed under the Modified BSD License.
  *
  * WARNING: This file must be included in the main unit test set
- *          after the definition of an appropriate Shard, Query, and Rec
- *          type. In particular, Rec needs to implement the key-value
+ *          after the definition of an appropriate Shard, Query, and R
+ *          type. In particular, R needs to implement the key-value
  *          pair interface. For other types of record, you'll need to
  *          use a different set of unit tests.
  */
@@ -30,7 +30,7 @@
 #include <check.h>
 
 //using namespace de;
-//typedef DynamicExtension<Rec, ISAMTree<Rec>, rq::Query<ISAMTree<Rec>, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE;
+//typedef DynamicExtension<R, ISAMTree<R>, rq::Query<ISAMTree<R>, R>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE;
 */
 
 
@@ -54,7 +54,7 @@ START_TEST(t_insert)
     uint64_t key = 0;
     uint32_t val = 0;
     for (size_t i=0; i<100; i++) {
-        Rec r = {key, val};
+        R r = {key, val};
         ck_assert_int_eq(test_de->insert(r), 1);
         key++;
         val++;
@@ -75,7 +75,7 @@ START_TEST(t_debug_insert)
     uint64_t key = 0;
     uint32_t val = 0;
     for (size_t i=0; i<1000; i++) {
-        Rec r = {key, val};
+        R r = {key, val};
         ck_assert_int_eq(test_de->insert(r), 1);
         ck_assert_int_eq(test_de->get_record_count(), i+1);
         key++;
@@ -94,7 +94,7 @@ START_TEST(t_insert_with_mem_merges)
     uint64_t key = 0;
     uint32_t val = 0;
 
-    Rec r = {key, val};
+    R r = {key, val};
     for (size_t i=0; i<1000; i++) {
         ck_assert_int_eq(test_de->insert(r), 1);
         r.key++;
@@ -148,7 +148,7 @@ START_TEST(t_range_query)
 
     size_t i=0;
     while ( i < keys.size()) {
-        Rec r = {keys[i], (uint32_t) i};
+        R r = {keys[i], (uint32_t) i};
         if (test_de->insert(r)) {
             i++;
         } else {
@@ -166,7 +166,7 @@ START_TEST(t_range_query)
     uint64_t lower_key = keys[idx];
     uint64_t upper_key = keys[idx + 250];
 
-    rq::Parms<Rec> p;
+    rq::Parms<R> p;
     p.lower_bound = lower_key;
     p.upper_bound = upper_key;
 
@@ -210,7 +210,7 @@ START_TEST(t_tombstone_merging_01)
     size_t deletes = 0;
     size_t cnt=0;
     for (auto rec : records) {
-        Rec r = {rec.first, rec.second};
+        R r = {rec.first, rec.second};
         while (!test_de->insert(r)) {
             _mm_pause();
         }
@@ -220,7 +220,7 @@ START_TEST(t_tombstone_merging_01)
             std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()});
 
             for (size_t i=0; i<del_vec.size(); i++) {
-                Rec dr = {del_vec[i].first, del_vec[i].second};
+                R dr = {del_vec[i].first, del_vec[i].second};
                 while (!test_de->erase(dr)) {
                     _mm_pause();
                 }
@@ -249,9 +249,9 @@ DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) {
 
     auto test_de = new DE(1000, 10000, 2);
 
-    std::set<Rec> records; 
-    std::set<Rec> to_delete;
-    std::set<Rec> deleted;
+    std::set<R> records; 
+    std::set<R> to_delete;
+    std::set<R> deleted;
 
     while (records.size() < reccnt) {
         uint64_t key = rand();
@@ -267,7 +267,7 @@ DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) {
         ck_assert_int_eq(test_de->insert(rec), 1);
 
          if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) {
-            std::vector<Rec> del_vec;
+            std::vector<R> del_vec;
             std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()});
 
             for (size_t i=0; i<del_vec.size(); i++) {
@@ -295,9 +295,9 @@ START_TEST(t_static_structure)
     size_t reccnt = 100000;
     auto test_de = new DE(100, 1000, 2);
 
-    std::set<Rec> records; 
-    std::set<Rec> to_delete;
-    std::set<Rec> deleted;
+    std::set<R> records; 
+    std::set<R> to_delete;
+    std::set<R> deleted;
 
     while (records.size() < reccnt) {
         uint64_t key = rand();
@@ -319,7 +319,7 @@ START_TEST(t_static_structure)
         t_reccnt++;
 
          if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) {
-            std::vector<Rec> del_vec;
+            std::vector<R> del_vec;
             std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()});
 
             for (size_t i=0; i<del_vec.size(); i++) {
@@ -339,11 +339,11 @@ START_TEST(t_static_structure)
     }
 
 
-    //fprintf(stderr, "Tombstones: %ld\tRecords: %ld\n", test_de->get_tombstone_count(), test_de->get_record_count());
+    //fprintf(stderr, "Tombstones: %ld\tRords: %ld\n", test_de->get_tombstone_count(), test_de->get_record_count());
     //fprintf(stderr, "Inserts: %ld\tDeletes:%ld\tNet:%ld\n", reccnt, deletes, reccnt - deletes);
 
     auto flat = test_de->create_static_structure(true);
-    //fprintf(stderr, "Flat: Tombstones: %ld\tRecords %ld\n", flat->get_tombstone_count(), flat->get_record_count());
+    //fprintf(stderr, "Flat: Tombstones: %ld\tRords %ld\n", flat->get_tombstone_count(), flat->get_record_count());
     //ck_assert_int_eq(flat->get_record_count(), reccnt - deletes);
 
     uint64_t prev_key = 0;
diff --git a/tests/include/dynamic_extension.h b/tests/include/dynamic_extension.h
index 5a08f5a..f0f13dd 100644
--- a/tests/include/dynamic_extension.h
+++ b/tests/include/dynamic_extension.h
@@ -8,8 +8,8 @@
  * Distributed under the Modified BSD License.
  *
  * WARNING: This file must be included in the main unit test set
- *          after the definition of an appropriate Shard, Query, and Rec
- *          type. In particular, Rec needs to implement the key-value
+ *          after the definition of an appropriate Shard, Query, and R
+ *          type. In particular, R needs to implement the key-value
  *          pair interface. For other types of record, you'll need to
  *          use a different set of unit tests.
  */
@@ -29,7 +29,7 @@
 //#include "query/rangequery.h"
 //#include <check.h>
 //using namespace de;
-//typedef DynamicExtension<Rec, ISAMTree<Rec>, rq::Query<ISAMTree<Rec>, Rec>, LayoutPolicy::TEIRING, DeletePolicy::TAGGING, SerialScheduler> DE;
+//typedef DynamicExtension<R, ISAMTree<R>, rq::Query<ISAMTree<R>, R>, LayoutPolicy::TEIRING, DeletePolicy::TAGGING, SerialScheduler> DE;
 
 
 START_TEST(t_create)
@@ -52,7 +52,7 @@ START_TEST(t_insert)
     uint64_t key = 0;
     uint32_t val = 0;
     for (size_t i=0; i<100; i++) {
-        Rec r = {key, val};
+        R r = {key, val};
         ck_assert_int_eq(test_de->insert(r), 1);
         key++;
         val++;
@@ -73,7 +73,7 @@ START_TEST(t_debug_insert)
     uint64_t key = 0;
     uint32_t val = 0;
     for (size_t i=0; i<1000; i++) {
-        Rec r = {key, val};
+        R r = {key, val};
         ck_assert_int_eq(test_de->insert(r), 1);
         ck_assert_int_eq(test_de->get_record_count(), i+1);
         key++;
@@ -92,7 +92,7 @@ START_TEST(t_insert_with_mem_merges)
     uint64_t key = 0;
     uint32_t val = 0;
     for (size_t i=0; i<300; i++) {
-        Rec r = {key, val};
+        R r = {key, val};
         ck_assert_int_eq(test_de->insert(r), 1);
         key++;
         val++;
@@ -123,7 +123,7 @@ START_TEST(t_range_query)
     std::shuffle(keys.begin(), keys.end(), gen);
 
     for (size_t i=0; i<keys.size(); i++) {
-        Rec r = {keys[i], (uint32_t) i};
+        R r = {keys[i], (uint32_t) i};
         ck_assert_int_eq(test_de->insert(r), 1);
     }
 
@@ -136,7 +136,7 @@ START_TEST(t_range_query)
     uint64_t lower_key = keys[idx];
     uint64_t upper_key = keys[idx + 250];
 
-    rq::Parms<Rec> p;
+    rq::Parms<R> p;
     p.lower_bound = lower_key;
     p.upper_bound = upper_key;
 
@@ -177,7 +177,7 @@ START_TEST(t_tombstone_merging_01)
     size_t deletes = 0;
     size_t cnt=0;
     for (auto rec : records) {
-        Rec r = {rec.first, rec.second};
+        R r = {rec.first, rec.second};
         ck_assert_int_eq(test_de->insert(r), 1);
 
          if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) {
@@ -185,7 +185,7 @@ START_TEST(t_tombstone_merging_01)
             std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()});
 
             for (size_t i=0; i<del_vec.size(); i++) {
-                Rec dr = {del_vec[i].first, del_vec[i].second};
+                R dr = {del_vec[i].first, del_vec[i].second};
                 test_de->erase(dr);
                 deletes++;
                 to_delete.erase(del_vec[i]);
@@ -212,9 +212,9 @@ DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) {
 
     auto test_de = new DE(1000, 10000, 2);
 
-    std::set<Rec> records; 
-    std::set<Rec> to_delete;
-    std::set<Rec> deleted;
+    std::set<R> records; 
+    std::set<R> to_delete;
+    std::set<R> deleted;
 
     while (records.size() < reccnt) {
         uint64_t key = rand();
@@ -230,7 +230,7 @@ DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) {
         ck_assert_int_eq(test_de->insert(rec), 1);
 
          if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) {
-            std::vector<Rec> del_vec;
+            std::vector<R> del_vec;
             std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()});
 
             for (size_t i=0; i<del_vec.size(); i++) {
@@ -258,9 +258,9 @@ START_TEST(t_static_structure)
     size_t reccnt = 100000;
     auto test_de = new DE(100, 1000, 2);
 
-    std::set<Rec> records; 
-    std::set<Rec> to_delete;
-    std::set<Rec> deleted;
+    std::set<R> records; 
+    std::set<R> to_delete;
+    std::set<R> deleted;
 
     while (records.size() < reccnt) {
         uint64_t key = rand();
@@ -280,7 +280,7 @@ START_TEST(t_static_structure)
         t_reccnt++;
 
          if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) {
-            std::vector<Rec> del_vec;
+            std::vector<R> del_vec;
             std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()});
 
             for (size_t i=0; i<del_vec.size(); i++) {
diff --git a/tests/include/rangecount.h b/tests/include/rangecount.h
index 471af27..fdd66d9 100644
--- a/tests/include/rangecount.h
+++ b/tests/include/rangecount.h
@@ -9,8 +9,8 @@
  * Distributed under the Modified BSD License.
  *
  * WARNING: This file must be included in the main unit test set
- *          after the definition of an appropriate Shard and Rec
- *          type. In particular, Rec needs to implement the key-value
+ *          after the definition of an appropriate Shard and R
+ *          type. In particular, R needs to implement the key-value
  *          pair interface and Shard needs to support lower_bound. 
  *          For other types of record and shard, you'll need to
  *          use a different set of unit tests.
@@ -29,21 +29,24 @@
 //#include "testing.h"
 //#include <check.h>
 //using namespace de;
-//typedef ISAMTree<Rec> Shard;
+//typedef ISAMTree<R> Shard;
+
+
+#include "query/rangecount.h"
 
 START_TEST(t_range_count)
 {
     
-    auto buffer = create_sequential_mbuffer<Rec>(100, 1000);
+    auto buffer = create_sequential_mbuffer<R>(100, 1000);
     auto shard = Shard(buffer->get_buffer_view());
 
-    rc::Parms<Rec> parms;
+    rc::Parms<R> parms;
     parms.lower_bound = 300;
     parms.upper_bound = 500;
 
-    auto state = rc::Query<Rec, Shard>::get_query_state(&shard, &parms);
-    auto result = rc::Query<Rec, Shard>::query(&shard, state, &parms);
-    rc::Query<Rec, Shard>::delete_query_state(state);
+    auto state = rc::Query<R, Shard>::get_query_state(&shard, &parms);
+    auto result = rc::Query<R, Shard>::query(&shard, state, &parms);
+    rc::Query<R, Shard>::delete_query_state(state);
 
     ck_assert_int_eq(result.size(), 1);
     ck_assert_int_eq(result[0].rec.key, parms.upper_bound - parms.lower_bound + 1);
@@ -55,17 +58,17 @@ END_TEST
 
 START_TEST(t_buffer_range_count)
 {
-    auto buffer = create_sequential_mbuffer<Rec>(100, 1000);
+    auto buffer = create_sequential_mbuffer<R>(100, 1000);
 
-    rc::Parms<Rec> parms;
+    rc::Parms<R> parms;
     parms.lower_bound = 300;
     parms.upper_bound = 500;
 
     {
         auto view = buffer->get_buffer_view();
-        auto state = rc::Query<Rec, Shard>::get_buffer_query_state(&view, &parms);
-        auto result = rc::Query<Rec, Shard>::buffer_query(state, &parms);
-        rc::Query<Rec, Shard>::delete_buffer_query_state(state);
+        auto state = rc::Query<R, Shard>::get_buffer_query_state(&view, &parms);
+        auto result = rc::Query<R, Shard>::buffer_query(state, &parms);
+        rc::Query<R, Shard>::delete_buffer_query_state(state);
 
         ck_assert_int_eq(result.size(), 1);
         ck_assert_int_eq(result[0].rec.key, parms.upper_bound - parms.lower_bound + 1);
@@ -78,32 +81,32 @@ END_TEST
 
 START_TEST(t_range_count_merge)
 {    
-    auto buffer1 = create_sequential_mbuffer<Rec>(100, 200);
-    auto buffer2 = create_sequential_mbuffer<Rec>(400, 1000);
+    auto buffer1 = create_sequential_mbuffer<R>(100, 200);
+    auto buffer2 = create_sequential_mbuffer<R>(400, 1000);
 
     auto shard1 = Shard(buffer1->get_buffer_view());
     auto shard2 = Shard(buffer2->get_buffer_view());
 
-    rc::Parms<Rec> parms;
+    rc::Parms<R> parms;
     parms.lower_bound = 150;
     parms.upper_bound = 500;
 
     size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200;
 
-    auto state1 = rc::Query<Rec, Shard>::get_query_state(&shard1, &parms);
-    auto state2 = rc::Query<Rec, Shard>::get_query_state(&shard2, &parms);
+    auto state1 = rc::Query<R, Shard>::get_query_state(&shard1, &parms);
+    auto state2 = rc::Query<R, Shard>::get_query_state(&shard2, &parms);
 
-    std::vector<std::vector<de::Wrapped<Rec>>> results(2);
-    results[0] = rc::Query<Rec, Shard>::query(&shard1, state1, &parms);
-    results[1] = rc::Query<Rec, Shard>::query(&shard2, state2, &parms);
+    std::vector<std::vector<de::Wrapped<R>>> results(2);
+    results[0] = rc::Query<R, Shard>::query(&shard1, state1, &parms);
+    results[1] = rc::Query<R, Shard>::query(&shard2, state2, &parms);
 
-    rc::Query<Rec, Shard>::delete_query_state(state1);
-    rc::Query<Rec, Shard>::delete_query_state(state2);
+    rc::Query<R, Shard>::delete_query_state(state1);
+    rc::Query<R, Shard>::delete_query_state(state2);
 
     ck_assert_int_eq(results[0].size(), 1);
     ck_assert_int_eq(results[1].size(), 1);
 
-    auto result = rc::Query<Rec, Shard>::merge(results, nullptr);
+    auto result = rc::Query<R, Shard>::merge(results, nullptr);
 
     ck_assert_int_eq(result[0].key, result_size);
 
@@ -115,8 +118,8 @@ END_TEST
 
 START_TEST(t_lower_bound)
 {
-    auto buffer1 = create_sequential_mbuffer<Rec>(100, 200);
-    auto buffer2 = create_sequential_mbuffer<Rec>(400, 1000);
+    auto buffer1 = create_sequential_mbuffer<R>(100, 200);
+    auto buffer2 = create_sequential_mbuffer<R>(400, 1000);
 
     auto shard1 = new Shard(buffer1->get_buffer_view());
     auto shard2 = new Shard(buffer2->get_buffer_view());
@@ -126,7 +129,7 @@ START_TEST(t_lower_bound)
     auto merged = Shard(shards);
 
     for (size_t i=100; i<1000; i++) {
-        Rec r;
+        R r;
         r.key = i;
         r.value = i;
 
diff --git a/tests/include/rangequery.h b/tests/include/rangequery.h
index dbb71db..a8a73f7 100644
--- a/tests/include/rangequery.h
+++ b/tests/include/rangequery.h
@@ -9,8 +9,8 @@
  * Distributed under the Modified BSD License.
  *
  * WARNING: This file must be included in the main unit test set
- *          after the definition of an appropriate Shard and Rec
- *          type. In particular, Rec needs to implement the key-value
+ *          after the definition of an appropriate Shard and R
+ *          type. In particular, R needs to implement the key-value
  *          pair interface and Shard needs to support lower_bound. 
  *          For other types of record and shard, you'll need to
  *          use a different set of unit tests.
@@ -29,21 +29,23 @@
 //#include "testing.h"
 //#include <check.h>
 //using namespace de;
-//typedef ISAMTree<Rec> Shard;
+//typedef ISAMTree<R> Shard;
+
+#include "query/rangequery.h"
 
 
 START_TEST(t_range_query)
 {
-    auto buffer = create_sequential_mbuffer<Rec>(100, 1000);
+    auto buffer = create_sequential_mbuffer<R>(100, 1000);
     auto shard = Shard(buffer->get_buffer_view());
 
-    rq::Parms<Rec> parms;
+    rq::Parms<R> parms;
     parms.lower_bound = 300;
     parms.upper_bound = 500;
 
-    auto state = rq::Query<Rec, Shard>::get_query_state(&shard, &parms);
-    auto result = rq::Query<Rec, Shard>::query(&shard, state, &parms);
-    rq::Query<Rec, Shard>::delete_query_state(state);
+    auto state = rq::Query<R, Shard>::get_query_state(&shard, &parms);
+    auto result = rq::Query<R, Shard>::query(&shard, state, &parms);
+    rq::Query<R, Shard>::delete_query_state(state);
 
     ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1);
     for (size_t i=0; i<result.size(); i++) {
@@ -58,17 +60,17 @@ END_TEST
 
 START_TEST(t_buffer_range_query)
 {
-    auto buffer = create_sequential_mbuffer<Rec>(100, 1000);
+    auto buffer = create_sequential_mbuffer<R>(100, 1000);
 
-    rq::Parms<Rec> parms;
+    rq::Parms<R> parms;
     parms.lower_bound = 300;
     parms.upper_bound = 500;
 
     {
         auto view = buffer->get_buffer_view();
-        auto state = rq::Query<Rec, Shard>::get_buffer_query_state(&view, &parms);
-        auto result = rq::Query<Rec, Shard>::buffer_query(state, &parms);
-        rq::Query<Rec, Shard>::delete_buffer_query_state(state);
+        auto state = rq::Query<R, Shard>::get_buffer_query_state(&view, &parms);
+        auto result = rq::Query<R, Shard>::buffer_query(state, &parms);
+        rq::Query<R, Shard>::delete_buffer_query_state(state);
 
         ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1);
         for (size_t i=0; i<result.size(); i++) {
@@ -84,40 +86,40 @@ END_TEST
 
 START_TEST(t_range_query_merge)
 {    
-    auto buffer1 = create_sequential_mbuffer<Rec>(100, 200);
-    auto buffer2 = create_sequential_mbuffer<Rec>(400, 1000);
+    auto buffer1 = create_sequential_mbuffer<R>(100, 200);
+    auto buffer2 = create_sequential_mbuffer<R>(400, 1000);
 
     auto shard1 = Shard(buffer1->get_buffer_view());
     auto shard2 = Shard(buffer2->get_buffer_view());
 
-    rq::Parms<Rec> parms;
+    rq::Parms<R> parms;
     parms.lower_bound = 150;
     parms.upper_bound = 500;
 
     size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200;
 
-    auto state1 = rq::Query<Rec, Shard>::get_query_state(&shard1, &parms);
-    auto state2 = rq::Query<Rec, Shard>::get_query_state(&shard2, &parms);
+    auto state1 = rq::Query<R, Shard>::get_query_state(&shard1, &parms);
+    auto state2 = rq::Query<R, Shard>::get_query_state(&shard2, &parms);
 
-    std::vector<std::vector<de::Wrapped<Rec>>> results(2);
-    results[0] = rq::Query<Rec, Shard>::query(&shard1, state1, &parms);
-    results[1] = rq::Query<Rec, Shard>::query(&shard2, state2, &parms);
+    std::vector<std::vector<de::Wrapped<R>>> results(2);
+    results[0] = rq::Query<R, Shard>::query(&shard1, state1, &parms);
+    results[1] = rq::Query<R, Shard>::query(&shard2, state2, &parms);
 
-    rq::Query<Rec, Shard>::delete_query_state(state1);
-    rq::Query<Rec, Shard>::delete_query_state(state2);
+    rq::Query<R, Shard>::delete_query_state(state1);
+    rq::Query<R, Shard>::delete_query_state(state2);
 
     ck_assert_int_eq(results[0].size() + results[1].size(), result_size);
 
-    std::vector<std::vector<Wrapped<Rec>>> proc_results;
+    std::vector<std::vector<Wrapped<R>>> proc_results;
 
     for (size_t j=0; j<results.size(); j++) {
-        proc_results.emplace_back(std::vector<Wrapped<Rec>>());
+        proc_results.emplace_back(std::vector<Wrapped<R>>());
         for (size_t i=0; i<results[j].size(); i++) {
             proc_results[j].emplace_back(results[j][i]);
         }
     }
 
-    auto result = rq::Query<Rec, Shard>::merge(proc_results, nullptr);
+    auto result = rq::Query<R, Shard>::merge(proc_results, nullptr);
     std::sort(result.begin(), result.end());
 
     ck_assert_int_eq(result.size(), result_size);
@@ -137,8 +139,8 @@ END_TEST
 
 START_TEST(t_lower_bound)
 {
-    auto buffer1 = create_sequential_mbuffer<Rec>(100, 200);
-    auto buffer2 = create_sequential_mbuffer<Rec>(400, 1000);
+    auto buffer1 = create_sequential_mbuffer<R>(100, 200);
+    auto buffer2 = create_sequential_mbuffer<R>(400, 1000);
 
     auto shard1 = new Shard(buffer1->get_buffer_view());
     auto shard2 = new Shard(buffer2->get_buffer_view());
@@ -148,7 +150,7 @@ START_TEST(t_lower_bound)
     auto merged = Shard(shards);
 
     for (size_t i=100; i<1000; i++) {
-        Rec r;
+        R r;
         r.key = i;
         r.value = i;
 
diff --git a/tests/include/shard_standard.h b/tests/include/shard_standard.h
index ddd7614..f50c1cb 100644
--- a/tests/include/shard_standard.h
+++ b/tests/include/shard_standard.h
@@ -8,8 +8,8 @@
  * Distributed under the Modified BSD License.
  *
  * WARNING: This file must be included in the main unit test set
- *          after the definition of an appropriate Shard and Rec
- *          type. In particular, Rec needs to implement the key-value
+ *          after the definition of an appropriate Shard and R
+ *          type. In particular, R needs to implement the key-value
  *          pair interface. For other types of record, you'll need to
  *          use a different set of unit tests.
  */
@@ -26,11 +26,11 @@
 //#include "testing.h"
 //#include <check.h>
 //using namespace de;
-//typedef ISAMTree<Rec> Shard;
+//typedef ISAMTree<R> Shard;
 
 START_TEST(t_mbuffer_init)
 {
-    auto buffer = new MutableBuffer<Rec>(512, 1024);
+    auto buffer = new MutableBuffer<R>(512, 1024);
     for (uint64_t i = 512; i > 0; i--) {
         uint32_t v = i;
         buffer->append({i,v, 1});
@@ -57,9 +57,9 @@ START_TEST(t_mbuffer_init)
 START_TEST(t_shard_init)
 {
     size_t n = 512;
-    auto mbuffer1 = create_test_mbuffer<Rec>(n);
-    auto mbuffer2 = create_test_mbuffer<Rec>(n);
-    auto mbuffer3 = create_test_mbuffer<Rec>(n);
+    auto mbuffer1 = create_test_mbuffer<R>(n);
+    auto mbuffer2 = create_test_mbuffer<R>(n);
+    auto mbuffer3 = create_test_mbuffer<R>(n);
 
     auto shard1 = new Shard(mbuffer1->get_buffer_view());
     auto shard2 = new Shard(mbuffer2->get_buffer_view());
@@ -108,8 +108,8 @@ START_TEST(t_shard_init)
 START_TEST(t_full_cancelation)
 {
     size_t n = 100;
-    auto buffer = create_double_seq_mbuffer<Rec>(n, false);
-    auto buffer_ts = create_double_seq_mbuffer<Rec>(n, true);
+    auto buffer = create_double_seq_mbuffer<R>(n, false);
+    auto buffer_ts = create_double_seq_mbuffer<R>(n, true);
 
     Shard* shard = new Shard(buffer->get_buffer_view());
     Shard* shard_ts = new Shard(buffer_ts->get_buffer_view());
@@ -139,14 +139,14 @@ START_TEST(t_point_lookup)
 {
     size_t n = 10000;
 
-    auto buffer = create_double_seq_mbuffer<Rec>(n, false);
+    auto buffer = create_double_seq_mbuffer<R>(n, false);
     auto isam = Shard(buffer->get_buffer_view());
 
     {
         auto view = buffer->get_buffer_view();
 
         for (size_t i=0; i<n; i++) {
-            Rec r;
+            R r;
             auto rec = view.get(i);
             r.key = rec->rec.key;
             r.value = rec->rec.value;
@@ -167,11 +167,11 @@ START_TEST(t_point_lookup_miss)
 {
     size_t n = 10000;
 
-    auto buffer = create_double_seq_mbuffer<Rec>(n, false);
+    auto buffer = create_double_seq_mbuffer<R>(n, false);
     auto isam = Shard(buffer->get_buffer_view());
 
     for (size_t i=n + 100; i<2*n; i++) {
-        Rec r;
+        R r;
         r.key = i;
         r.value = i;
 
diff --git a/tests/include/wirs.h b/tests/include/wirs.h
new file mode 100644
index 0000000..90cd22d
--- /dev/null
+++ b/tests/include/wirs.h
@@ -0,0 +1,181 @@
+/*
+ * tests/include/rangequery.h
+ *
+ * Standardized unit tests for range queries against supporting
+ * shard types
+ *
+ * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> 
+ *
+ * Distributed under the Modified BSD License.
+ *
+ * WARNING: This file must be included in the main unit test set
+ *          after the definition of an appropriate Shard and R
+ *          type. In particular, R needs to implement the key-value
+ *          pair interface and Shard needs to support lower_bound. 
+ *          For other types of record and shard, you'll need to
+ *          use a different set of unit tests.
+ */
+#pragma once
+
+/*
+ * Uncomment these lines temporarily to remove errors in this file
+ * temporarily for development purposes. They should be removed prior
+ * to building, to ensure no duplicate definitions. These includes/defines
+ * should be included in the source file that includes this one, above the
+ * include statement.
+ */
+//#include "shard/ISAMTree.h"
+//#include "query/rangequery.h"
+//#include "testing.h"
+//#include <check.h>
+//using namespace de;
+//typedef ISAMTree<R> Shard;
+
+
+START_TEST(t_range_query)
+{
+    auto buffer = create_sequential_mbuffer<R>(100, 1000);
+    auto shard = Shard(buffer->get_buffer_view());
+
+    rq::Parms<R> parms;
+    parms.lower_bound = 300;
+    parms.upper_bound = 500;
+
+    auto state = rq::Query<R, Shard>::get_query_state(&shard, &parms);
+    auto result = rq::Query<R, Shard>::query(&shard, state, &parms);
+    rq::Query<R, Shard>::delete_query_state(state);
+
+    ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1);
+    for (size_t i=0; i<result.size(); i++) {
+        ck_assert_int_le(result[i].rec.key, parms.upper_bound);
+        ck_assert_int_ge(result[i].rec.key, parms.lower_bound);
+    }
+
+    delete buffer;
+}
+END_TEST
+
+
+START_TEST(t_buffer_range_query)
+{
+    auto buffer = create_sequential_mbuffer<R>(100, 1000);
+
+    rq::Parms<R> parms;
+    parms.lower_bound = 300;
+    parms.upper_bound = 500;
+
+    {
+        auto view = buffer->get_buffer_view();
+        auto state = rq::Query<R, Shard>::get_buffer_query_state(&view, &parms);
+        auto result = rq::Query<R, Shard>::buffer_query(state, &parms);
+        rq::Query<R, Shard>::delete_buffer_query_state(state);
+
+        ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1);
+        for (size_t i=0; i<result.size(); i++) {
+            ck_assert_int_le(result[i].rec.key, parms.upper_bound);
+            ck_assert_int_ge(result[i].rec.key, parms.lower_bound);
+        }
+    }
+
+    delete buffer;
+}
+END_TEST
+
+
+START_TEST(t_range_query_merge)
+{    
+    auto buffer1 = create_sequential_mbuffer<R>(100, 200);
+    auto buffer2 = create_sequential_mbuffer<R>(400, 1000);
+
+    auto shard1 = Shard(buffer1->get_buffer_view());
+    auto shard2 = Shard(buffer2->get_buffer_view());
+
+    rq::Parms<R> parms;
+    parms.lower_bound = 150;
+    parms.upper_bound = 500;
+
+    size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200;
+
+    auto state1 = rq::Query<R, Shard>::get_query_state(&shard1, &parms);
+    auto state2 = rq::Query<R, Shard>::get_query_state(&shard2, &parms);
+
+    std::vector<std::vector<de::Wrapped<R>>> results(2);
+    results[0] = rq::Query<R, Shard>::query(&shard1, state1, &parms);
+    results[1] = rq::Query<R, Shard>::query(&shard2, state2, &parms);
+
+    rq::Query<R, Shard>::delete_query_state(state1);
+    rq::Query<R, Shard>::delete_query_state(state2);
+
+    ck_assert_int_eq(results[0].size() + results[1].size(), result_size);
+
+    std::vector<std::vector<Wrapped<R>>> proc_results;
+
+    for (size_t j=0; j<results.size(); j++) {
+        proc_results.emplace_back(std::vector<Wrapped<R>>());
+        for (size_t i=0; i<results[j].size(); i++) {
+            proc_results[j].emplace_back(results[j][i]);
+        }
+    }
+
+    auto result = rq::Query<R, Shard>::merge(proc_results, nullptr);
+    std::sort(result.begin(), result.end());
+
+    ck_assert_int_eq(result.size(), result_size);
+    auto key = parms.lower_bound;
+    for (size_t i=0; i<result.size(); i++) {
+        ck_assert_int_eq(key++, result[i].key);
+        if (key == 200) {
+            key = 400;
+        }
+    }
+
+    delete buffer1;
+    delete buffer2;
+}
+END_TEST
+
+
+START_TEST(t_lower_bound)
+{
+    auto buffer1 = create_sequential_mbuffer<R>(100, 200);
+    auto buffer2 = create_sequential_mbuffer<R>(400, 1000);
+
+    auto shard1 = new Shard(buffer1->get_buffer_view());
+    auto shard2 = new Shard(buffer2->get_buffer_view());
+
+    std::vector<Shard*> shards = {shard1, shard2};
+
+    auto merged = Shard(shards);
+
+    for (size_t i=100; i<1000; i++) {
+        R r;
+        r.key = i;
+        r.value = i;
+
+        auto idx = merged.get_lower_bound(i);
+
+        assert(idx < merged.get_record_count());
+
+        auto res = merged.get_record_at(idx);
+
+        if (i >=200 && i <400) {
+            ck_assert_int_lt(res->rec.key, i);
+        } else {
+            ck_assert_int_eq(res->rec.key, i);
+        }
+    }
+
+    delete buffer1;
+    delete buffer2;
+    delete shard1;
+    delete shard2;
+}
+END_TEST
+
+static void inject_rangequery_tests(Suite *suite) {
+    TCase *range_query = tcase_create("Range Query Testing"); 
+    tcase_add_test(range_query, t_range_query); 
+    tcase_add_test(range_query, t_buffer_range_query); 
+    tcase_add_test(range_query, t_range_query_merge); 
+    suite_add_tcase(suite, range_query);
+}
diff --git a/tests/include/wss.h b/tests/include/wss.h
new file mode 100644
index 0000000..f0ac74c
--- /dev/null
+++ b/tests/include/wss.h
@@ -0,0 +1,144 @@
+/*
+ * tests/include/rangequery.h
+ *
+ * Standardized unit tests for range queries against supporting
+ * shard types
+ *
+ * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> 
+ *
+ * Distributed under the Modified BSD License.
+ *
+ * WARNING: This file must be included in the main unit test set
+ *          after the definition of an appropriate Shard and R
+ *          type. In particular, R needs to implement the key-value
+ *          pair interface and Shard needs to support lower_bound. 
+ *          For other types of record and shard, you'll need to
+ *          use a different set of unit tests.
+ */
+#pragma once
+
+/*
+ * Uncomment these lines temporarily to remove errors in this file
+ * temporarily for development purposes. They should be removed prior
+ * to building, to ensure no duplicate definitions. These includes/defines
+ * should be included in the source file that includes this one, above the
+ * include statement.
+ */
+#include "shard/Alias.h"
+#include "testing.h"
+#include <check.h>
+using namespace de;
+typedef Alias<R> Shard;
+
+#include "query/wss.h"
+
+START_TEST(t_wss_query)
+{
+    auto buffer = create_weighted_mbuffer<R>(1000);
+    auto shard = Shard(buffer->get_buffer_view());
+
+    auto rng = gsl_rng_alloc(gsl_rng_mt19937);
+
+    wss::Parms<R> parms;
+    parms.rng = rng;
+    parms.sample_size = 20;
+
+    auto state = wss::Query<R, Shard>::get_query_state(&shard, &parms);
+    auto result = wss::Query<R, Shard>::query(&shard, state, &parms);
+    wss::Query<R, Shard>::delete_query_state(state);
+
+    delete buffer;
+    gsl_rng_free(rng);
+}
+END_TEST
+
+
+START_TEST(t_buffer_wss_query)
+{
+    auto buffer = create_weighted_mbuffer<R>(1000);
+
+
+    auto rng = gsl_rng_alloc(gsl_rng_mt19937);
+
+    wss::Parms<R> parms;
+    parms.rng = rng;
+
+    {
+        auto view = buffer->get_buffer_view();
+        auto state = wss::Query<R, Shard>::get_buffer_query_state(&view, &parms);
+        auto result = wss::Query<R, Shard>::buffer_query(state, &parms);
+        wss::Query<R, Shard>::delete_buffer_query_state(state);
+
+        ck_assert_int_eq(result.size(), parms.sample_size);
+        for (size_t i=0; i<result.size(); i++) {
+            
+        }
+    }
+
+    delete buffer;
+}
+END_TEST
+
+
+/*
+START_TEST(t_range_query_merge)
+{    
+    auto buffer1 = create_sequential_mbuffer<R>(100, 200);
+    auto buffer2 = create_sequential_mbuffer<R>(400, 1000);
+
+    auto shard1 = Shard(buffer1->get_buffer_view());
+    auto shard2 = Shard(buffer2->get_buffer_view());
+
+    wss::Parms<R> parms;
+    parms.lower_bound = 150;
+    parms.upper_bound = 500;
+
+    size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200;
+
+    auto state1 = wss::Query<R, Shard>::get_query_state(&shard1, &parms);
+    auto state2 = wss::Query<R, Shard>::get_query_state(&shard2, &parms);
+
+    std::vector<std::vector<de::Wrapped<R>>> results(2);
+    results[0] = wss::Query<R, Shard>::query(&shard1, state1, &parms);
+    results[1] = wss::Query<R, Shard>::query(&shard2, state2, &parms);
+
+    wss::Query<R, Shard>::delete_query_state(state1);
+    wss::Query<R, Shard>::delete_query_state(state2);
+
+    ck_assert_int_eq(results[0].size() + results[1].size(), result_size);
+
+    std::vector<std::vector<Wrapped<R>>> proc_results;
+
+    for (size_t j=0; j<results.size(); j++) {
+        proc_results.emplace_back(std::vector<Wrapped<R>>());
+        for (size_t i=0; i<results[j].size(); i++) {
+            proc_results[j].emplace_back(results[j][i]);
+        }
+    }
+
+    auto result = wss::Query<R, Shard>::merge(proc_results, nullptr);
+    std::sort(result.begin(), result.end());
+
+    ck_assert_int_eq(result.size(), result_size);
+    auto key = parms.lower_bound;
+    for (size_t i=0; i<result.size(); i++) {
+        ck_assert_int_eq(key++, result[i].key);
+        if (key == 200) {
+            key = 400;
+        }
+    }
+
+    delete buffer1;
+    delete buffer2;
+}
+END_TEST
+*/
+
+
+static void inject_wss_tests(Suite *suite) {
+    TCase *wss_query = tcase_create("WSS Query Testing"); 
+    tcase_add_test(wss_query, t_wss_query); 
+    tcase_add_test(wss_query, t_buffer_wss_query); 
+    //tcase_add_test(wss_query, t_wss_query_merge); 
+    suite_add_tcase(suite, wss_query);
+}
-- 
cgit v1.2.3


From 711769574e647839677739192698e400529efe75 Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Thu, 8 Feb 2024 16:38:44 -0500
Subject: Updated VPTree to new shard/query interfaces

---
 tests/include/shard_standard.h | 16 +++++++-----
 tests/include/testing.h        | 59 +++++++++++++++++++-----------------------
 2 files changed, 36 insertions(+), 39 deletions(-)

(limited to 'tests/include')

diff --git a/tests/include/shard_standard.h b/tests/include/shard_standard.h
index f50c1cb..7d17dcb 100644
--- a/tests/include/shard_standard.h
+++ b/tests/include/shard_standard.h
@@ -22,18 +22,22 @@
  * should be included in the source file that includes this one, above the
  * include statement.
  */
-//#include "shard/ISAMTree.h"
-//#include "testing.h"
-//#include <check.h>
-//using namespace de;
-//typedef ISAMTree<R> Shard;
+/*
+#include "shard/ISAMTree.h"
+#include "shard/ISAMTree.h"
+#include "testing.h"
+#include <check.h>
+using namespace de;
+typedef Rec R;
+typedef ISAMTree<R> Shard;
+*/
 
 START_TEST(t_mbuffer_init)
 {
     auto buffer = new MutableBuffer<R>(512, 1024);
     for (uint64_t i = 512; i > 0; i--) {
         uint32_t v = i;
-        buffer->append({i,v, 1});
+        buffer->append({i, v, 1});
     }
     
     for (uint64_t i = 1; i <= 256; ++i) {
diff --git a/tests/include/testing.h b/tests/include/testing.h
index 4e660dd..f935b53 100644
--- a/tests/include/testing.h
+++ b/tests/include/testing.h
@@ -23,7 +23,7 @@
 
 typedef de::WeightedRecord<uint64_t, uint32_t, uint64_t> WRec;
 typedef de::Record<uint64_t, uint32_t> Rec;
-typedef de::EuclidPoint<int64_t> PRec;
+typedef de::EuclidPoint<uint64_t> PRec;
 
 template <de::RecordInterface R> 
 std::vector<R> strip_wrapping(std::vector<de::Wrapped<R>> vec) {
@@ -76,55 +76,48 @@ static bool roughly_equal(int n1, int n2, size_t mag, double epsilon) {
     return ((double) std::abs(n1 - n2) / (double) mag) < epsilon;
 }
 
-static de::MutableBuffer<PRec> *create_2d_mbuffer(size_t cnt) {
-    auto buffer = new de::MutableBuffer<PRec>(cnt/2, cnt);
-
-    for (int64_t i=0; i<cnt; i++) {
-        buffer->append({rand(), rand()});
-    }
-
-    return buffer;
-}
-
-static de::MutableBuffer<PRec> *create_2d_sequential_mbuffer(size_t cnt) {
-    auto buffer = new de::MutableBuffer<PRec>(cnt/2, cnt);
-    for (int64_t i=0; i<cnt; i++) {
-        buffer->append({i, i});
-    }
-
-    return buffer;
-}
-
-template <de::KVPInterface R>
+template <de::RecordInterface R>
 static de::MutableBuffer<R> *create_test_mbuffer(size_t cnt)
 {
     auto buffer = new de::MutableBuffer<R>(cnt/2, cnt);
 
     R rec;
-    for (size_t i = 0; i < cnt; i++) {
-        rec.key = rand();
-        rec.value = rand();
+    if constexpr (de::KVPInterface<R>) {
+        for (size_t i = 0; i < cnt; i++) {
+            rec.key = rand();
+            rec.value = rand();
 
-        if constexpr (de::WeightedRecordInterface<R>) {
-            rec.weight = 1;
-        }
+            if constexpr (de::WeightedRecordInterface<R>) {
+                rec.weight = 1;
+            }
 
-        buffer->append(rec);
-    }
+            buffer->append(rec);
+        }
+    } else if constexpr (de::NDRecordInterface<R>) {
+        for (size_t i=0; i<cnt; i++) {
+            uint64_t a = rand();
+            uint64_t b = rand();
+            buffer->append({a, b});
+        }
+    } 
 
     return buffer;
 }
 
-template <de::KVPInterface R>
-static de::MutableBuffer<R> *create_sequential_mbuffer(decltype(R::key) start, decltype(R::key) stop)
+template <de::RecordInterface R>
+static de::MutableBuffer<R> *create_sequential_mbuffer(size_t start, size_t stop)
 {
     size_t cnt = stop - start;
     auto buffer = new de::MutableBuffer<R>(cnt/2, cnt);
 
     for (size_t i=start; i<stop; i++) {
         R rec;
-        rec.key = i;
-        rec.value = i;
+        if constexpr (de::KVPInterface<R>) {
+            rec.key = i;
+            rec.value = i;
+        } else if constexpr (de::NDRecordInterface<R>) {
+            rec = {i, i};
+        }
 
         if constexpr (de::WeightedRecordInterface<R>) {
             rec.weight = 1;
-- 
cgit v1.2.3


From aa1b40e9249afc03bf1a2f35de4cbf67c7f9b47e Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Fri, 9 Feb 2024 12:42:55 -0500
Subject: Framework: Fixed a bug where tagged deletes didn't release the epoch

---
 tests/include/dynamic_extension.h | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

(limited to 'tests/include')

diff --git a/tests/include/dynamic_extension.h b/tests/include/dynamic_extension.h
index f0f13dd..6e9b16c 100644
--- a/tests/include/dynamic_extension.h
+++ b/tests/include/dynamic_extension.h
@@ -22,14 +22,16 @@
  * should be included in the source file that includes this one, above the
  * include statement.
  */
-//#include "testing.h"
-//#include "framework/DynamicExtension.h"
-//#include "framework/scheduling/SerialScheduler.h"
-//#include "shard/ISAMTree.h"
-//#include "query/rangequery.h"
-//#include <check.h>
-//using namespace de;
-//typedef DynamicExtension<R, ISAMTree<R>, rq::Query<ISAMTree<R>, R>, LayoutPolicy::TEIRING, DeletePolicy::TAGGING, SerialScheduler> DE;
+/*
+#include "testing.h"
+#include "framework/DynamicExtension.h"
+#include "framework/scheduling/SerialScheduler.h"
+#include "shard/ISAMTree.h"
+#include "query/rangequery.h"
+#include <check.h>
+using namespace de;
+typedef DynamicExtension<R, ISAMTree<R>, rq::Query<ISAMTree<R>, R>, LayoutPolicy::TEIRING, DeletePolicy::TAGGING, SerialScheduler> DE;
+*/
 
 
 START_TEST(t_create)
-- 
cgit v1.2.3