WSS tests + bugfixes

4 files changed, 376 insertions, 16 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1c32700..367f2e9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -52,6 +52,10 @@ if (tests)
     add_executable(memisam_tests ${CMAKE_CURRENT_SOURCE_DIR}/tests/memisam_tests.cpp)
     target_link_libraries(memisam_tests PUBLIC gsl check subunit pthread)
     target_include_directories(memisam_tests PRIVATE include)
+
+    add_executable(wss_tests ${CMAKE_CURRENT_SOURCE_DIR}/tests/wss_tests.cpp)
+    target_link_libraries(wss_tests PUBLIC gsl check subunit pthread)
+    target_include_directories(wss_tests PRIVATE include)
 endif()
 
 # Benchmark build instructions
diff --git a/include/shard/WSS.h b/include/shard/WSS.h
index 929f9df..bb7ee2a 100644
--- a/include/shard/WSS.h
+++ b/include/shard/WSS.h
@@ -74,7 +74,7 @@ public:
     friend class WSSQuery<R, false>;
 
     WSS(MutableBuffer<R>* buffer)
-    : m_reccnt(0), m_tombstone_cnt(0), m_total_weight(0) {
+    : m_reccnt(0), m_tombstone_cnt(0), m_total_weight(0), m_alias(nullptr), m_bf(nullptr) {
 
         size_t alloc_size = (buffer->get_record_count() * sizeof(Wrapped<R>)) + (CACHELINE_SIZE - (buffer->get_record_count() * sizeof(Wrapped<R>)) % CACHELINE_SIZE);
         assert(alloc_size % CACHELINE_SIZE == 0);
@@ -117,12 +117,12 @@ public:
         }
 
         if (m_reccnt > 0) {
-            build_alias_structure();
+            build_alias_structure(weights);
         }
     }
 
     WSS(WSS** shards, size_t len)
-    : m_reccnt(0), m_tombstone_cnt(0), m_total_weight(0) {
+    : m_reccnt(0), m_tombstone_cnt(0), m_total_weight(0), m_alias(nullptr), m_bf(nullptr) {
         std::vector<Cursor<Wrapped<R>>> cursors;
         cursors.reserve(len);
 
@@ -186,7 +186,7 @@ public:
 
     ~WSS() {
         if (m_data) free(m_data);
-        if (m_alias) free(m_alias);
+        if (m_alias) delete m_alias;
         if (m_bf) delete m_bf;
 
     }
@@ -253,13 +253,16 @@ private:
     }
 
     void build_alias_structure(std::vector<W> &weights) {
+
         // normalize the weights vector
+        std::vector<double> norm_weights(weights.size());
+
         for (size_t i=0; i<weights.size(); i++) {
-            weights[i] = weights[i] / (double) m_total_weight;
+            norm_weights[i] = (double) weights[i] / (double) m_total_weight;
         }
 
         // build the alias structure
-        m_alias = new Alias(weights);
+        m_alias = new Alias(norm_weights);
     }
 
     Wrapped<R>* m_data;
@@ -298,7 +301,6 @@ public:
         for (size_t i = 0; i <= state->cutoff; i++) {
             auto rec = buffer->get_data() + i;
             weights.push_back(rec->rec.weight);
-            state->records.push_back(*rec);
             tot_weight += rec->rec.weight;
         }
 
diff --git a/tests/wirs_tests.cpp b/tests/wirs_tests.cpp
index 32cb863..eab94de 100644
--- a/tests/wirs_tests.cpp
+++ b/tests/wirs_tests.cpp
@@ -142,10 +142,14 @@ START_TEST(t_wirs_query)
     wirs_query_parms<WRec> parms = {lower_key, upper_key, k};
     parms.rng = gsl_rng_alloc(gsl_rng_mt19937);
 
+    size_t total_samples = 0;
+
     for (size_t i=0; i<1000; i++) {
         auto state = WIRSQuery<WRec>::get_query_state(shard, &parms);
         auto result = WIRSQuery<WRec>::query(shard, state, &parms);
 
+        total_samples += result.size();
+
         for (size_t j=0; j<result.size(); j++) {
             cnt[result[j].rec.key - 1]++;
         }
@@ -153,9 +157,9 @@ START_TEST(t_wirs_query)
         WIRSQuery<WRec>::delete_query_state(state);
     }
 
-    ck_assert(roughly_equal(cnt[0] / 1000, (double) k/4.0, k, .05));
-    ck_assert(roughly_equal(cnt[1] / 1000, (double) k/4.0, k, .05));
-    ck_assert(roughly_equal(cnt[2] / 1000, (double) k/2.0, k, .05));
+    ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05));
+    ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05));
+    ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05));
 
     gsl_rng_free(parms.rng);
     delete shard;
@@ -233,10 +237,14 @@ START_TEST(t_wirs_buffer_query_scan)
     wirs_query_parms<WRec> parms = {lower_key, upper_key, k};
     parms.rng = gsl_rng_alloc(gsl_rng_mt19937);
 
+    size_t total_samples = 0;
+
     for (size_t i=0; i<1000; i++) {
         auto state = WIRSQuery<WRec, false>::get_buffer_query_state(buffer, &parms);
         auto result = WIRSQuery<WRec, false>::buffer_query(buffer, state, &parms);
 
+        total_samples += result.size();
+
         for (size_t j=0; j<result.size(); j++) {
             cnt[result[j].rec.key - 1]++;
         }
@@ -244,9 +252,9 @@ START_TEST(t_wirs_buffer_query_scan)
         WIRSQuery<WRec, false>::delete_buffer_query_state(state);
     }
 
-    ck_assert(roughly_equal(cnt[0] / 1000, (double) k/4.0, k, .05));
-    ck_assert(roughly_equal(cnt[1] / 1000, (double) k/4.0, k, .05));
-    ck_assert(roughly_equal(cnt[2] / 1000, (double) k/2.0, k, .05));
+    ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05));
+    ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05));
+    ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05));
 
     gsl_rng_free(parms.rng);
     delete buffer;
@@ -283,9 +291,9 @@ START_TEST(t_wirs_buffer_query_rejection)
         WIRSQuery<WRec>::delete_buffer_query_state(state);
     }
 
-    ck_assert(roughly_equal(cnt[0] / total_samples, (double) k/4.0, k, .05));
-    ck_assert(roughly_equal(cnt[1] / total_samples, (double) k/4.0, k, .05));
-    ck_assert(roughly_equal(cnt[2] / total_samples, (double) k/2.0, k, .05));
+    ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05));
+    ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05));
+    ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05));
 
     gsl_rng_free(parms.rng);
     delete buffer;
diff --git a/tests/wss_tests.cpp b/tests/wss_tests.cpp
new file mode 100644
index 0000000..235f664
--- /dev/null
+++ b/tests/wss_tests.cpp
@@ -0,0 +1,346 @@
+/*
+ * tests/wss_tests.cpp
+ *
+ * Unit tests for WSS (Augmented B+Tree) shard
+ *
+ * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> 
+ *                    Dong Xie <dongx@psu.edu>
+ *
+ * All rights reserved. Published under the Modified BSD License.
+ *
+ */
+
+#include "shard/WSS.h"
+#include "testing.h"
+
+#include <check.h>
+
+using namespace de;
+
+typedef WSS<WRec> Shard;
+
+START_TEST(t_mbuffer_init)
+{
+    auto buffer = new MutableBuffer<WRec>(1024, true, 1024);
+    for (uint64_t i = 512; i > 0; i--) {
+        uint32_t v = i;
+        buffer->append({i,v, 1});
+    }
+    
+    for (uint64_t i = 1; i <= 256; ++i) {
+        uint32_t v = i;
+        buffer->append({i, v, 1}, true);
+    }
+
+    for (uint64_t i = 257; i <= 512; ++i) {
+        uint32_t v = i + 1;
+        buffer->append({i, v, 1});
+    }
+
+    Shard* shard = new Shard(buffer);
+    ck_assert_uint_eq(shard->get_record_count(), 512);
+
+    delete buffer;
+    delete shard;
+}
+
+
+START_TEST(t_wss_init)
+{
+    size_t n = 512;
+    auto mbuffer1 = create_test_mbuffer<WRec>(n);
+    auto mbuffer2 = create_test_mbuffer<WRec>(n);
+    auto mbuffer3 = create_test_mbuffer<WRec>(n);
+
+    auto shard1 = new Shard(mbuffer1);
+    auto shard2 = new Shard(mbuffer2);
+    auto shard3 = new Shard(mbuffer3);
+
+    Shard* shards[3] = {shard1, shard2, shard3};
+    auto shard4 = new Shard(shards, 3);
+
+    ck_assert_int_eq(shard4->get_record_count(), n * 3);
+    ck_assert_int_eq(shard4->get_tombstone_count(), 0);
+
+    size_t total_cnt = 0;
+    size_t shard1_idx = 0;
+    size_t shard2_idx = 0;
+    size_t shard3_idx = 0;
+
+    for (size_t i = 0; i < shard4->get_record_count(); ++i) {
+        auto rec1 = shard1->get_record_at(shard1_idx);
+        auto rec2 = shard2->get_record_at(shard2_idx);
+        auto rec3 = shard3->get_record_at(shard3_idx);
+
+        auto cur_rec = shard4->get_record_at(i);
+
+        if (shard1_idx < n && cur_rec->rec == rec1->rec) {
+            ++shard1_idx;
+        } else if (shard2_idx < n && cur_rec->rec == rec2->rec) {
+            ++shard2_idx;
+        } else if (shard3_idx < n && cur_rec->rec == rec3->rec) {
+            ++shard3_idx;
+        } else {
+           assert(false);
+        }
+    }
+
+    delete mbuffer1;
+    delete mbuffer2;
+    delete mbuffer3;
+
+    delete shard1;
+    delete shard2;
+    delete shard3;
+    delete shard4;
+}
+
+
+START_TEST(t_full_cancelation)
+{
+    size_t n = 100;
+    auto buffer = create_double_seq_mbuffer<WRec>(n, false);
+    auto buffer_ts = create_double_seq_mbuffer<WRec>(n, true);
+
+    Shard* shard = new Shard(buffer);
+    Shard* shard_ts = new Shard(buffer_ts);
+
+    ck_assert_int_eq(shard->get_record_count(), n);
+    ck_assert_int_eq(shard->get_tombstone_count(), 0);
+    ck_assert_int_eq(shard_ts->get_record_count(), n);
+    ck_assert_int_eq(shard_ts->get_tombstone_count(), n);
+
+    Shard* shards[] = {shard, shard_ts};
+
+    Shard* merged = new Shard(shards, 2);
+
+    ck_assert_int_eq(merged->get_tombstone_count(), 0);
+    ck_assert_int_eq(merged->get_record_count(), 0);
+
+    delete buffer;
+    delete buffer_ts;
+    delete shard;
+    delete shard_ts;
+    delete merged;
+}
+END_TEST
+
+
+START_TEST(t_wss_query)
+{
+    size_t n=1000;
+    auto buffer = create_weighted_mbuffer<WRec>(n);
+
+    Shard* shard = new Shard(buffer);
+
+    size_t k = 1000;
+
+    size_t cnt[3] = {0};
+    wss_query_parms<WRec> parms = {k};
+    parms.rng = gsl_rng_alloc(gsl_rng_mt19937);
+
+    size_t total_samples = 0;
+
+    for (size_t i=0; i<1000; i++) {
+        auto state = WSSQuery<WRec>::get_query_state(shard, &parms);
+        auto result = WSSQuery<WRec>::query(shard, state, &parms);
+
+        total_samples += result.size();
+
+        for (size_t j=0; j<result.size(); j++) {
+            cnt[result[j].rec.key - 1]++;
+        }
+
+        WSSQuery<WRec>::delete_query_state(state);
+    }
+
+    ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05));
+    ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05));
+    ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05));
+
+    gsl_rng_free(parms.rng);
+    delete shard;
+    delete buffer;
+}
+END_TEST
+
+
+template <RecordInterface R> 
+std::vector<R> strip_wrapping(std::vector<Wrapped<R>> vec) {
+    std::vector<R> out(vec.size());
+    for (size_t i=0; i<vec.size(); i++) {
+        out[i] = vec[i].rec;
+    }
+
+    return out;
+}
+
+
+START_TEST(t_wss_query_merge)
+{
+    size_t n=1000;
+    auto buffer = create_weighted_mbuffer<WRec>(n);
+
+    Shard* shard = new Shard(buffer);
+
+    uint64_t lower_key = 0;
+    uint64_t upper_key = 5;
+
+    size_t k = 1000;
+
+    size_t cnt[3] = {0};
+    wss_query_parms<WRec> parms = {k};
+    parms.rng = gsl_rng_alloc(gsl_rng_mt19937);
+
+    std::vector<std::vector<WRec>> results(2);
+
+    for (size_t i=0; i<1000; i++) {
+        auto state1 = WSSQuery<WRec>::get_query_state(shard, &parms);
+        results[0] = strip_wrapping(WSSQuery<WRec>::query(shard, state1, &parms));
+
+        auto state2 = WSSQuery<WRec>::get_query_state(shard, &parms);
+        results[1] = strip_wrapping(WSSQuery<WRec>::query(shard, state2, &parms));
+
+        WSSQuery<WRec>::delete_query_state(state1);
+        WSSQuery<WRec>::delete_query_state(state2);
+    }
+
+    auto merged = WSSQuery<WRec>::merge(results);
+
+    ck_assert_int_eq(merged.size(), 2*k);
+    for (size_t i=0; i<merged.size(); i++) {
+        ck_assert_int_ge(merged[i].key, lower_key);
+        ck_assert_int_le(merged[i].key, upper_key);
+    }
+
+    gsl_rng_free(parms.rng);
+    delete shard;
+    delete buffer;
+}
+END_TEST
+
+
+START_TEST(t_wss_buffer_query_scan)
+{
+    size_t n=1000;
+    auto buffer = create_weighted_mbuffer<WRec>(n);
+
+    uint64_t lower_key = 0;
+    uint64_t upper_key = 5;
+
+    size_t k = 1000;
+
+    size_t cnt[3] = {0};
+    wss_query_parms<WRec> parms = {k};
+    parms.rng = gsl_rng_alloc(gsl_rng_mt19937);
+
+    size_t total_samples = 0;
+
+    for (size_t i=0; i<1000; i++) {
+        auto state = WSSQuery<WRec, false>::get_buffer_query_state(buffer, &parms);
+        auto result = WSSQuery<WRec, false>::buffer_query(buffer, state, &parms);
+        total_samples += result.size();
+
+        for (size_t j=0; j<result.size(); j++) {
+            cnt[result[j].rec.key - 1]++;
+        }
+
+        WSSQuery<WRec, false>::delete_buffer_query_state(state);
+    }
+
+    ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05));
+    ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05));
+    ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05));
+
+    gsl_rng_free(parms.rng);
+    delete buffer;
+}
+END_TEST
+
+
+START_TEST(t_wss_buffer_query_rejection)
+{
+    size_t n=1000;
+    auto buffer = create_weighted_mbuffer<WRec>(n);
+
+    uint64_t lower_key = 0;
+    uint64_t upper_key = 5;
+
+    size_t k = 1000;
+
+    size_t cnt[3] = {0};
+    wss_query_parms<WRec> parms = {k};
+    parms.rng = gsl_rng_alloc(gsl_rng_mt19937);
+
+    size_t total_samples = 0;
+
+    for (size_t i=0; i<1000; i++) {
+        auto state = WSSQuery<WRec>::get_buffer_query_state(buffer, &parms);
+        auto result = WSSQuery<WRec>::buffer_query(buffer, state, &parms);
+
+        total_samples += result.size();
+
+        for (size_t j=0; j<result.size(); j++) {
+            cnt[result[j].rec.key - 1]++;
+        }
+
+        WSSQuery<WRec>::delete_buffer_query_state(state);
+    }
+
+    ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .1));
+    ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .1));
+    ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .1));
+
+    gsl_rng_free(parms.rng);
+    delete buffer;
+}
+END_TEST
+
+
+Suite *unit_testing()
+{
+    Suite *unit = suite_create("WSS Shard Unit Testing");
+
+    TCase *create = tcase_create("de::WSS constructor Testing");
+    tcase_add_test(create, t_mbuffer_init);
+    tcase_add_test(create, t_wss_init);
+    tcase_set_timeout(create, 100);
+    suite_add_tcase(unit, create);
+
+
+    TCase *tombstone = tcase_create("de:WSS::tombstone cancellation Testing");
+    tcase_add_test(tombstone, t_full_cancelation);
+    suite_add_tcase(unit, tombstone);
+
+
+    TCase *sampling = tcase_create("de:WSS::WSSQuery Testing");
+    tcase_add_test(sampling, t_wss_query);
+    tcase_add_test(sampling, t_wss_query_merge);
+    tcase_add_test(sampling, t_wss_buffer_query_rejection);
+    tcase_add_test(sampling, t_wss_buffer_query_scan);
+    suite_add_tcase(unit, sampling);
+
+    return unit;
+}
+
+
+int shard_unit_tests()
+{
+    int failed = 0;
+    Suite *unit = unit_testing();
+    SRunner *unit_shardner = srunner_create(unit);
+
+    srunner_run_all(unit_shardner, CK_NORMAL);
+    failed = srunner_ntests_failed(unit_shardner);
+    srunner_free(unit_shardner);
+
+    return failed;
+}
+
+
+int main() 
+{
+    int unit_failed = shard_unit_tests();
+
+    return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE;
+}