From 7dcec45b0339cd8b9b2bfab8ce1fc4c6080ea958 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Mon, 5 Jun 2023 11:43:14 -0400 Subject: WSS tests + bugfixes --- CMakeLists.txt | 4 + include/shard/WSS.h | 16 +-- tests/wirs_tests.cpp | 26 ++-- tests/wss_tests.cpp | 346 +++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 376 insertions(+), 16 deletions(-) create mode 100644 tests/wss_tests.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c32700..367f2e9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -52,6 +52,10 @@ if (tests) add_executable(memisam_tests ${CMAKE_CURRENT_SOURCE_DIR}/tests/memisam_tests.cpp) target_link_libraries(memisam_tests PUBLIC gsl check subunit pthread) target_include_directories(memisam_tests PRIVATE include) + + add_executable(wss_tests ${CMAKE_CURRENT_SOURCE_DIR}/tests/wss_tests.cpp) + target_link_libraries(wss_tests PUBLIC gsl check subunit pthread) + target_include_directories(wss_tests PRIVATE include) endif() # Benchmark build instructions diff --git a/include/shard/WSS.h b/include/shard/WSS.h index 929f9df..bb7ee2a 100644 --- a/include/shard/WSS.h +++ b/include/shard/WSS.h @@ -74,7 +74,7 @@ public: friend class WSSQuery; WSS(MutableBuffer* buffer) - : m_reccnt(0), m_tombstone_cnt(0), m_total_weight(0) { + : m_reccnt(0), m_tombstone_cnt(0), m_total_weight(0), m_alias(nullptr), m_bf(nullptr) { size_t alloc_size = (buffer->get_record_count() * sizeof(Wrapped)) + (CACHELINE_SIZE - (buffer->get_record_count() * sizeof(Wrapped)) % CACHELINE_SIZE); assert(alloc_size % CACHELINE_SIZE == 0); @@ -117,12 +117,12 @@ public: } if (m_reccnt > 0) { - build_alias_structure(); + build_alias_structure(weights); } } WSS(WSS** shards, size_t len) - : m_reccnt(0), m_tombstone_cnt(0), m_total_weight(0) { + : m_reccnt(0), m_tombstone_cnt(0), m_total_weight(0), m_alias(nullptr), m_bf(nullptr) { std::vector>> cursors; cursors.reserve(len); @@ -186,7 +186,7 @@ public: ~WSS() { if (m_data) free(m_data); - if (m_alias) free(m_alias); + if (m_alias) delete m_alias; if (m_bf) delete m_bf; } @@ -253,13 +253,16 @@ private: } void build_alias_structure(std::vector &weights) { + // normalize the weights vector + std::vector norm_weights(weights.size()); + for (size_t i=0; i* m_data; @@ -298,7 +301,6 @@ public: for (size_t i = 0; i <= state->cutoff; i++) { auto rec = buffer->get_data() + i; weights.push_back(rec->rec.weight); - state->records.push_back(*rec); tot_weight += rec->rec.weight; } diff --git a/tests/wirs_tests.cpp b/tests/wirs_tests.cpp index 32cb863..eab94de 100644 --- a/tests/wirs_tests.cpp +++ b/tests/wirs_tests.cpp @@ -142,10 +142,14 @@ START_TEST(t_wirs_query) wirs_query_parms parms = {lower_key, upper_key, k}; parms.rng = gsl_rng_alloc(gsl_rng_mt19937); + size_t total_samples = 0; + for (size_t i=0; i<1000; i++) { auto state = WIRSQuery::get_query_state(shard, &parms); auto result = WIRSQuery::query(shard, state, &parms); + total_samples += result.size(); + for (size_t j=0; j::delete_query_state(state); } - ck_assert(roughly_equal(cnt[0] / 1000, (double) k/4.0, k, .05)); - ck_assert(roughly_equal(cnt[1] / 1000, (double) k/4.0, k, .05)); - ck_assert(roughly_equal(cnt[2] / 1000, (double) k/2.0, k, .05)); + ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05)); + ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05)); + ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05)); gsl_rng_free(parms.rng); delete shard; @@ -233,10 +237,14 @@ START_TEST(t_wirs_buffer_query_scan) wirs_query_parms parms = {lower_key, upper_key, k}; parms.rng = gsl_rng_alloc(gsl_rng_mt19937); + size_t total_samples = 0; + for (size_t i=0; i<1000; i++) { auto state = WIRSQuery::get_buffer_query_state(buffer, &parms); auto result = WIRSQuery::buffer_query(buffer, state, &parms); + total_samples += result.size(); + for (size_t j=0; j::delete_buffer_query_state(state); } - ck_assert(roughly_equal(cnt[0] / 1000, (double) k/4.0, k, .05)); - ck_assert(roughly_equal(cnt[1] / 1000, (double) k/4.0, k, .05)); - ck_assert(roughly_equal(cnt[2] / 1000, (double) k/2.0, k, .05)); + ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05)); + ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05)); + ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05)); gsl_rng_free(parms.rng); delete buffer; @@ -283,9 +291,9 @@ START_TEST(t_wirs_buffer_query_rejection) WIRSQuery::delete_buffer_query_state(state); } - ck_assert(roughly_equal(cnt[0] / total_samples, (double) k/4.0, k, .05)); - ck_assert(roughly_equal(cnt[1] / total_samples, (double) k/4.0, k, .05)); - ck_assert(roughly_equal(cnt[2] / total_samples, (double) k/2.0, k, .05)); + ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05)); + ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05)); + ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05)); gsl_rng_free(parms.rng); delete buffer; diff --git a/tests/wss_tests.cpp b/tests/wss_tests.cpp new file mode 100644 index 0000000..235f664 --- /dev/null +++ b/tests/wss_tests.cpp @@ -0,0 +1,346 @@ +/* + * tests/wss_tests.cpp + * + * Unit tests for WSS (Augmented B+Tree) shard + * + * Copyright (C) 2023 Douglas Rumbaugh + * Dong Xie + * + * All rights reserved. Published under the Modified BSD License. + * + */ + +#include "shard/WSS.h" +#include "testing.h" + +#include + +using namespace de; + +typedef WSS Shard; + +START_TEST(t_mbuffer_init) +{ + auto buffer = new MutableBuffer(1024, true, 1024); + for (uint64_t i = 512; i > 0; i--) { + uint32_t v = i; + buffer->append({i,v, 1}); + } + + for (uint64_t i = 1; i <= 256; ++i) { + uint32_t v = i; + buffer->append({i, v, 1}, true); + } + + for (uint64_t i = 257; i <= 512; ++i) { + uint32_t v = i + 1; + buffer->append({i, v, 1}); + } + + Shard* shard = new Shard(buffer); + ck_assert_uint_eq(shard->get_record_count(), 512); + + delete buffer; + delete shard; +} + + +START_TEST(t_wss_init) +{ + size_t n = 512; + auto mbuffer1 = create_test_mbuffer(n); + auto mbuffer2 = create_test_mbuffer(n); + auto mbuffer3 = create_test_mbuffer(n); + + auto shard1 = new Shard(mbuffer1); + auto shard2 = new Shard(mbuffer2); + auto shard3 = new Shard(mbuffer3); + + Shard* shards[3] = {shard1, shard2, shard3}; + auto shard4 = new Shard(shards, 3); + + ck_assert_int_eq(shard4->get_record_count(), n * 3); + ck_assert_int_eq(shard4->get_tombstone_count(), 0); + + size_t total_cnt = 0; + size_t shard1_idx = 0; + size_t shard2_idx = 0; + size_t shard3_idx = 0; + + for (size_t i = 0; i < shard4->get_record_count(); ++i) { + auto rec1 = shard1->get_record_at(shard1_idx); + auto rec2 = shard2->get_record_at(shard2_idx); + auto rec3 = shard3->get_record_at(shard3_idx); + + auto cur_rec = shard4->get_record_at(i); + + if (shard1_idx < n && cur_rec->rec == rec1->rec) { + ++shard1_idx; + } else if (shard2_idx < n && cur_rec->rec == rec2->rec) { + ++shard2_idx; + } else if (shard3_idx < n && cur_rec->rec == rec3->rec) { + ++shard3_idx; + } else { + assert(false); + } + } + + delete mbuffer1; + delete mbuffer2; + delete mbuffer3; + + delete shard1; + delete shard2; + delete shard3; + delete shard4; +} + + +START_TEST(t_full_cancelation) +{ + size_t n = 100; + auto buffer = create_double_seq_mbuffer(n, false); + auto buffer_ts = create_double_seq_mbuffer(n, true); + + Shard* shard = new Shard(buffer); + Shard* shard_ts = new Shard(buffer_ts); + + ck_assert_int_eq(shard->get_record_count(), n); + ck_assert_int_eq(shard->get_tombstone_count(), 0); + ck_assert_int_eq(shard_ts->get_record_count(), n); + ck_assert_int_eq(shard_ts->get_tombstone_count(), n); + + Shard* shards[] = {shard, shard_ts}; + + Shard* merged = new Shard(shards, 2); + + ck_assert_int_eq(merged->get_tombstone_count(), 0); + ck_assert_int_eq(merged->get_record_count(), 0); + + delete buffer; + delete buffer_ts; + delete shard; + delete shard_ts; + delete merged; +} +END_TEST + + +START_TEST(t_wss_query) +{ + size_t n=1000; + auto buffer = create_weighted_mbuffer(n); + + Shard* shard = new Shard(buffer); + + size_t k = 1000; + + size_t cnt[3] = {0}; + wss_query_parms parms = {k}; + parms.rng = gsl_rng_alloc(gsl_rng_mt19937); + + size_t total_samples = 0; + + for (size_t i=0; i<1000; i++) { + auto state = WSSQuery::get_query_state(shard, &parms); + auto result = WSSQuery::query(shard, state, &parms); + + total_samples += result.size(); + + for (size_t j=0; j::delete_query_state(state); + } + + ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05)); + ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05)); + ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05)); + + gsl_rng_free(parms.rng); + delete shard; + delete buffer; +} +END_TEST + + +template +std::vector strip_wrapping(std::vector> vec) { + std::vector out(vec.size()); + for (size_t i=0; i(n); + + Shard* shard = new Shard(buffer); + + uint64_t lower_key = 0; + uint64_t upper_key = 5; + + size_t k = 1000; + + size_t cnt[3] = {0}; + wss_query_parms parms = {k}; + parms.rng = gsl_rng_alloc(gsl_rng_mt19937); + + std::vector> results(2); + + for (size_t i=0; i<1000; i++) { + auto state1 = WSSQuery::get_query_state(shard, &parms); + results[0] = strip_wrapping(WSSQuery::query(shard, state1, &parms)); + + auto state2 = WSSQuery::get_query_state(shard, &parms); + results[1] = strip_wrapping(WSSQuery::query(shard, state2, &parms)); + + WSSQuery::delete_query_state(state1); + WSSQuery::delete_query_state(state2); + } + + auto merged = WSSQuery::merge(results); + + ck_assert_int_eq(merged.size(), 2*k); + for (size_t i=0; i(n); + + uint64_t lower_key = 0; + uint64_t upper_key = 5; + + size_t k = 1000; + + size_t cnt[3] = {0}; + wss_query_parms parms = {k}; + parms.rng = gsl_rng_alloc(gsl_rng_mt19937); + + size_t total_samples = 0; + + for (size_t i=0; i<1000; i++) { + auto state = WSSQuery::get_buffer_query_state(buffer, &parms); + auto result = WSSQuery::buffer_query(buffer, state, &parms); + total_samples += result.size(); + + for (size_t j=0; j::delete_buffer_query_state(state); + } + + ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05)); + ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05)); + ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05)); + + gsl_rng_free(parms.rng); + delete buffer; +} +END_TEST + + +START_TEST(t_wss_buffer_query_rejection) +{ + size_t n=1000; + auto buffer = create_weighted_mbuffer(n); + + uint64_t lower_key = 0; + uint64_t upper_key = 5; + + size_t k = 1000; + + size_t cnt[3] = {0}; + wss_query_parms parms = {k}; + parms.rng = gsl_rng_alloc(gsl_rng_mt19937); + + size_t total_samples = 0; + + for (size_t i=0; i<1000; i++) { + auto state = WSSQuery::get_buffer_query_state(buffer, &parms); + auto result = WSSQuery::buffer_query(buffer, state, &parms); + + total_samples += result.size(); + + for (size_t j=0; j::delete_buffer_query_state(state); + } + + ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .1)); + ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .1)); + ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .1)); + + gsl_rng_free(parms.rng); + delete buffer; +} +END_TEST + + +Suite *unit_testing() +{ + Suite *unit = suite_create("WSS Shard Unit Testing"); + + TCase *create = tcase_create("de::WSS constructor Testing"); + tcase_add_test(create, t_mbuffer_init); + tcase_add_test(create, t_wss_init); + tcase_set_timeout(create, 100); + suite_add_tcase(unit, create); + + + TCase *tombstone = tcase_create("de:WSS::tombstone cancellation Testing"); + tcase_add_test(tombstone, t_full_cancelation); + suite_add_tcase(unit, tombstone); + + + TCase *sampling = tcase_create("de:WSS::WSSQuery Testing"); + tcase_add_test(sampling, t_wss_query); + tcase_add_test(sampling, t_wss_query_merge); + tcase_add_test(sampling, t_wss_buffer_query_rejection); + tcase_add_test(sampling, t_wss_buffer_query_scan); + suite_add_tcase(unit, sampling); + + return unit; +} + + +int shard_unit_tests() +{ + int failed = 0; + Suite *unit = unit_testing(); + SRunner *unit_shardner = srunner_create(unit); + + srunner_run_all(unit_shardner, CK_NORMAL); + failed = srunner_ntests_failed(unit_shardner); + srunner_free(unit_shardner); + + return failed; +} + + +int main() +{ + int unit_failed = shard_unit_tests(); + + return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} -- cgit v1.2.3