diff options
| author | Douglas B. Rumbaugh <dbr4@psu.edu> | 2024-05-14 16:31:05 -0400 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-05-14 16:31:05 -0400 |
| commit | 47916da2ba5ed5bee2dda3cbcc58d39e1e931bfc (patch) | |
| tree | ee5613ce182b2c9caa228d3abeb65dc27fef2db3 /tests/include | |
| parent | 4a834497d5f82c817d634925250158d85ca825c2 (diff) | |
| parent | 8643fe194dec05b4e3f3ea31e162ac0b2b00e162 (diff) | |
| download | dynamic-extension-47916da2ba5ed5bee2dda3cbcc58d39e1e931bfc.tar.gz | |
Merge pull request #4 from dbrumbaugh/master
Updates for VLDB revision
Diffstat (limited to 'tests/include')
| -rw-r--r-- | tests/include/concurrent_extension.h | 15 | ||||
| -rw-r--r-- | tests/include/pointlookup.h | 113 | ||||
| -rw-r--r-- | tests/include/rangecount.h | 21 | ||||
| -rw-r--r-- | tests/include/rangequery.h | 21 | ||||
| -rw-r--r-- | tests/include/shard_standard.h | 16 | ||||
| -rw-r--r-- | tests/include/shard_string.h | 165 | ||||
| -rw-r--r-- | tests/include/testing.h | 111 | ||||
| -rw-r--r-- | tests/include/wirs.h | 3 |
8 files changed, 373 insertions, 92 deletions
diff --git a/tests/include/concurrent_extension.h b/tests/include/concurrent_extension.h index 0993fac..927a094 100644 --- a/tests/include/concurrent_extension.h +++ b/tests/include/concurrent_extension.h @@ -97,8 +97,7 @@ START_TEST(t_insert_with_mem_merges) R r = {key, val}; for (size_t i=0; i<1000; i++) { ck_assert_int_eq(test_de->insert(r), 1); - r.key++; - r.value++; + r = R{r.key + 1, r.value + 1}; } ck_assert_int_eq(test_de->get_record_count(), 1000); @@ -114,8 +113,7 @@ START_TEST(t_insert_with_mem_merges) size_t cnt = 0; do { if (test_de->insert(r)) { - r.key++; - r.value++; + r = R{r.key + 1, r.value + 1}; cnt++; ck_assert_int_eq(test_de->get_record_count(), cnt + 1000); } else { @@ -170,10 +168,8 @@ START_TEST(t_range_query) p.lower_bound = lower_key; p.upper_bound = upper_key; - //fprintf(stderr, "query start\n"); auto result = test_de->query(&p); auto r = result.get(); - //fprintf(stderr, "query stop\n"); std::sort(r.begin(), r.end()); ck_assert_int_eq(r.size(), 251); @@ -349,12 +345,7 @@ START_TEST(t_static_structure) uint64_t prev_key = 0; for (size_t i=0; i<flat->get_record_count(); i++) { auto k = flat->get_record_at(i)->rec.key; - if (flat->get_record_at(i)->is_tombstone()) { - fprintf(stderr, "%ld %ld %ld\n", flat->get_record_at(i-1)->rec.key, - flat->get_record_at(i)->rec.key, - flat->get_record_at(i+1)->rec.key); - } - // ck_assert(!flat->get_record_at(i)->is_tombstone()); + //ck_assert(!flat->get_record_at(i)->is_tombstone()); ck_assert_int_ge(k, prev_key); prev_key = k; } diff --git a/tests/include/pointlookup.h b/tests/include/pointlookup.h new file mode 100644 index 0000000..71a1099 --- /dev/null +++ b/tests/include/pointlookup.h @@ -0,0 +1,113 @@ +/* + * tests/include/pointlookup.h + * + * Standardized unit tests for point lookups against supporting + * shard types (must be unique for the moment) + * + * Copyright (C) 2024 Douglas Rumbaugh <drumbaugh@psu.edu> + * + * Distributed under the Modified BSD License. + * + * WARNING: This file must be included in the main unit test set + * after the definition of an appropriate Shard and R + * type. In particular, R needs to implement the key-value + * pair interface and Shard needs to support lower_bound. + * For other types of record and shard, you'll need to + * use a different set of unit tests. + */ +#pragma once + +/* + * Uncomment these lines temporarily to remove errors in this file + * temporarily for development purposes. They should be removed prior + * to building, to ensure no duplicate definitions. These includes/defines + * should be included in the source file that includes this one, above the + * include statement. + */ + +//#include "shard/FSTrie.h" +#include "query/pointlookup.h" +#include "testing.h" + +#include <check.h> + +using namespace de; +//typedef StringRec R; +//typedef FSTrie<R> Shard; + +START_TEST(t_point_lookup_query) +{ + auto buffer = create_test_mbuffer<R>(1000); + auto shard = Shard(buffer->get_buffer_view()); + + { + auto bv = buffer->get_buffer_view(); + for (size_t i=0; i<bv.get_record_count(); i++) { + auto key = bv.get(i)->rec.key; + + pl::Parms<R> parms = {key}; + auto state = pl::Query<R, Shard>::get_query_state(&shard, &parms); + auto result = pl::Query<R, Shard>::query(&shard, state, &parms); + pl::Query<R, Shard>::delete_query_state(state); + + ck_assert_int_eq(result.size(), 1); + //ck_assert_str_eq(result[0].rec.key, key); + //ck_assert_int_eq(result[0].rec.value, bv.get(i)->rec.value); + } + + /* point lookup miss; result size should be 0 */ + const char *c = "computer"; + pl::Parms<R> parms = {c}; + + auto state = pl::Query<R, Shard>::get_query_state(&shard, &parms); + auto result = pl::Query<R, Shard>::query(&shard, state, &parms); + pl::Query<R, Shard>::delete_query_state(state); + + ck_assert_int_eq(result.size(), 0); + } + + delete buffer; +} +END_TEST + + +START_TEST(t_buffer_point_lookup) +{ + + auto buffer = create_test_mbuffer<R>(1000); + { + auto view = buffer->get_buffer_view(); + for (int i=view.get_record_count()-1; i>=0; i--) { + pl::Parms<R> parms = {view.get(i)->rec.key}; + + auto state = pl::Query<R, Shard>::get_buffer_query_state(&view, &parms); + auto result = pl::Query<R, Shard>::buffer_query(state, &parms); + pl::Query<R, Shard>::delete_buffer_query_state(state); + + ck_assert_int_eq(result.size(), 1); + //ck_assert_str_eq(result[0].rec.key, view.get(i)->rec.key); + //ck_assert_int_eq(result[0].rec.value, view.get(i)->rec.value); + } + + /* point lookup miss; result size should be 0 */ + const char *c = "computer"; + pl::Parms<R> parms = {c}; + + auto state = pl::Query<R, Shard>::get_buffer_query_state(&view, &parms); + auto result = pl::Query<R, Shard>::buffer_query(state, &parms); + pl::Query<R, Shard>::delete_buffer_query_state(state); + + ck_assert_int_eq(result.size(), 0); + } + + delete buffer; +} +END_TEST + + +static void inject_pointlookup_tests(Suite *suite) { + TCase *point_lookup_query = tcase_create("Point Lookup Testing"); + tcase_add_test(point_lookup_query, t_point_lookup_query); + tcase_add_test(point_lookup_query, t_buffer_point_lookup); + suite_add_tcase(suite, point_lookup_query); +} diff --git a/tests/include/rangecount.h b/tests/include/rangecount.h index fdd66d9..1951221 100644 --- a/tests/include/rangecount.h +++ b/tests/include/rangecount.h @@ -40,9 +40,7 @@ START_TEST(t_range_count) auto buffer = create_sequential_mbuffer<R>(100, 1000); auto shard = Shard(buffer->get_buffer_view()); - rc::Parms<R> parms; - parms.lower_bound = 300; - parms.upper_bound = 500; + rc::Parms<R> parms = {300, 500}; auto state = rc::Query<R, Shard>::get_query_state(&shard, &parms); auto result = rc::Query<R, Shard>::query(&shard, state, &parms); @@ -60,9 +58,7 @@ START_TEST(t_buffer_range_count) { auto buffer = create_sequential_mbuffer<R>(100, 1000); - rc::Parms<R> parms; - parms.lower_bound = 300; - parms.upper_bound = 500; + rc::Parms<R> parms = {300, 500}; { auto view = buffer->get_buffer_view(); @@ -87,9 +83,7 @@ START_TEST(t_range_count_merge) auto shard1 = Shard(buffer1->get_buffer_view()); auto shard2 = Shard(buffer2->get_buffer_view()); - rc::Parms<R> parms; - parms.lower_bound = 150; - parms.upper_bound = 500; + rc::Parms<R> parms = {150, 500}; size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200; @@ -106,7 +100,8 @@ START_TEST(t_range_count_merge) ck_assert_int_eq(results[0].size(), 1); ck_assert_int_eq(results[1].size(), 1); - auto result = rc::Query<R, Shard>::merge(results, nullptr); + std::vector<R> result; + rc::Query<R, Shard>::merge(results, nullptr, result); ck_assert_int_eq(result[0].key, result_size); @@ -128,10 +123,8 @@ START_TEST(t_lower_bound) auto merged = Shard(shards); - for (size_t i=100; i<1000; i++) { - R r; - r.key = i; - r.value = i; + for (uint32_t i=100; i<1000; i++) { + R r = R{i, i}; auto idx = merged.get_lower_bound(i); diff --git a/tests/include/rangequery.h b/tests/include/rangequery.h index a8a73f7..f90e107 100644 --- a/tests/include/rangequery.h +++ b/tests/include/rangequery.h @@ -39,9 +39,7 @@ START_TEST(t_range_query) auto buffer = create_sequential_mbuffer<R>(100, 1000); auto shard = Shard(buffer->get_buffer_view()); - rq::Parms<R> parms; - parms.lower_bound = 300; - parms.upper_bound = 500; + rq::Parms<R> parms = {300, 500}; auto state = rq::Query<R, Shard>::get_query_state(&shard, &parms); auto result = rq::Query<R, Shard>::query(&shard, state, &parms); @@ -62,9 +60,7 @@ START_TEST(t_buffer_range_query) { auto buffer = create_sequential_mbuffer<R>(100, 1000); - rq::Parms<R> parms; - parms.lower_bound = 300; - parms.upper_bound = 500; + rq::Parms<R> parms = {300, 500}; { auto view = buffer->get_buffer_view(); @@ -92,9 +88,7 @@ START_TEST(t_range_query_merge) auto shard1 = Shard(buffer1->get_buffer_view()); auto shard2 = Shard(buffer2->get_buffer_view()); - rq::Parms<R> parms; - parms.lower_bound = 150; - parms.upper_bound = 500; + rq::Parms<R> parms = {150, 500}; size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200; @@ -119,7 +113,8 @@ START_TEST(t_range_query_merge) } } - auto result = rq::Query<R, Shard>::merge(proc_results, nullptr); + std::vector<R> result; + rq::Query<R, Shard>::merge(proc_results, nullptr, result); std::sort(result.begin(), result.end()); ck_assert_int_eq(result.size(), result_size); @@ -149,10 +144,8 @@ START_TEST(t_lower_bound) auto merged = Shard(shards); - for (size_t i=100; i<1000; i++) { - R r; - r.key = i; - r.value = i; + for (uint32_t i=100; i<1000; i++) { + R r = R{i, i}; auto idx = merged.get_lower_bound(i); diff --git a/tests/include/shard_standard.h b/tests/include/shard_standard.h index 7d17dcb..2809d74 100644 --- a/tests/include/shard_standard.h +++ b/tests/include/shard_standard.h @@ -37,17 +37,17 @@ START_TEST(t_mbuffer_init) auto buffer = new MutableBuffer<R>(512, 1024); for (uint64_t i = 512; i > 0; i--) { uint32_t v = i; - buffer->append({i, v, 1}); + buffer->append({i, v}); } for (uint64_t i = 1; i <= 256; ++i) { uint32_t v = i; - buffer->append({i, v, 1}, true); + buffer->append({i, v}, true); } for (uint64_t i = 257; i <= 512; ++i) { uint32_t v = i + 1; - buffer->append({i, v, 1}); + buffer->append({i, v}); } Shard* shard = new Shard(buffer->get_buffer_view()); @@ -150,10 +150,8 @@ START_TEST(t_point_lookup) auto view = buffer->get_buffer_view(); for (size_t i=0; i<n; i++) { - R r; auto rec = view.get(i); - r.key = rec->rec.key; - r.value = rec->rec.value; + R r = {rec->rec.key, rec->rec.value}; auto result = isam.point_lookup(r); ck_assert_ptr_nonnull(result); @@ -174,10 +172,8 @@ START_TEST(t_point_lookup_miss) auto buffer = create_double_seq_mbuffer<R>(n, false); auto isam = Shard(buffer->get_buffer_view()); - for (size_t i=n + 100; i<2*n; i++) { - R r; - r.key = i; - r.value = i; + for (uint32_t i=n + 100; i<2*n; i++) { + R r = R{i, i}; auto result = isam.point_lookup(r); ck_assert_ptr_null(result); diff --git a/tests/include/shard_string.h b/tests/include/shard_string.h new file mode 100644 index 0000000..881f41a --- /dev/null +++ b/tests/include/shard_string.h @@ -0,0 +1,165 @@ +/* + * tests/include/shard_string.h + * + * Standardized unit tests for Shard objects with string keys + * + * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> + * + * Distributed under the Modified BSD License. + * + * WARNING: This file must be included in the main unit test set + * after the definition of an appropriate Shard and R + * type. In particular, R needs to implement the key-value + * pair interface. For other types of record, you'll need to + * use a different set of unit tests. + */ +#pragma once + +/* + * Uncomment these lines temporarily to remove errors in this file + * temporarily for development purposes. They should be removed prior + * to building, to ensure no duplicate definitions. These includes/defines + * should be included in the source file that includes this one, above the + * include statement. + */ +#include "shard/FSTrie.h" +#include "testing.h" +#include <check.h> +using namespace de; +//typedef StringRec R; +//typedef FSTrie<R> Shard; + +START_TEST(t_mbuffer_init) +{ + + auto recs = read_string_data(kjv_wordlist, 1024); + + auto buffer = new MutableBuffer<R>(512, 1024); + + for (uint64_t i = 0; i < 512; i++) { + buffer->append(recs[i]); + } + + for (uint64_t i = 0; i < 256; ++i) { + buffer->delete_record(recs[i]); + } + + for (uint64_t i = 512; i < 768; ++i) { + buffer->append(recs[i]); + } + + Shard* shard = new Shard(buffer->get_buffer_view()); + ck_assert_uint_eq(shard->get_record_count(), 512); + + delete buffer; + delete shard; +} + + +START_TEST(t_shard_init) +{ + size_t n = 2048; + auto mbuffer1 = create_test_mbuffer<R>(n); + auto mbuffer2 = create_test_mbuffer<R>(n); + auto mbuffer3 = create_test_mbuffer<R>(n); + + auto shard1 = new Shard(mbuffer1->get_buffer_view()); + auto shard2 = new Shard(mbuffer2->get_buffer_view()); + auto shard3 = new Shard(mbuffer3->get_buffer_view()); + + std::vector<Shard*> shards = {shard1, shard2, shard3}; + auto shard4 = new Shard(shards); + + ck_assert_int_eq(shard4->get_record_count(), n * 3); + ck_assert_int_eq(shard4->get_tombstone_count(), 0); + + size_t total_cnt = 0; + size_t shard1_idx = 0; + size_t shard2_idx = 0; + size_t shard3_idx = 0; + + for (size_t i = 0; i < shard4->get_record_count(); ++i) { + auto rec1 = shard1->get_record_at(shard1_idx); + auto rec2 = shard2->get_record_at(shard2_idx); + auto rec3 = shard3->get_record_at(shard3_idx); + + auto cur_rec = shard4->get_record_at(i); + + if (shard1_idx < n && cur_rec->rec == rec1->rec) { + ++shard1_idx; + } else if (shard2_idx < n && cur_rec->rec == rec2->rec) { + ++shard2_idx; + } else if (shard3_idx < n && cur_rec->rec == rec3->rec) { + ++shard3_idx; + } else { + assert(false); + } + } + + delete mbuffer1; + delete mbuffer2; + delete mbuffer3; + + delete shard1; + delete shard2; + delete shard3; + delete shard4; +} + +START_TEST(t_point_lookup) +{ + size_t n = 10000; + + auto buffer = create_test_mbuffer<R>(n); + auto shard = Shard(buffer->get_buffer_view()); + + { + auto view = buffer->get_buffer_view(); + + for (size_t i=0; i<n; i++) { + auto rec = view.get(i); + R r = rec->rec; + + auto result = shard.point_lookup(r); + ck_assert_ptr_nonnull(result); + //ck_assert_str_eq(result->rec.key, r.key); + //ck_assert_int_eq(result->rec.value, r.value); + //fprintf(stderr, "%ld\n", i); + } + } + + delete buffer; +} +END_TEST + + +START_TEST(t_point_lookup_miss) +{ + size_t n = 10000; + + auto buffer = create_test_mbuffer<R>(n); + auto shard = Shard(buffer->get_buffer_view()); + + for (size_t i=n + 100; i<2*n; i++) { + const char *c = "computer"; + R r = {c, 1234, 8}; + + auto result = shard.point_lookup(r); + ck_assert_ptr_null(result); + } + + delete buffer; +} + +static void inject_shard_tests(Suite *suite) { + TCase *create = tcase_create("Shard constructor Testing"); + tcase_add_test(create, t_mbuffer_init); + tcase_add_test(create, t_shard_init); + tcase_set_timeout(create, 100); + suite_add_tcase(suite, create); + + TCase *pointlookup = tcase_create("Shard point lookup Testing"); + tcase_add_test(pointlookup, t_point_lookup); + tcase_add_test(pointlookup, t_point_lookup_miss); + suite_add_tcase(suite, pointlookup); +} diff --git a/tests/include/testing.h b/tests/include/testing.h index f935b53..d0bff2d 100644 --- a/tests/include/testing.h +++ b/tests/include/testing.h @@ -15,6 +15,8 @@ #include <unistd.h> #include <fcntl.h> +#include <fstream> +#include <sstream> #include "util/types.h" #include "psu-util/alignment.h" @@ -25,10 +27,49 @@ typedef de::WeightedRecord<uint64_t, uint32_t, uint64_t> WRec; typedef de::Record<uint64_t, uint32_t> Rec; typedef de::EuclidPoint<uint64_t> PRec; +typedef de::Record<const char*, uint64_t> StringRec; + +static std::string kjv_wordlist = "tests/data/kjv-wordlist.txt"; +static std::string summa_wordlist = "tests/data/summa-wordlist.txt"; + +static std::vector<std::unique_ptr<char[]>> string_data; + +static std::vector<StringRec> read_string_data(std::string fname, size_t n) { + std::vector<StringRec> vec; + vec.reserve(n); + string_data.reserve(n); + + std::fstream file; + file.open(fname, std::ios::in); + + for (size_t i=0; i<n; i++) { + std::string line; + if (!std::getline(file, line, '\n')) break; + + std::stringstream ls(line); + std::string field; + + std::getline(ls, field, '\t'); + auto val = atol(field.c_str()); + std::getline(ls, field, '\n'); + + char *c = strdup(field.c_str()); + + string_data.push_back(std::unique_ptr<char[]>(c)); + + StringRec r(string_data[string_data.size() -1].get(), val, field.size()); + + vec.push_back(r); + } + + return vec; +} + + template <de::RecordInterface R> std::vector<R> strip_wrapping(std::vector<de::Wrapped<R>> vec) { std::vector<R> out(vec.size()); - for (size_t i=0; i<vec.size(); i++) { + for (uint32_t i=0; i<vec.size(); i++) { out[i] = vec[i].rec; } @@ -81,23 +122,24 @@ static de::MutableBuffer<R> *create_test_mbuffer(size_t cnt) { auto buffer = new de::MutableBuffer<R>(cnt/2, cnt); - R rec; - if constexpr (de::KVPInterface<R>) { - for (size_t i = 0; i < cnt; i++) { - rec.key = rand(); - rec.value = rand(); - - if constexpr (de::WeightedRecordInterface<R>) { - rec.weight = 1; + if constexpr (de::KVPInterface<R>){ + if constexpr (std::is_same_v<decltype(R::key), const char*>){ + auto records = read_string_data(kjv_wordlist, cnt); + for (size_t i=0; i<cnt; i++) { + buffer->append(records[i]); + } + } else { + for (size_t i = 0; i < cnt; i++) { + if constexpr (de::WeightedRecordInterface<R>) { + buffer->append({(uint64_t) rand(), (uint32_t) rand(), 1}); + } else { + buffer->append({(uint64_t) rand(), (uint32_t) rand()}); + } } - - buffer->append(rec); } } else if constexpr (de::NDRecordInterface<R>) { for (size_t i=0; i<cnt; i++) { - uint64_t a = rand(); - uint64_t b = rand(); - buffer->append({a, b}); + buffer->append({(uint64_t) rand(), (uint64_t) rand()}); } } @@ -110,25 +152,19 @@ static de::MutableBuffer<R> *create_sequential_mbuffer(size_t start, size_t stop size_t cnt = stop - start; auto buffer = new de::MutableBuffer<R>(cnt/2, cnt); - for (size_t i=start; i<stop; i++) { - R rec; - if constexpr (de::KVPInterface<R>) { - rec.key = i; - rec.value = i; - } else if constexpr (de::NDRecordInterface<R>) { - rec = {i, i}; - } + for (uint32_t i=start; i<stop; i++) { if constexpr (de::WeightedRecordInterface<R>) { - rec.weight = 1; + buffer->append({i, i, 1}); + } else { + buffer->append({i, i}); } - - buffer->append(rec); } return buffer; } +/* template <de::KVPInterface R> static de::MutableBuffer<R> *create_test_mbuffer_tombstones(size_t cnt, size_t ts_cnt) { @@ -138,13 +174,13 @@ static de::MutableBuffer<R> *create_test_mbuffer_tombstones(size_t cnt, size_t t R rec; for (size_t i = 0; i < cnt; i++) { - rec.key = rand(); - rec.value = rand(); - if constexpr (de::WeightedRecordInterface<R>) { - rec.weight = 1; + rec = {rand(), rand(), 1}; + } else { + rec = {rand(), rand(), 1}; } + if (i < ts_cnt) { tombstones.push_back({rec.key, rec.value}); } @@ -159,6 +195,7 @@ static de::MutableBuffer<R> *create_test_mbuffer_tombstones(size_t cnt, size_t t return buffer; } +*/ template <typename R> requires de::WeightedRecordInterface<R> && de::KVPInterface<R> @@ -189,20 +226,12 @@ static de::MutableBuffer<R> *create_double_seq_mbuffer(size_t cnt, bool ts=false { auto buffer = new de::MutableBuffer<R>(cnt/2, cnt); - for (size_t i = 0; i < cnt / 2; i++) { - R rec; - rec.key = i; - rec.value = i; - - buffer->append(rec, ts); + for (uint32_t i = 0; i < cnt / 2; i++) { + buffer->append({i, i}, ts); } - for (size_t i = 0; i < cnt / 2; i++) { - R rec; - rec.key = i; - rec.value = i + 1; - - buffer->append(rec, ts); + for (uint32_t i = 0; i < cnt / 2; i++) { + buffer->append({i, i+1}, ts); } return buffer; diff --git a/tests/include/wirs.h b/tests/include/wirs.h index 90cd22d..4c0630f 100644 --- a/tests/include/wirs.h +++ b/tests/include/wirs.h @@ -117,7 +117,8 @@ START_TEST(t_range_query_merge) } } - auto result = rq::Query<R, Shard>::merge(proc_results, nullptr); + std::vector<R> result; + rq::Query<R, Shard>::merge(proc_results, nullptr, result); std::sort(result.begin(), result.end()); ck_assert_int_eq(result.size(), result_size); |