From 7c03d771475421c1d5a2bbc135242536af1a371c Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Mon, 25 Sep 2023 10:49:36 -0400
Subject: Re-structuring Project + scheduling updates

This is a big one--probably should have split it apart, but I'm feeling
lazy this morning.

* Organized the mess of header files in include/framework by splitting
them out into their own subdirectories, and renaming a few files to
remove redundancies introduced by the directory structure.

* Introduced a new framework/ShardRequirements.h header file for simpler
shard development. This header simply contains the necessary includes
from framework/* for creating shard files. This should help to remove
structural dependencies from the framework file structure and shards,
as well as centralizing the necessary framework files to make shard
development easier.

* Created a (currently dummy) SchedulerInterface, and make the scheduler
implementation a template parameter of the dynamic extension for easier
testing of various scheduling policies. There's still more work to be
done to fully integrate the scheduler (queries, multiple buffers), but
some more of the necessary framework code for this has been added as well.

* Adjusted the Task interface setup for the scheduler. The task structures
have been removed from ExtensionStructure and placed in their own header
file. Additionally, I started experimenting with using std::variant,
as opposed to inheritence, to implement subtype polymorphism on the
Merge and Query tasks. The scheduler now has a general task queue that
contains both, and std::variant, std::visit, and std::get are used to
manipulate them without virtual functions.

* Removed Alex.h, as it can't build anyway. There's a branch out there
containing the Alex implementation stripped of the C++20 stuff. So
there's no need to keep it here.
---
 tests/testing.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'tests/testing.h')

diff --git a/tests/testing.h b/tests/testing.h
index bdf4869..023be7f 100644
--- a/tests/testing.h
+++ b/tests/testing.h
@@ -18,8 +18,8 @@
 
 #include "util/types.h"
 #include "psu-util/alignment.h"
-#include "framework/MutableBuffer.h"
-#include "framework/RecordInterface.h"
+#include "framework/structure/MutableBuffer.h"
+#include "framework/interface/Record.h"
 
 typedef de::WeightedRecord<uint64_t, uint32_t, uint64_t> WRec;
 typedef de::Record<uint64_t, uint32_t> Rec;
-- 
cgit v1.2.3


From 357cab549c2ed33970562b84ff6f83923742343d Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Tue, 7 Nov 2023 15:34:24 -0500
Subject: Comment and License updates

---
 tests/testing.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'tests/testing.h')

diff --git a/tests/testing.h b/tests/testing.h
index 023be7f..e1d3402 100644
--- a/tests/testing.h
+++ b/tests/testing.h
@@ -6,7 +6,7 @@
  * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> 
  *                    Dong Xie <dongx@psu.edu>
  *
- * All rights reserved. Published under the Modified BSD License.
+ * Distributed under the Modified BSD License.
  *
  */
 #pragma once
-- 
cgit v1.2.3


From 5a2c378aad3f1a9923db3191ffaa3fb807d392b2 Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Thu, 11 Jan 2024 15:29:51 -0500
Subject: Ported ISAMTree over to new buffer setup

I may still play with the shard from shards constructor, and queries
need some work yet too.
---
 tests/testing.h | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

(limited to 'tests/testing.h')

diff --git a/tests/testing.h b/tests/testing.h
index e1d3402..4e660dd 100644
--- a/tests/testing.h
+++ b/tests/testing.h
@@ -77,7 +77,7 @@ static bool roughly_equal(int n1, int n2, size_t mag, double epsilon) {
 }
 
 static de::MutableBuffer<PRec> *create_2d_mbuffer(size_t cnt) {
-    auto buffer = new de::MutableBuffer<PRec>(cnt, cnt);
+    auto buffer = new de::MutableBuffer<PRec>(cnt/2, cnt);
 
     for (int64_t i=0; i<cnt; i++) {
         buffer->append({rand(), rand()});
@@ -87,7 +87,7 @@ static de::MutableBuffer<PRec> *create_2d_mbuffer(size_t cnt) {
 }
 
 static de::MutableBuffer<PRec> *create_2d_sequential_mbuffer(size_t cnt) {
-    auto buffer = new de::MutableBuffer<PRec>(cnt, cnt);
+    auto buffer = new de::MutableBuffer<PRec>(cnt/2, cnt);
     for (int64_t i=0; i<cnt; i++) {
         buffer->append({i, i});
     }
@@ -98,7 +98,7 @@ static de::MutableBuffer<PRec> *create_2d_sequential_mbuffer(size_t cnt) {
 template <de::KVPInterface R>
 static de::MutableBuffer<R> *create_test_mbuffer(size_t cnt)
 {
-    auto buffer = new de::MutableBuffer<R>(cnt, cnt);
+    auto buffer = new de::MutableBuffer<R>(cnt/2, cnt);
 
     R rec;
     for (size_t i = 0; i < cnt; i++) {
@@ -119,7 +119,7 @@ template <de::KVPInterface R>
 static de::MutableBuffer<R> *create_sequential_mbuffer(decltype(R::key) start, decltype(R::key) stop)
 {
     size_t cnt = stop - start;
-    auto buffer = new de::MutableBuffer<R>(cnt, cnt);
+    auto buffer = new de::MutableBuffer<R>(cnt/2, cnt);
 
     for (size_t i=start; i<stop; i++) {
         R rec;
@@ -139,7 +139,7 @@ static de::MutableBuffer<R> *create_sequential_mbuffer(decltype(R::key) start, d
 template <de::KVPInterface R>
 static de::MutableBuffer<R> *create_test_mbuffer_tombstones(size_t cnt, size_t ts_cnt) 
 {
-    auto buffer = new de::MutableBuffer<R>(cnt, ts_cnt);
+    auto buffer = new de::MutableBuffer<R>(cnt/2, cnt);
 
     std::vector<std::pair<uint64_t, uint32_t>> tombstones;
 
@@ -171,7 +171,7 @@ template <typename R>
 requires de::WeightedRecordInterface<R> && de::KVPInterface<R>
 static de::MutableBuffer<R> *create_weighted_mbuffer(size_t cnt)
 {
-    auto buffer = new de::MutableBuffer<R>(cnt, cnt);
+    auto buffer = new de::MutableBuffer<R>(cnt/2, cnt);
     
     // Put in half of the count with weight one.
     for (uint32_t i=0; i< cnt / 2; i++) {
@@ -194,7 +194,7 @@ static de::MutableBuffer<R> *create_weighted_mbuffer(size_t cnt)
 template <de::KVPInterface R>
 static de::MutableBuffer<R> *create_double_seq_mbuffer(size_t cnt, bool ts=false) 
 {
-    auto buffer = new de::MutableBuffer<R>(cnt, cnt);
+    auto buffer = new de::MutableBuffer<R>(cnt/2, cnt);
 
     for (size_t i = 0; i < cnt / 2; i++) { 
         R rec;
-- 
cgit v1.2.3


From c4514c2e62a711189cf3c914297885d97fb51a09 Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Fri, 12 Jan 2024 14:08:33 -0500
Subject: Initial pass at unit test refactoring

Restructured unit tests to be a bit more modular. I have some further
plans to expand on this, particular for the query tests (including both
shard and framework level test functions that can be injected at will).
---
 tests/testing.h | 218 --------------------------------------------------------
 1 file changed, 218 deletions(-)
 delete mode 100644 tests/testing.h

(limited to 'tests/testing.h')

diff --git a/tests/testing.h b/tests/testing.h
deleted file mode 100644
index 4e660dd..0000000
--- a/tests/testing.h
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * tests/testing.h
- *
- * Unit test utility functions/definitions
- *
- * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> 
- *                    Dong Xie <dongx@psu.edu>
- *
- * Distributed under the Modified BSD License.
- *
- */
-#pragma once
-
-#include <string>
-
-#include <unistd.h>
-#include <fcntl.h>
-
-#include "util/types.h"
-#include "psu-util/alignment.h"
-#include "framework/structure/MutableBuffer.h"
-#include "framework/interface/Record.h"
-
-typedef de::WeightedRecord<uint64_t, uint32_t, uint64_t> WRec;
-typedef de::Record<uint64_t, uint32_t> Rec;
-typedef de::EuclidPoint<int64_t> PRec;
-
-template <de::RecordInterface R> 
-std::vector<R> strip_wrapping(std::vector<de::Wrapped<R>> vec) {
-    std::vector<R> out(vec.size());
-    for (size_t i=0; i<vec.size(); i++) {
-        out[i] = vec[i].rec;
-    }
-
-    return out;
-}
-
-static bool initialize_test_file(std::string fname, size_t page_cnt)
-{
-    auto flags = O_RDWR | O_CREAT | O_TRUNC;
-    mode_t mode = 0640;
-    char *page = nullptr;
-
-    int fd = open(fname.c_str(), flags, mode);
-    if (fd == -1) {
-        goto error;
-    }
-
-    page = (char *) aligned_alloc(psudb::SECTOR_SIZE, psudb::PAGE_SIZE);
-    if (!page) {
-        goto error_opened;
-    }
-
-    for (size_t i=0; i<=page_cnt; i++) {
-        *((int *) page) = i;
-        if (write(fd, page, psudb::PAGE_SIZE) == -1) {
-            goto error_alloced;
-        }
-    }
-
-    free(page);
-
-    return 1;
-
-error_alloced:
-    free(page);
-
-error_opened:
-    close(fd);
-
-error:
-    return 0;
-}
-
-static bool roughly_equal(int n1, int n2, size_t mag, double epsilon) {
-    return ((double) std::abs(n1 - n2) / (double) mag) < epsilon;
-}
-
-static de::MutableBuffer<PRec> *create_2d_mbuffer(size_t cnt) {
-    auto buffer = new de::MutableBuffer<PRec>(cnt/2, cnt);
-
-    for (int64_t i=0; i<cnt; i++) {
-        buffer->append({rand(), rand()});
-    }
-
-    return buffer;
-}
-
-static de::MutableBuffer<PRec> *create_2d_sequential_mbuffer(size_t cnt) {
-    auto buffer = new de::MutableBuffer<PRec>(cnt/2, cnt);
-    for (int64_t i=0; i<cnt; i++) {
-        buffer->append({i, i});
-    }
-
-    return buffer;
-}
-
-template <de::KVPInterface R>
-static de::MutableBuffer<R> *create_test_mbuffer(size_t cnt)
-{
-    auto buffer = new de::MutableBuffer<R>(cnt/2, cnt);
-
-    R rec;
-    for (size_t i = 0; i < cnt; i++) {
-        rec.key = rand();
-        rec.value = rand();
-
-        if constexpr (de::WeightedRecordInterface<R>) {
-            rec.weight = 1;
-        }
-
-        buffer->append(rec);
-    }
-
-    return buffer;
-}
-
-template <de::KVPInterface R>
-static de::MutableBuffer<R> *create_sequential_mbuffer(decltype(R::key) start, decltype(R::key) stop)
-{
-    size_t cnt = stop - start;
-    auto buffer = new de::MutableBuffer<R>(cnt/2, cnt);
-
-    for (size_t i=start; i<stop; i++) {
-        R rec;
-        rec.key = i;
-        rec.value = i;
-
-        if constexpr (de::WeightedRecordInterface<R>) {
-            rec.weight = 1;
-        }
-
-        buffer->append(rec);
-    }
-
-    return buffer;
-}
-
-template <de::KVPInterface R>
-static de::MutableBuffer<R> *create_test_mbuffer_tombstones(size_t cnt, size_t ts_cnt) 
-{
-    auto buffer = new de::MutableBuffer<R>(cnt/2, cnt);
-
-    std::vector<std::pair<uint64_t, uint32_t>> tombstones;
-
-    R rec;
-    for (size_t i = 0; i < cnt; i++) {
-        rec.key = rand();
-        rec.value = rand();
-
-        if constexpr (de::WeightedRecordInterface<R>) {
-            rec.weight = 1;
-        }
-
-        if (i < ts_cnt) {
-            tombstones.push_back({rec.key, rec.value});
-        }
-
-        buffer->append(rec);
-    }
-
-    rec.set_tombstone();
-    for (size_t i=0; i<ts_cnt; i++) {
-        buffer->append(rec);
-    }
-
-    return buffer;
-}
-
-template <typename R>
-requires de::WeightedRecordInterface<R> && de::KVPInterface<R>
-static de::MutableBuffer<R> *create_weighted_mbuffer(size_t cnt)
-{
-    auto buffer = new de::MutableBuffer<R>(cnt/2, cnt);
-    
-    // Put in half of the count with weight one.
-    for (uint32_t i=0; i< cnt / 2; i++) {
-        buffer->append(R {1, i, 2});
-    }
-
-    // put in a quarter of the count with weight four.
-    for (uint32_t i=0; i< cnt / 4; i++) {
-        buffer->append(R {2, i, 4});
-    }
-
-    // the remaining quarter with weight eight.
-    for (uint32_t i=0; i< cnt / 4; i++) {
-        buffer->append(R {3, i, 8});
-    }
-
-    return buffer;
-}
-
-template <de::KVPInterface R>
-static de::MutableBuffer<R> *create_double_seq_mbuffer(size_t cnt, bool ts=false) 
-{
-    auto buffer = new de::MutableBuffer<R>(cnt/2, cnt);
-
-    for (size_t i = 0; i < cnt / 2; i++) { 
-        R rec;
-        rec.key = i;
-        rec.value = i;
-
-        buffer->append(rec, ts);
-    }
-
-    for (size_t i = 0; i < cnt / 2; i++) {
-        R rec;
-        rec.key = i;
-        rec.value = i + 1;
-
-        buffer->append(rec, ts);
-    }
-
-    return buffer;
-}
-
-
-- 
cgit v1.2.3