From 7c03d771475421c1d5a2bbc135242536af1a371c Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Mon, 25 Sep 2023 10:49:36 -0400
Subject: Re-structuring Project + scheduling updates

This is a big one--probably should have split it apart, but I'm feeling
lazy this morning.

* Organized the mess of header files in include/framework by splitting
them out into their own subdirectories, and renaming a few files to
remove redundancies introduced by the directory structure.

* Introduced a new framework/ShardRequirements.h header file for simpler
shard development. This header simply contains the necessary includes
from framework/* for creating shard files. This should help to remove
structural dependencies from the framework file structure and shards,
as well as centralizing the necessary framework files to make shard
development easier.

* Created a (currently dummy) SchedulerInterface, and make the scheduler
implementation a template parameter of the dynamic extension for easier
testing of various scheduling policies. There's still more work to be
done to fully integrate the scheduler (queries, multiple buffers), but
some more of the necessary framework code for this has been added as well.

* Adjusted the Task interface setup for the scheduler. The task structures
have been removed from ExtensionStructure and placed in their own header
file. Additionally, I started experimenting with using std::variant,
as opposed to inheritence, to implement subtype polymorphism on the
Merge and Query tasks. The scheduler now has a general task queue that
contains both, and std::variant, std::visit, and std::get are used to
manipulate them without virtual functions.

* Removed Alex.h, as it can't build anyway. There's a branch out there
containing the Alex implementation stripped of the C++20 stuff. So
there's no need to keep it here.
---
 include/framework/structure/InternalLevel.h | 258 ++++++++++++++++++++++++++++
 1 file changed, 258 insertions(+)
 create mode 100644 include/framework/structure/InternalLevel.h

(limited to 'include/framework/structure/InternalLevel.h')

diff --git a/include/framework/structure/InternalLevel.h b/include/framework/structure/InternalLevel.h
new file mode 100644
index 0000000..b9230f4
--- /dev/null
+++ b/include/framework/structure/InternalLevel.h
@@ -0,0 +1,258 @@
+/*
+ * include/framework/InternalLevel.h
+ *
+ * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> 
+ *                    Dong Xie <dongx@psu.edu>
+ *
+ * All rights reserved. Published under the Modified BSD License.
+ *
+ */
+#pragma once
+
+#include <vector>
+#include <memory>
+
+#include "util/types.h"
+#include "framework/interface/Shard.h"
+#include "framework/interface/Query.h"
+#include "framework/interface/Record.h"
+#include "framework/structure/MutableBuffer.h"
+
+namespace de {
+template <RecordInterface R, ShardInterface S, QueryInterface Q>
+class InternalLevel;
+
+
+
+template <RecordInterface R, ShardInterface S, QueryInterface Q>
+class InternalLevel {
+    typedef S Shard;
+    typedef MutableBuffer<R> Buffer;
+public:
+    InternalLevel(ssize_t level_no, size_t shard_cap)
+    : m_level_no(level_no)
+    , m_shard_cnt(0)
+    , m_shards(shard_cap, nullptr)
+    , m_owns(shard_cap, true)
+    , m_pending_shard(nullptr)
+    {}
+
+    // Create a new memory level sharing the shards and repurposing it as previous level_no + 1
+    // WARNING: for leveling only.
+    InternalLevel(InternalLevel* level)
+    : m_level_no(level->m_level_no + 1)
+    , m_shard_cnt(level->m_shard_cnt)
+    , m_shards(level->m_shards.size(), nullptr)
+    , m_owns(level->m_owns.size(), true) 
+    , m_pending_shard(nullptr)
+    {
+        assert(m_shard_cnt == 1 && m_shards.size() == 1);
+
+        for (size_t i=0; i<m_shards.size(); i++) {
+            level->m_owns[i] = false;
+            m_shards[i] = level->m_shards[i];
+        }
+    }
+
+    ~InternalLevel() { 
+        for (size_t i=0; i<m_shards.size(); i++) {
+            if (m_owns[i]) delete m_shards[i];
+        }
+
+        delete m_pending_shard;
+    }
+
+    // WARNING: for leveling only.
+    // assuming the base level is the level new level is merging into. (base_level is larger.)
+    static std::shared_ptr<InternalLevel> merge_levels(InternalLevel* base_level, InternalLevel* new_level) {
+        assert(base_level->m_level_no > new_level->m_level_no || (base_level->m_level_no == 0 && new_level->m_level_no == 0));
+        auto res = new InternalLevel(base_level->m_level_no, 1);
+        res->m_shard_cnt = 1;
+        Shard* shards[2];
+        shards[0] = base_level->m_shards[0];
+        shards[1] = new_level->m_shards[0];
+
+        res->m_shards[0] = new S(shards, 2);
+        return std::shared_ptr<InternalLevel>(res);
+    }
+
+    void append_buffer(Buffer* buffer) {
+        if (m_shard_cnt == m_shards.size()) {
+            assert(m_pending_shard == nullptr);
+            m_pending_shard = new S(buffer);
+            return;
+        }
+
+        m_shards[m_shard_cnt] = new S(buffer);
+        m_owns[m_shard_cnt] = true;
+        ++m_shard_cnt;
+    }
+
+    void append_merged_shards(InternalLevel* level) {
+        if (m_shard_cnt == m_shards.size()) {
+            m_pending_shard = new S(level->m_shards.data(), level->m_shard_cnt);
+            return;
+        }
+
+        m_shards[m_shard_cnt] = new S(level->m_shards.data(), level->m_shard_cnt);
+        m_owns[m_shard_cnt] = true;
+
+        ++m_shard_cnt;
+    }
+
+
+    void finalize() {
+        if (m_pending_shard) {
+            for (size_t i=0; i<m_shards.size(); i++) {
+                if (m_owns[i]) {
+                    delete m_shards[i];
+                    m_shards[i] = nullptr;
+                    m_owns[i] = false;
+                }
+            }
+
+            m_shards[0] = m_pending_shard;
+            m_owns[0] = true;
+            m_pending_shard = nullptr;
+            m_shard_cnt = 1;
+        }
+    }
+
+    Shard *get_merged_shard() {
+        if (m_shard_cnt == 0) {
+            return nullptr;
+        }
+
+        Shard *shards[m_shard_cnt];
+
+        for (size_t i=0; i<m_shard_cnt; i++) {
+            shards[i] = m_shards[i];
+        }
+
+        return new S(shards, m_shard_cnt);
+    }
+
+    // Append the sample range in-order.....
+    void get_query_states(std::vector<std::pair<ShardID, Shard *>> &shards, std::vector<void*>& shard_states, void *query_parms) {
+        for (size_t i=0; i<m_shard_cnt; i++) {
+            if (m_shards[i]) {
+                auto shard_state = Q::get_query_state(m_shards[i], query_parms);
+                shards.push_back({{m_level_no, (ssize_t) i}, m_shards[i]});
+                shard_states.emplace_back(shard_state);
+            }
+        }
+    }
+
+    bool check_tombstone(size_t shard_stop, const R& rec) {
+        if (m_shard_cnt == 0) return false;
+
+        for (int i = m_shard_cnt - 1; i >= (ssize_t) shard_stop;  i--) {
+            if (m_shards[i]) {
+                auto res = m_shards[i]->point_lookup(rec, true);
+                if (res && res->is_tombstone()) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    bool delete_record(const R &rec) {
+        if (m_shard_cnt == 0) return false;
+
+        for (size_t i = 0; i < m_shards.size();  ++i) {
+            if (m_shards[i]) {
+                auto res = m_shards[i]->point_lookup(rec);
+                if (res) {
+                    res->set_delete();
+                    return true;
+                }
+            }
+        }
+
+        return false;
+    }
+
+    Shard* get_shard(size_t idx) {
+        return m_shards[idx];
+    }
+
+    size_t get_shard_count() {
+        return m_shard_cnt;
+    }
+
+    size_t get_record_count() {
+        size_t cnt = 0;
+        for (size_t i=0; i<m_shard_cnt; i++) {
+            cnt += m_shards[i]->get_record_count();
+        }
+
+        return cnt;
+    }
+    
+    size_t get_tombstone_count() {
+        size_t res = 0;
+        for (size_t i = 0; i < m_shard_cnt; ++i) {
+            res += m_shards[i]->get_tombstone_count();
+        }
+        return res;
+    }
+
+    size_t get_aux_memory_usage() {
+        size_t cnt = 0;
+        for (size_t i=0; i<m_shard_cnt; i++) {
+            cnt += m_shards[i]->get_aux_memory_usage();
+        }
+
+        return cnt;
+    }
+
+    size_t get_memory_usage() {
+        size_t cnt = 0;
+        for (size_t i=0; i<m_shard_cnt; i++) {
+            if (m_shards[i]) {
+                cnt += m_shards[i]->get_memory_usage();
+            }
+        }
+
+        return cnt;
+    }
+
+    double get_tombstone_prop() {
+        size_t tscnt = 0;
+        size_t reccnt = 0;
+        for (size_t i=0; i<m_shard_cnt; i++) {
+            if (m_shards[i]) {
+                tscnt += m_shards[i]->get_tombstone_count();
+                reccnt += (*m_shards[i])->get_record_count();
+            }
+        }
+
+        return (double) tscnt / (double) (tscnt + reccnt);
+    }
+
+private:
+    ssize_t m_level_no;
+    
+    size_t m_shard_cnt;
+    size_t m_shard_size_cap;
+
+    std::vector<Shard*> m_shards;
+
+    Shard *m_pending_shard;
+
+    std::vector<bool> m_owns;
+
+    std::shared_ptr<InternalLevel> clone() {
+        auto new_level = std::make_shared<InternalLevel>(m_level_no, m_shards.size());
+        for (size_t i=0; i<m_shard_cnt; i++) {
+            new_level->m_shards[i] = m_shards[i];
+            new_level->m_owns[i] = true;
+            m_owns[i] = false;
+        }
+
+        return new_level;
+    }
+};
+
+}
-- 
cgit v1.2.3


From 39ae3e0441d8297a09197aba98bd494b5ada12c1 Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Mon, 30 Oct 2023 14:17:59 -0400
Subject: Concurrency updates + fixes for compile errors

---
 include/framework/structure/InternalLevel.h | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

(limited to 'include/framework/structure/InternalLevel.h')

diff --git a/include/framework/structure/InternalLevel.h b/include/framework/structure/InternalLevel.h
index b9230f4..342a2c7 100644
--- a/include/framework/structure/InternalLevel.h
+++ b/include/framework/structure/InternalLevel.h
@@ -231,6 +231,17 @@ public:
         return (double) tscnt / (double) (tscnt + reccnt);
     }
 
+    std::shared_ptr<InternalLevel> clone() {
+        auto new_level = std::make_shared<InternalLevel>(m_level_no, m_shards.size());
+        for (size_t i=0; i<m_shard_cnt; i++) {
+            new_level->m_shards[i] = m_shards[i];
+            new_level->m_owns[i] = true;
+            m_owns[i] = false;
+        }
+
+        return new_level;
+    }
+
 private:
     ssize_t m_level_no;
     
@@ -243,16 +254,6 @@ private:
 
     std::vector<bool> m_owns;
 
-    std::shared_ptr<InternalLevel> clone() {
-        auto new_level = std::make_shared<InternalLevel>(m_level_no, m_shards.size());
-        for (size_t i=0; i<m_shard_cnt; i++) {
-            new_level->m_shards[i] = m_shards[i];
-            new_level->m_owns[i] = true;
-            m_owns[i] = false;
-        }
-
-        return new_level;
-    }
 };
 
 }
-- 
cgit v1.2.3


From d2279e1b96d352a0af1d425dcaaf93e8a26a8d52 Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Mon, 30 Oct 2023 17:15:05 -0400
Subject: General Comment + Consistency updates

---
 include/framework/structure/InternalLevel.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'include/framework/structure/InternalLevel.h')

diff --git a/include/framework/structure/InternalLevel.h b/include/framework/structure/InternalLevel.h
index 342a2c7..7a7b98c 100644
--- a/include/framework/structure/InternalLevel.h
+++ b/include/framework/structure/InternalLevel.h
@@ -1,7 +1,7 @@
 /*
- * include/framework/InternalLevel.h
+ * include/framework/structure/InternalLevel.h
  *
- * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> 
+ * Copyright (C) 2023 Douglas B. Rumbaugh <drumbaugh@psu.edu> 
  *                    Dong Xie <dongx@psu.edu>
  *
  * All rights reserved. Published under the Modified BSD License.
-- 
cgit v1.2.3


From 56cc8f63a218bc13e0c8395b479267862de19714 Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Mon, 6 Nov 2023 14:01:39 -0500
Subject: InternalLevel: switched to std::sharedptr for shard memory management

---
 include/framework/structure/InternalLevel.h | 81 +++++++++++------------------
 1 file changed, 29 insertions(+), 52 deletions(-)

(limited to 'include/framework/structure/InternalLevel.h')

diff --git a/include/framework/structure/InternalLevel.h b/include/framework/structure/InternalLevel.h
index 7a7b98c..632fe17 100644
--- a/include/framework/structure/InternalLevel.h
+++ b/include/framework/structure/InternalLevel.h
@@ -33,32 +33,10 @@ public:
     : m_level_no(level_no)
     , m_shard_cnt(0)
     , m_shards(shard_cap, nullptr)
-    , m_owns(shard_cap, true)
     , m_pending_shard(nullptr)
     {}
 
-    // Create a new memory level sharing the shards and repurposing it as previous level_no + 1
-    // WARNING: for leveling only.
-    InternalLevel(InternalLevel* level)
-    : m_level_no(level->m_level_no + 1)
-    , m_shard_cnt(level->m_shard_cnt)
-    , m_shards(level->m_shards.size(), nullptr)
-    , m_owns(level->m_owns.size(), true) 
-    , m_pending_shard(nullptr)
-    {
-        assert(m_shard_cnt == 1 && m_shards.size() == 1);
-
-        for (size_t i=0; i<m_shards.size(); i++) {
-            level->m_owns[i] = false;
-            m_shards[i] = level->m_shards[i];
-        }
-    }
-
     ~InternalLevel() { 
-        for (size_t i=0; i<m_shards.size(); i++) {
-            if (m_owns[i]) delete m_shards[i];
-        }
-
         delete m_pending_shard;
     }
 
@@ -69,10 +47,10 @@ public:
         auto res = new InternalLevel(base_level->m_level_no, 1);
         res->m_shard_cnt = 1;
         Shard* shards[2];
-        shards[0] = base_level->m_shards[0];
-        shards[1] = new_level->m_shards[0];
+        shards[0] = base_level->m_shards[0].get();
+        shards[1] = new_level->m_shards[0].get();
 
-        res->m_shards[0] = new S(shards, 2);
+        res->m_shards[0] = std::make_shared<S>(shards, 2);
         return std::shared_ptr<InternalLevel>(res);
     }
 
@@ -83,19 +61,23 @@ public:
             return;
         }
 
-        m_shards[m_shard_cnt] = new S(buffer);
-        m_owns[m_shard_cnt] = true;
+        m_shards[m_shard_cnt] = std::make_shared<S>(buffer);
         ++m_shard_cnt;
     }
 
     void append_merged_shards(InternalLevel* level) {
+        Shard *shards[level->m_shard_cnt];
+        for (size_t i=0; i<level->m_shard_cnt; i++) {
+            shards[i] = level->m_shards[i].get();
+        }
+
         if (m_shard_cnt == m_shards.size()) {
-            m_pending_shard = new S(level->m_shards.data(), level->m_shard_cnt);
+            m_pending_shard = new S(shards, level->m_shard_cnt);
             return;
         }
 
-        m_shards[m_shard_cnt] = new S(level->m_shards.data(), level->m_shard_cnt);
-        m_owns[m_shard_cnt] = true;
+        auto tmp = new S(shards, level->m_shard_cnt);
+        m_shards[m_shard_cnt] = std::shared_ptr<S>(tmp);
 
         ++m_shard_cnt;
     }
@@ -104,15 +86,10 @@ public:
     void finalize() {
         if (m_pending_shard) {
             for (size_t i=0; i<m_shards.size(); i++) {
-                if (m_owns[i]) {
-                    delete m_shards[i];
-                    m_shards[i] = nullptr;
-                    m_owns[i] = false;
-                }
+                m_shards[i] = nullptr;
             }
 
-            m_shards[0] = m_pending_shard;
-            m_owns[0] = true;
+            m_shards[0] = std::shared_ptr<S>(m_pending_shard);
             m_pending_shard = nullptr;
             m_shard_cnt = 1;
         }
@@ -126,7 +103,7 @@ public:
         Shard *shards[m_shard_cnt];
 
         for (size_t i=0; i<m_shard_cnt; i++) {
-            shards[i] = m_shards[i];
+            shards[i] = m_shards[i].get();
         }
 
         return new S(shards, m_shard_cnt);
@@ -136,8 +113,8 @@ public:
     void get_query_states(std::vector<std::pair<ShardID, Shard *>> &shards, std::vector<void*>& shard_states, void *query_parms) {
         for (size_t i=0; i<m_shard_cnt; i++) {
             if (m_shards[i]) {
-                auto shard_state = Q::get_query_state(m_shards[i], query_parms);
-                shards.push_back({{m_level_no, (ssize_t) i}, m_shards[i]});
+                auto shard_state = Q::get_query_state(m_shards[i].get(), query_parms);
+                shards.push_back({{m_level_no, (ssize_t) i}, m_shards[i].get()});
                 shard_states.emplace_back(shard_state);
             }
         }
@@ -174,7 +151,7 @@ public:
     }
 
     Shard* get_shard(size_t idx) {
-        return m_shards[idx];
+        return m_shards[idx].get();
     }
 
     size_t get_shard_count() {
@@ -184,7 +161,9 @@ public:
     size_t get_record_count() {
         size_t cnt = 0;
         for (size_t i=0; i<m_shard_cnt; i++) {
-            cnt += m_shards[i]->get_record_count();
+            if (m_shards[i]) {
+                cnt += m_shards[i]->get_record_count();
+            }
         }
 
         return cnt;
@@ -193,7 +172,9 @@ public:
     size_t get_tombstone_count() {
         size_t res = 0;
         for (size_t i = 0; i < m_shard_cnt; ++i) {
-            res += m_shards[i]->get_tombstone_count();
+            if (m_shards[i]) {
+                res += m_shards[i]->get_tombstone_count();
+            }
         }
         return res;
     }
@@ -201,7 +182,9 @@ public:
     size_t get_aux_memory_usage() {
         size_t cnt = 0;
         for (size_t i=0; i<m_shard_cnt; i++) {
-            cnt += m_shards[i]->get_aux_memory_usage();
+            if (m_shards[i]){
+                cnt += m_shards[i]->get_aux_memory_usage();
+            }
         }
 
         return cnt;
@@ -224,7 +207,7 @@ public:
         for (size_t i=0; i<m_shard_cnt; i++) {
             if (m_shards[i]) {
                 tscnt += m_shards[i]->get_tombstone_count();
-                reccnt += (*m_shards[i])->get_record_count();
+                reccnt += m_shards[i]->get_record_count();
             }
         }
 
@@ -235,8 +218,6 @@ public:
         auto new_level = std::make_shared<InternalLevel>(m_level_no, m_shards.size());
         for (size_t i=0; i<m_shard_cnt; i++) {
             new_level->m_shards[i] = m_shards[i];
-            new_level->m_owns[i] = true;
-            m_owns[i] = false;
         }
 
         return new_level;
@@ -248,12 +229,8 @@ private:
     size_t m_shard_cnt;
     size_t m_shard_size_cap;
 
-    std::vector<Shard*> m_shards;
-
+    std::vector<std::shared_ptr<Shard>> m_shards;
     Shard *m_pending_shard;
-
-    std::vector<bool> m_owns;
-
 };
 
 }
-- 
cgit v1.2.3


From 357cab549c2ed33970562b84ff6f83923742343d Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Tue, 7 Nov 2023 15:34:24 -0500
Subject: Comment and License updates

---
 include/framework/structure/InternalLevel.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'include/framework/structure/InternalLevel.h')

diff --git a/include/framework/structure/InternalLevel.h b/include/framework/structure/InternalLevel.h
index 632fe17..00e0c58 100644
--- a/include/framework/structure/InternalLevel.h
+++ b/include/framework/structure/InternalLevel.h
@@ -4,7 +4,7 @@
  * Copyright (C) 2023 Douglas B. Rumbaugh <drumbaugh@psu.edu> 
  *                    Dong Xie <dongx@psu.edu>
  *
- * All rights reserved. Published under the Modified BSD License.
+ * Distributed under the Modified BSD License.
  *
  */
 #pragma once
-- 
cgit v1.2.3


From 83486744600e8be338c75c2e3d2339452a392a9d Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Mon, 13 Nov 2023 10:41:13 -0500
Subject: Fixed merge logic bug in tiering

In InternalLevel::clone(), the m_shard_cnt variable was not being set
appropriately in the clone, resulting in the record counts reported for
a multi-shard level to be reported incorrectly.

In DynamicExtension::merge(), the merges were being performed in the
wrong order, resulting in multi-level merges deleting records. The
leveling tests all passed even with this bug for some reason, but it
caused tiering tests to fail. It isn't clear _why_ leveling appeared to
work, but the bug is now fixed, so that's largely irrelevant I suppose.
---
 include/framework/structure/InternalLevel.h | 1 +
 1 file changed, 1 insertion(+)

(limited to 'include/framework/structure/InternalLevel.h')

diff --git a/include/framework/structure/InternalLevel.h b/include/framework/structure/InternalLevel.h
index 00e0c58..d146b73 100644
--- a/include/framework/structure/InternalLevel.h
+++ b/include/framework/structure/InternalLevel.h
@@ -219,6 +219,7 @@ public:
         for (size_t i=0; i<m_shard_cnt; i++) {
             new_level->m_shards[i] = m_shards[i];
         }
+        new_level->m_shard_cnt = m_shard_cnt;
 
         return new_level;
     }
-- 
cgit v1.2.3


From 3c127eda69295cb306739bdd3c5ddccff6026a8d Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Wed, 13 Dec 2023 12:39:54 -0500
Subject: Refactoring: corrected a number of names and added more comments

---
 include/framework/structure/InternalLevel.h | 58 ++++++++++++++++++++---------
 1 file changed, 41 insertions(+), 17 deletions(-)

(limited to 'include/framework/structure/InternalLevel.h')

diff --git a/include/framework/structure/InternalLevel.h b/include/framework/structure/InternalLevel.h
index d146b73..e70ed76 100644
--- a/include/framework/structure/InternalLevel.h
+++ b/include/framework/structure/InternalLevel.h
@@ -40,9 +40,14 @@ public:
         delete m_pending_shard;
     }
 
-    // WARNING: for leveling only.
-    // assuming the base level is the level new level is merging into. (base_level is larger.)
-    static std::shared_ptr<InternalLevel> merge_levels(InternalLevel* base_level, InternalLevel* new_level) {
+    /*
+     * Create a new shard combining the records from base_level and new_level,
+     * and return a shared_ptr to a new level containing this shard. This is used
+     * for reconstructions under the leveling layout policy.
+     *
+     * No changes are made to the levels provided as arguments.
+     */
+    static std::shared_ptr<InternalLevel> reconstruction(InternalLevel* base_level, InternalLevel* new_level) {
         assert(base_level->m_level_no > new_level->m_level_no || (base_level->m_level_no == 0 && new_level->m_level_no == 0));
         auto res = new InternalLevel(base_level->m_level_no, 1);
         res->m_shard_cnt = 1;
@@ -54,18 +59,15 @@ public:
         return std::shared_ptr<InternalLevel>(res);
     }
 
-    void append_buffer(Buffer* buffer) {
-        if (m_shard_cnt == m_shards.size()) {
-            assert(m_pending_shard == nullptr);
-            m_pending_shard = new S(buffer);
-            return;
-        }
-
-        m_shards[m_shard_cnt] = std::make_shared<S>(buffer);
-        ++m_shard_cnt;
-    }
-
-    void append_merged_shards(InternalLevel* level) {
+    /*
+     * Create a new shard combining the records from all of
+     * the shards in level, and append this new shard into
+     * this level. This is used for reconstructions under
+     * the tiering layout policy.
+     *
+     * No changes are made to the level provided as an argument.
+     */
+    void append_level(InternalLevel* level) {
         Shard *shards[level->m_shard_cnt];
         for (size_t i=0; i<level->m_shard_cnt; i++) {
             shards[i] = level->m_shards[i].get();
@@ -82,6 +84,22 @@ public:
         ++m_shard_cnt;
     }
 
+    /*
+     * Create a new shard using the records in the
+     * provided buffer, and append this new shard
+     * into this level. This is used for buffer
+     * flushes under the tiering layout policy.
+     */
+    void append_buffer(Buffer* buffer) {
+        if (m_shard_cnt == m_shards.size()) {
+            assert(m_pending_shard == nullptr);
+            m_pending_shard = new S(buffer);
+            return;
+        }
+
+        m_shards[m_shard_cnt] = std::make_shared<S>(buffer);
+        ++m_shard_cnt;
+    }
 
     void finalize() {
         if (m_pending_shard) {
@@ -95,7 +113,13 @@ public:
         }
     }
 
-    Shard *get_merged_shard() {
+    /*
+     * Create a new shard containing the combined records
+     * from all shards on this level and return it.
+     *
+     * No changes are made to this level.
+     */
+    Shard *get_combined_shard() {
         if (m_shard_cnt == 0) {
             return nullptr;
         }
@@ -109,7 +133,7 @@ public:
         return new S(shards, m_shard_cnt);
     }
 
-    // Append the sample range in-order.....
+    /* Append the sample range in-order */
     void get_query_states(std::vector<std::pair<ShardID, Shard *>> &shards, std::vector<void*>& shard_states, void *query_parms) {
         for (size_t i=0; i<m_shard_cnt; i++) {
             if (m_shards[i]) {
-- 
cgit v1.2.3


From 46dfe1e0f3bb05016da14b39b2e35babbba4027a Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Thu, 21 Dec 2023 17:05:30 -0500
Subject: InternalLevel: appending an empty level is a no-op

The existing reconstruction logic will occasionally attempt to append an
empty level to another empty level, for some reason. While the underlying
cause of this needs to be looked into, this special case should prevent
shard constructors being called with a shard count of 0 under tiering,
reducing the error handling overhead of shard code.
---
 include/framework/structure/InternalLevel.h | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'include/framework/structure/InternalLevel.h')

diff --git a/include/framework/structure/InternalLevel.h b/include/framework/structure/InternalLevel.h
index e70ed76..ee85cb3 100644
--- a/include/framework/structure/InternalLevel.h
+++ b/include/framework/structure/InternalLevel.h
@@ -68,6 +68,13 @@ public:
      * No changes are made to the level provided as an argument.
      */
     void append_level(InternalLevel* level) {
+        // FIXME: that this is happening probably means that
+        // something is going terribly wrong earlier in the
+        // reconstruction logic.
+        if (level->get_shard_count() == 0) {
+            return;
+        }
+
         Shard *shards[level->m_shard_cnt];
         for (size_t i=0; i<level->m_shard_cnt; i++) {
             shards[i] = level->m_shards[i].get();
-- 
cgit v1.2.3


From 7e503464176adbd0880373325e30a6bfd58616f0 Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Thu, 11 Jan 2024 16:31:24 -0500
Subject: InternalLevel update and tests

Plus some assorted fixes for move semantics stuff in BufferView that
accompanied these changes.
---
 include/framework/structure/InternalLevel.h | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

(limited to 'include/framework/structure/InternalLevel.h')

diff --git a/include/framework/structure/InternalLevel.h b/include/framework/structure/InternalLevel.h
index ee85cb3..b35cadd 100644
--- a/include/framework/structure/InternalLevel.h
+++ b/include/framework/structure/InternalLevel.h
@@ -16,7 +16,7 @@
 #include "framework/interface/Shard.h"
 #include "framework/interface/Query.h"
 #include "framework/interface/Record.h"
-#include "framework/structure/MutableBuffer.h"
+#include "framework/structure/BufferView.h"
 
 namespace de {
 template <RecordInterface R, ShardInterface S, QueryInterface Q>
@@ -27,7 +27,7 @@ class InternalLevel;
 template <RecordInterface R, ShardInterface S, QueryInterface Q>
 class InternalLevel {
     typedef S Shard;
-    typedef MutableBuffer<R> Buffer;
+    typedef BufferView<R> BuffView;
 public:
     InternalLevel(ssize_t level_no, size_t shard_cap)
     : m_level_no(level_no)
@@ -97,14 +97,14 @@ public:
      * into this level. This is used for buffer
      * flushes under the tiering layout policy.
      */
-    void append_buffer(Buffer* buffer) {
+    void append_buffer(BuffView buffer) {
         if (m_shard_cnt == m_shards.size()) {
             assert(m_pending_shard == nullptr);
-            m_pending_shard = new S(buffer);
+            m_pending_shard = new S(std::move(buffer));
             return;
         }
 
-        m_shards[m_shard_cnt] = std::make_shared<S>(buffer);
+        m_shards[m_shard_cnt] = std::make_shared<S>(std::move(buffer));
         ++m_shard_cnt;
     }
 
@@ -140,7 +140,6 @@ public:
         return new S(shards, m_shard_cnt);
     }
 
-    /* Append the sample range in-order */
     void get_query_states(std::vector<std::pair<ShardID, Shard *>> &shards, std::vector<void*>& shard_states, void *query_parms) {
         for (size_t i=0; i<m_shard_cnt; i++) {
             if (m_shards[i]) {
-- 
cgit v1.2.3


From 138c793b0a58577713d98c98bb140cf1d9c79bee Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Wed, 17 Jan 2024 18:22:00 -0500
Subject: Multiple concurrency bug fixes

A poorly organized commit with fixes for a variety of bugs that were
causing missing records. The core problems all appear to be fixed,
though there is an outstanding problem with tombstones not being
completely canceled. A very small number are appearing in the wrong
order during the static structure test.
---
 include/framework/structure/InternalLevel.h | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

(limited to 'include/framework/structure/InternalLevel.h')

diff --git a/include/framework/structure/InternalLevel.h b/include/framework/structure/InternalLevel.h
index b35cadd..e9874e0 100644
--- a/include/framework/structure/InternalLevel.h
+++ b/include/framework/structure/InternalLevel.h
@@ -51,11 +51,10 @@ public:
         assert(base_level->m_level_no > new_level->m_level_no || (base_level->m_level_no == 0 && new_level->m_level_no == 0));
         auto res = new InternalLevel(base_level->m_level_no, 1);
         res->m_shard_cnt = 1;
-        Shard* shards[2];
-        shards[0] = base_level->m_shards[0].get();
-        shards[1] = new_level->m_shards[0].get();
+        std::vector<Shard *> shards = {base_level->m_shards[0].get(),
+                                       new_level->m_shards[0].get()};
 
-        res->m_shards[0] = std::make_shared<S>(shards, 2);
+        res->m_shards[0] = std::make_shared<S>(shards);
         return std::shared_ptr<InternalLevel>(res);
     }
 
@@ -75,17 +74,17 @@ public:
             return;
         }
 
-        Shard *shards[level->m_shard_cnt];
-        for (size_t i=0; i<level->m_shard_cnt; i++) {
-            shards[i] = level->m_shards[i].get();
+        std::vector<S*> shards;
+        for (auto shard : level->m_shards) {
+            if (shard) shards.emplace_back(shard.get());
         }
 
         if (m_shard_cnt == m_shards.size()) {
-            m_pending_shard = new S(shards, level->m_shard_cnt);
+            m_pending_shard = new S(shards);
             return;
         }
 
-        auto tmp = new S(shards, level->m_shard_cnt);
+        auto tmp = new S(shards);
         m_shards[m_shard_cnt] = std::shared_ptr<S>(tmp);
 
         ++m_shard_cnt;
@@ -131,13 +130,12 @@ public:
             return nullptr;
         }
 
-        Shard *shards[m_shard_cnt];
-
-        for (size_t i=0; i<m_shard_cnt; i++) {
-            shards[i] = m_shards[i].get();
+        std::vector<Shard *> shards;
+        for (auto shard : m_shards) {
+            if (shard) shards.emplace_back(shard.get());
         }
 
-        return new S(shards, m_shard_cnt);
+        return new S(shards);
     }
 
     void get_query_states(std::vector<std::pair<ShardID, Shard *>> &shards, std::vector<void*>& shard_states, void *query_parms) {
-- 
cgit v1.2.3


From 10b4425e842d10b7fbfa85978969ed4591d6b98e Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Wed, 7 Feb 2024 10:56:52 -0500
Subject: Fully implemented Query concept and adjusted queries to use it

---
 include/framework/structure/InternalLevel.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'include/framework/structure/InternalLevel.h')

diff --git a/include/framework/structure/InternalLevel.h b/include/framework/structure/InternalLevel.h
index e9874e0..0fd5275 100644
--- a/include/framework/structure/InternalLevel.h
+++ b/include/framework/structure/InternalLevel.h
@@ -19,12 +19,12 @@
 #include "framework/structure/BufferView.h"
 
 namespace de {
-template <RecordInterface R, ShardInterface S, QueryInterface Q>
+template <RecordInterface R, ShardInterface S, QueryInterface<R, S> Q>
 class InternalLevel;
 
 
 
-template <RecordInterface R, ShardInterface S, QueryInterface Q>
+template <RecordInterface R, ShardInterface S, QueryInterface<R, S> Q>
 class InternalLevel {
     typedef S Shard;
     typedef BufferView<R> BuffView;
-- 
cgit v1.2.3


From 2c5d549b3618b9ea72e6eece4cb4f3da5a6811a8 Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Wed, 7 Feb 2024 13:42:34 -0500
Subject: Fully realized shard concept interface

---
 include/framework/structure/InternalLevel.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'include/framework/structure/InternalLevel.h')

diff --git a/include/framework/structure/InternalLevel.h b/include/framework/structure/InternalLevel.h
index 0fd5275..d586869 100644
--- a/include/framework/structure/InternalLevel.h
+++ b/include/framework/structure/InternalLevel.h
@@ -19,12 +19,12 @@
 #include "framework/structure/BufferView.h"
 
 namespace de {
-template <RecordInterface R, ShardInterface S, QueryInterface<R, S> Q>
+template <RecordInterface R, ShardInterface<R> S, QueryInterface<R, S> Q>
 class InternalLevel;
 
 
 
-template <RecordInterface R, ShardInterface S, QueryInterface<R, S> Q>
+template <RecordInterface R, ShardInterface<R> S, QueryInterface<R, S> Q>
 class InternalLevel {
     typedef S Shard;
     typedef BufferView<R> BuffView;
-- 
cgit v1.2.3


From 402fc269c0aaa671d84a6d15918735ad4b90e6b2 Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Fri, 9 Feb 2024 12:30:21 -0500
Subject: Comment updates/fixes

---
 include/framework/structure/InternalLevel.h | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'include/framework/structure/InternalLevel.h')

diff --git a/include/framework/structure/InternalLevel.h b/include/framework/structure/InternalLevel.h
index d586869..db38946 100644
--- a/include/framework/structure/InternalLevel.h
+++ b/include/framework/structure/InternalLevel.h
@@ -6,6 +6,12 @@
  *
  * Distributed under the Modified BSD License.
  *
+ * The word `Internal` in this class's name refers to memory. The current
+ * model, inherited from the framework in Practical Dynamic Extension for
+ * Sampling Indexes, would use a different ExternalLevel for shards stored
+ * on external storage. This is a distinction that can probably be avoided
+ * with some more thought being put into interface design.
+ *
  */
 #pragma once
 
-- 
cgit v1.2.3