From 7c03d771475421c1d5a2bbc135242536af1a371c Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Mon, 25 Sep 2023 10:49:36 -0400
Subject: Re-structuring Project + scheduling updates

This is a big one--probably should have split it apart, but I'm feeling
lazy this morning.

* Organized the mess of header files in include/framework by splitting
them out into their own subdirectories, and renaming a few files to
remove redundancies introduced by the directory structure.

* Introduced a new framework/ShardRequirements.h header file for simpler
shard development. This header simply contains the necessary includes
from framework/* for creating shard files. This should help to remove
structural dependencies from the framework file structure and shards,
as well as centralizing the necessary framework files to make shard
development easier.

* Created a (currently dummy) SchedulerInterface, and make the scheduler
implementation a template parameter of the dynamic extension for easier
testing of various scheduling policies. There's still more work to be
done to fully integrate the scheduler (queries, multiple buffers), but
some more of the necessary framework code for this has been added as well.

* Adjusted the Task interface setup for the scheduler. The task structures
have been removed from ExtensionStructure and placed in their own header
file. Additionally, I started experimenting with using std::variant,
as opposed to inheritence, to implement subtype polymorphism on the
Merge and Query tasks. The scheduler now has a general task queue that
contains both, and std::variant, std::visit, and std::get are used to
manipulate them without virtual functions.

* Removed Alex.h, as it can't build anyway. There's a branch out there
containing the Alex implementation stripped of the C++20 stuff. So
there's no need to keep it here.
---
 include/framework/interface/Query.h     |  35 ++++++
 include/framework/interface/Record.h    | 214 ++++++++++++++++++++++++++++++++
 include/framework/interface/Scheduler.h |  31 +++++
 include/framework/interface/Shard.h     |  26 ++++
 4 files changed, 306 insertions(+)
 create mode 100644 include/framework/interface/Query.h
 create mode 100644 include/framework/interface/Record.h
 create mode 100644 include/framework/interface/Scheduler.h
 create mode 100644 include/framework/interface/Shard.h

(limited to 'include/framework/interface')

diff --git a/include/framework/interface/Query.h b/include/framework/interface/Query.h
new file mode 100644
index 0000000..46a1ce1
--- /dev/null
+++ b/include/framework/interface/Query.h
@@ -0,0 +1,35 @@
+/*
+ * include/framework/QueryInterface.h
+ *
+ * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> 
+ *
+ * All rights reserved. Published under the Modified BSD License.
+ *
+ */
+#pragma once
+
+#include <vector>
+#include <concepts>
+#include "util/types.h"
+
+template <typename Q>
+concept QueryInterface = requires(Q q, void *p, std::vector<void*> &s) {
+
+/*
+    {q.get_query_state(p, p)} -> std::convertible_to<void*>;
+    {q.get_buffer_query_state(p, p)};
+    {q.query(p, p)};
+    {q.buffer_query(p, p)};
+    {q.merge()};
+    {q.delete_query_state(p)};
+*/
+    {Q::EARLY_ABORT} -> std::convertible_to<bool>;
+    {Q::SKIP_DELETE_FILTER} -> std::convertible_to<bool>;
+    //{Q::get_query_state(p, p)} -> std::convertible_to<void*>;
+    //{Q::get_buffer_query_state(p, p)} -> std::convertible_to<void*>;
+    {Q::process_query_states(p, s, p)};
+
+    {Q::delete_query_state(std::declval<void*>())} -> std::same_as<void>;
+    {Q::delete_buffer_query_state(p)};
+
+};
diff --git a/include/framework/interface/Record.h b/include/framework/interface/Record.h
new file mode 100644
index 0000000..1ef1984
--- /dev/null
+++ b/include/framework/interface/Record.h
@@ -0,0 +1,214 @@
+/*
+ * include/framework/RecordInterface.h
+ *
+ * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu>
+ *                    Dong Xie <dongx@psu.edu>
+ *
+ * All rights reserved. Published under the Modified BSD License.
+ *
+ */
+#pragma once
+
+#include <cstring>
+#include <concepts>
+#include <cmath>
+
+#include "psu-util/hash.h"
+
+namespace de {
+
+template<typename R>
+concept RecordInterface = requires(R r, R s) {
+    { r < s } ->std::convertible_to<bool>;
+    { r == s } ->std::convertible_to<bool>;
+};
+
+template<typename R>
+concept WeightedRecordInterface = requires(R r) {
+    {r.weight} -> std::convertible_to<double>;
+};
+
+template<typename R>
+concept NDRecordInterface = RecordInterface<R> && requires(R r, R s) {
+    {r.calc_distance(s)} -> std::convertible_to<double>;
+};
+
+template <typename R>
+concept KVPInterface = RecordInterface<R> && requires(R r) {
+    r.key;
+    r.value;
+};
+
+template<typename R>
+concept AlexInterface = KVPInterface<R> && requires(R r) {
+    {r.key} -> std::convertible_to<size_t>;
+    {r.value} -> std::convertible_to<size_t>;
+};
+
+template<typename R>
+concept WrappedInterface = RecordInterface<R> && requires(R r, R s, bool b) {
+    {r.header} -> std::convertible_to<uint32_t>;
+     r.rec;
+    {r.set_delete()};
+    {r.is_deleted()} -> std::convertible_to<bool>;
+    {r.set_tombstone(b)};
+    {r.is_tombstone()} -> std::convertible_to<bool>;
+    {r < s} -> std::convertible_to<bool>;
+    {r == s} ->std::convertible_to<bool>;
+};
+
+template<RecordInterface R>
+struct Wrapped {
+    uint32_t header;
+    R rec;
+
+    inline void set_delete() {
+        header |= 2;
+    }
+
+    inline bool is_deleted() const {
+        return header & 2;
+    }
+
+    inline void set_tombstone(bool val=true) {
+        if (val) {
+            header |= val;
+        } else {
+            header &= 0;
+        }
+    }
+
+    inline bool is_tombstone() const {
+        return header & 1;
+    }
+
+    inline bool operator<(const Wrapped& other) const {
+        return rec < other.rec || (rec == other.rec && header < other.header);
+    }
+
+    inline bool operator==(const Wrapped& other) const {
+        return rec == other.rec;
+    }
+
+};
+
+template <typename K, typename V>
+struct Record {
+    K key;
+    V value;
+    uint32_t header = 0;
+
+       inline bool operator<(const Record& other) const {
+        return key < other.key || (key == other.key && value < other.value);
+    }
+
+    inline bool operator==(const Record& other) const {
+        return key == other.key && value == other.value;
+    }
+};
+
+template <typename K, typename V, typename W>
+struct WeightedRecord {
+    K key;
+    V value;
+    W weight = 1;
+
+    inline bool operator==(const WeightedRecord& other) const {
+        return key == other.key && value == other.value;
+    }
+
+   inline bool operator<(const WeightedRecord& other) const {
+        return key < other.key || (key == other.key && value < other.value);
+    }
+};
+
+
+template <typename V, size_t D=2>
+struct CosinePoint{
+    V data[D];
+
+    inline bool operator==(const CosinePoint& other) const {
+        for (size_t i=0; i<D; i++) {
+            if (data[i] != other.data[i]) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    // lexicographic order
+    inline bool operator<(const CosinePoint& other) const {
+        for (size_t i=0; i<D; i++) {
+            if (data[i] < other.data[i]) {
+                return true;
+            } else if (data[i] > other.data[i]) {
+                return false;
+            }
+        }
+
+        return false;
+    }
+
+    inline double calc_distance(const CosinePoint& other) const {
+
+        double prod = 0;
+        double asquared = 0;
+        double bsquared = 0;
+
+        for (size_t i=0; i<D; i++) {
+            prod += data[i] * other.data[i];
+            asquared += data[i]*data[i];
+            bsquared += other.data[i]*other.data[i];
+        }
+
+        return prod / std::sqrt(asquared * bsquared);
+    }
+};
+
+
+template <typename V, size_t D=2>
+struct EuclidPoint{
+    V data[D];
+
+    inline bool operator==(const EuclidPoint& other) const {
+        for (size_t i=0; i<D; i++) {
+            if (data[i] != other.data[i]) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    // lexicographic order
+    inline bool operator<(const EuclidPoint& other) const {
+        for (size_t i=0; i<D; i++) {
+            if (data[i] < other.data[i]) {
+                return true;
+            } else if (data[i] > other.data[i]) {
+                return false;
+            }
+        }
+
+        return false;
+    }
+
+    inline double calc_distance(const EuclidPoint& other) const {
+        double dist = 0;
+        for (size_t i=0; i<D; i++) {
+            dist += (data[i] - other.data[i]) * (data[i] - other.data[i]);
+        }
+        
+        return std::sqrt(dist);
+    }
+};
+
+template<RecordInterface R>
+struct RecordHash {
+    size_t operator()(R const &rec) const {
+        return psudb::hash_bytes((std::byte *) &rec, sizeof(R));
+    }
+};
+
+}
diff --git a/include/framework/interface/Scheduler.h b/include/framework/interface/Scheduler.h
new file mode 100644
index 0000000..1445e90
--- /dev/null
+++ b/include/framework/interface/Scheduler.h
@@ -0,0 +1,31 @@
+/*
+ * include/framework/QueryInterface.h
+ *
+ * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> 
+ *
+ * All rights reserved. Published under the Modified BSD License.
+ *
+ */
+#pragma once
+
+#include <vector>
+#include <concepts>
+#include "framework/interface/Record.h"
+#include "util/types.h"
+
+template <typename S>
+concept SchedulerInterface = requires(S s, size_t i, void *vp) {
+    {S(i, i)};
+//    {s.schedule_merge(vp, vp)};
+    
+/*
+    {q.get_query_state(p, p)} -> std::convertible_to<void*>;
+    {q.get_buffer_query_state(p, p)};
+    {q.query(p, p)};
+    {q.buffer_query(p, p)};
+    {q.merge()};
+    {q.delete_query_state(p)};
+*/
+    //{Q::get_query_state(p, p)} -> std::convertible_to<void*>;
+    //{Q::get_buffer_query_state(p, p)} -> std::convertible_to<void*>;
+};
diff --git a/include/framework/interface/Shard.h b/include/framework/interface/Shard.h
new file mode 100644
index 0000000..ea58b2a
--- /dev/null
+++ b/include/framework/interface/Shard.h
@@ -0,0 +1,26 @@
+/*
+ * include/framework/ShardInterface.h
+ *
+ * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> 
+ *
+ * All rights reserved. Published under the Modified BSD License.
+ *
+ */
+#pragma once
+
+#include <concepts>
+
+#include "util/types.h"
+#include "framework/interface/Record.h"
+
+namespace de {
+
+//template <template<typename> typename S, typename R>
+template <typename S>
+concept ShardInterface = requires(S s, void *p, bool b) {
+    //{s.point_lookup(r, b) } -> std::same_as<R*>;
+    {s.get_record_count()} -> std::convertible_to<size_t>;
+    {s.get_memory_usage()} -> std::convertible_to<size_t>;
+};
+
+}
-- 
cgit v1.2.3


From 7ecfb22c32b7986ed1a2439c1abbeed298e4153a Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Fri, 20 Oct 2023 17:00:42 -0400
Subject: Initial pass w/ new scheduler setup

currently there's a race condition of some type to sort out.
---
 include/framework/interface/Scheduler.h | 17 ++++-------------
 1 file changed, 4 insertions(+), 13 deletions(-)

(limited to 'include/framework/interface')

diff --git a/include/framework/interface/Scheduler.h b/include/framework/interface/Scheduler.h
index 1445e90..e8ffd08 100644
--- a/include/framework/interface/Scheduler.h
+++ b/include/framework/interface/Scheduler.h
@@ -12,20 +12,11 @@
 #include <concepts>
 #include "framework/interface/Record.h"
 #include "util/types.h"
+#include "framework/scheduling/Task.h"
 
 template <typename S>
-concept SchedulerInterface = requires(S s, size_t i, void *vp) {
+concept SchedulerInterface = requires(S s, size_t i, void *vp, de::Job j) {
     {S(i, i)};
-//    {s.schedule_merge(vp, vp)};
-    
-/*
-    {q.get_query_state(p, p)} -> std::convertible_to<void*>;
-    {q.get_buffer_query_state(p, p)};
-    {q.query(p, p)};
-    {q.buffer_query(p, p)};
-    {q.merge()};
-    {q.delete_query_state(p)};
-*/
-    //{Q::get_query_state(p, p)} -> std::convertible_to<void*>;
-    //{Q::get_buffer_query_state(p, p)} -> std::convertible_to<void*>;
+    {s.schedule_job(j, i, vp)} -> std::convertible_to<void>;
+    {s.shutdown()};
 };
-- 
cgit v1.2.3


From 3afacb7702e6d8fa67749a2a41dc776d315e02a9 Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Mon, 23 Oct 2023 17:43:22 -0400
Subject: Began moving to an explicit epoch-based system

I started moving over to an explicit Epoch based system, which has
necessitated a ton of changes throughout the code base. This will
ultimately allow for a much cleaner set of abstractions for managing
concurrency.
---
 include/framework/interface/Query.h | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'include/framework/interface')

diff --git a/include/framework/interface/Query.h b/include/framework/interface/Query.h
index 46a1ce1..9b1d2d6 100644
--- a/include/framework/interface/Query.h
+++ b/include/framework/interface/Query.h
@@ -14,7 +14,6 @@
 
 template <typename Q>
 concept QueryInterface = requires(Q q, void *p, std::vector<void*> &s) {
-
 /*
     {q.get_query_state(p, p)} -> std::convertible_to<void*>;
     {q.get_buffer_query_state(p, p)};
@@ -27,7 +26,7 @@ concept QueryInterface = requires(Q q, void *p, std::vector<void*> &s) {
     {Q::SKIP_DELETE_FILTER} -> std::convertible_to<bool>;
     //{Q::get_query_state(p, p)} -> std::convertible_to<void*>;
     //{Q::get_buffer_query_state(p, p)} -> std::convertible_to<void*>;
-    {Q::process_query_states(p, s, p)};
+    {Q::process_query_states(p, s, s)};
 
     {Q::delete_query_state(std::declval<void*>())} -> std::same_as<void>;
     {Q::delete_buffer_query_state(p)};
-- 
cgit v1.2.3


From d2279e1b96d352a0af1d425dcaaf93e8a26a8d52 Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Mon, 30 Oct 2023 17:15:05 -0400
Subject: General Comment + Consistency updates

---
 include/framework/interface/Query.h     | 34 ++++++++++++++++++---------------
 include/framework/interface/Record.h    |  5 +++--
 include/framework/interface/Scheduler.h |  4 ++--
 include/framework/interface/Shard.h     | 20 ++++++++++++++-----
 4 files changed, 39 insertions(+), 24 deletions(-)

(limited to 'include/framework/interface')

diff --git a/include/framework/interface/Query.h b/include/framework/interface/Query.h
index 9b1d2d6..21cadcb 100644
--- a/include/framework/interface/Query.h
+++ b/include/framework/interface/Query.h
@@ -1,7 +1,7 @@
 /*
- * include/framework/QueryInterface.h
+ * include/framework/interface/Query.h
  *
- * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> 
+ * Copyright (C) 2023 Douglas B. Rumbaugh <drumbaugh@psu.edu> 
  *
  * All rights reserved. Published under the Modified BSD License.
  *
@@ -10,25 +10,29 @@
 
 #include <vector>
 #include <concepts>
+
 #include "util/types.h"
 
+// FIXME: The interface is not completely specified yet, as it is pending
+//        determining a good way to handle additional template arguments 
+//        to get the Shard and Record types into play
 template <typename Q>
 concept QueryInterface = requires(Q q, void *p, std::vector<void*> &s) {
-/*
-    {q.get_query_state(p, p)} -> std::convertible_to<void*>;
-    {q.get_buffer_query_state(p, p)};
-    {q.query(p, p)};
-    {q.buffer_query(p, p)};
-    {q.merge()};
-    {q.delete_query_state(p)};
-*/
-    {Q::EARLY_ABORT} -> std::convertible_to<bool>;
-    {Q::SKIP_DELETE_FILTER} -> std::convertible_to<bool>;
-    //{Q::get_query_state(p, p)} -> std::convertible_to<void*>;
-    //{Q::get_buffer_query_state(p, p)} -> std::convertible_to<void*>;
+
+    /*
+    {Q::get_query_state(p, p)} -> std::convertible_to<void*>;
+    {Q::get_buffer_query_state(p, p)} -> std::convertible_to<void *>;
+    */
     {Q::process_query_states(p, s, s)};
+    /*
+    {Q::query(s, p, p)} -> std::convertible_to<std::vector<Wrapped<R>>>;
+    {Q::buffer_query(p, p)} -> std::convertible_to<std::vector<Wrapped<R>>>;
+    {Q::merge(rv, p)} -> std::convertible_to<std::vector<R>>;
+    */
 
     {Q::delete_query_state(std::declval<void*>())} -> std::same_as<void>;
-    {Q::delete_buffer_query_state(p)};
+    {Q::delete_buffer_query_state(std::declval<void*>())} -> std::same_as<void>;
 
+    {Q::EARLY_ABORT} -> std::convertible_to<bool>;
+    {Q::SKIP_DELETE_FILTER} -> std::convertible_to<bool>;
 };
diff --git a/include/framework/interface/Record.h b/include/framework/interface/Record.h
index 1ef1984..bf495df 100644
--- a/include/framework/interface/Record.h
+++ b/include/framework/interface/Record.h
@@ -1,11 +1,12 @@
 /*
- * include/framework/RecordInterface.h
+ * include/framework/interface/Record.h
  *
  * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu>
- *                    Dong Xie <dongx@psu.edu>
  *
  * All rights reserved. Published under the Modified BSD License.
  *
+ * FIXME: the record implementations could probably be broken out into 
+ *        different files, leaving only the interface here
  */
 #pragma once
 
diff --git a/include/framework/interface/Scheduler.h b/include/framework/interface/Scheduler.h
index e8ffd08..63581d2 100644
--- a/include/framework/interface/Scheduler.h
+++ b/include/framework/interface/Scheduler.h
@@ -1,7 +1,7 @@
 /*
- * include/framework/QueryInterface.h
+ * include/framework/interface/Scheduler.h
  *
- * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> 
+ * Copyright (C) 2023 Douglas B. Rumbaugh <drumbaugh@psu.edu> 
  *
  * All rights reserved. Published under the Modified BSD License.
  *
diff --git a/include/framework/interface/Shard.h b/include/framework/interface/Shard.h
index ea58b2a..d3a6cf8 100644
--- a/include/framework/interface/Shard.h
+++ b/include/framework/interface/Shard.h
@@ -1,7 +1,7 @@
 /*
- * include/framework/ShardInterface.h
+ * include/framework/interface/Shard.h
  *
- * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> 
+ * Copyright (C) 2023 Douglas B. Rumbaugh <drumbaugh@psu.edu> 
  *
  * All rights reserved. Published under the Modified BSD License.
  *
@@ -15,12 +15,22 @@
 
 namespace de {
 
-//template <template<typename> typename S, typename R>
+// FIXME: The interface is not completely specified yet, as it is pending
+//        determining a good way to handle additional template arguments 
+//        to get the Record type into play
 template <typename S>
-concept ShardInterface = requires(S s, void *p, bool b) {
-    //{s.point_lookup(r, b) } -> std::same_as<R*>;
+concept ShardInterface = requires(S s, S **spp, void *p, bool b, size_t i) {
+    {S(spp, i)};
+    /*
+    {S(mutable buffer)}
+    {s.point_lookup(r, b) } -> std::convertible_to<void*>
+    */
+    {s.get_data()} -> std::convertible_to<void*>;
+
     {s.get_record_count()} -> std::convertible_to<size_t>;
+    {s.get_tombstone_count()} -> std::convertible_to<size_t>;
     {s.get_memory_usage()} -> std::convertible_to<size_t>;
+    {s.get_aux_memory_usage()} -> std::convertible_to<size_t>;
 };
 
 }
-- 
cgit v1.2.3


From 0b723322a611de83872dd83b55d2e10e8886a283 Mon Sep 17 00:00:00 2001
From: "Douglas B. Rumbaugh" <douglas@t480.douglasrumbaugh.com>
Date: Thu, 2 Nov 2023 08:01:45 -0400
Subject: started refactoring queries interface

---
 include/framework/interface/Shard.h | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'include/framework/interface')

diff --git a/include/framework/interface/Shard.h b/include/framework/interface/Shard.h
index d3a6cf8..40a696b 100644
--- a/include/framework/interface/Shard.h
+++ b/include/framework/interface/Shard.h
@@ -33,4 +33,10 @@ concept ShardInterface = requires(S s, S **spp, void *p, bool b, size_t i) {
     {s.get_aux_memory_usage()} -> std::convertible_to<size_t>;
 };
 
+template <typename S, typename R>
+concept SortedShardInterface = ShardInterface<S> && requires(S s, R r, R *rp) {
+    {s.lower_bound(r)} -> std::convertible_to<size_t>;
+    {s.upper_bound(r)} -> std::convertible_to<size_t>;
+}
+
 }
-- 
cgit v1.2.3


From e02742b07540dd5a9bcbb44dae14856bf10955ed Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Mon, 6 Nov 2023 15:18:53 -0500
Subject: Refactoring progress

---
 include/framework/interface/Shard.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'include/framework/interface')

diff --git a/include/framework/interface/Shard.h b/include/framework/interface/Shard.h
index 40a696b..92cdca0 100644
--- a/include/framework/interface/Shard.h
+++ b/include/framework/interface/Shard.h
@@ -37,6 +37,6 @@ template <typename S, typename R>
 concept SortedShardInterface = ShardInterface<S> && requires(S s, R r, R *rp) {
     {s.lower_bound(r)} -> std::convertible_to<size_t>;
     {s.upper_bound(r)} -> std::convertible_to<size_t>;
-}
+};
 
 }
-- 
cgit v1.2.3


From 357cab549c2ed33970562b84ff6f83923742343d Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Tue, 7 Nov 2023 15:34:24 -0500
Subject: Comment and License updates

---
 include/framework/interface/Query.h     | 2 +-
 include/framework/interface/Record.h    | 2 +-
 include/framework/interface/Scheduler.h | 2 +-
 include/framework/interface/Shard.h     | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

(limited to 'include/framework/interface')

diff --git a/include/framework/interface/Query.h b/include/framework/interface/Query.h
index 21cadcb..8b92c45 100644
--- a/include/framework/interface/Query.h
+++ b/include/framework/interface/Query.h
@@ -3,7 +3,7 @@
  *
  * Copyright (C) 2023 Douglas B. Rumbaugh <drumbaugh@psu.edu> 
  *
- * All rights reserved. Published under the Modified BSD License.
+ * Distributed under the Modified BSD License.
  *
  */
 #pragma once
diff --git a/include/framework/interface/Record.h b/include/framework/interface/Record.h
index bf495df..457078d 100644
--- a/include/framework/interface/Record.h
+++ b/include/framework/interface/Record.h
@@ -3,7 +3,7 @@
  *
  * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu>
  *
- * All rights reserved. Published under the Modified BSD License.
+ * Distributed under the Modified BSD License.
  *
  * FIXME: the record implementations could probably be broken out into 
  *        different files, leaving only the interface here
diff --git a/include/framework/interface/Scheduler.h b/include/framework/interface/Scheduler.h
index 63581d2..a8544a7 100644
--- a/include/framework/interface/Scheduler.h
+++ b/include/framework/interface/Scheduler.h
@@ -3,7 +3,7 @@
  *
  * Copyright (C) 2023 Douglas B. Rumbaugh <drumbaugh@psu.edu> 
  *
- * All rights reserved. Published under the Modified BSD License.
+ * Distributed under the Modified BSD License.
  *
  */
 #pragma once
diff --git a/include/framework/interface/Shard.h b/include/framework/interface/Shard.h
index 92cdca0..2357795 100644
--- a/include/framework/interface/Shard.h
+++ b/include/framework/interface/Shard.h
@@ -3,7 +3,7 @@
  *
  * Copyright (C) 2023 Douglas B. Rumbaugh <drumbaugh@psu.edu> 
  *
- * All rights reserved. Published under the Modified BSD License.
+ * Distributed under the Modified BSD License.
  *
  */
 #pragma once
-- 
cgit v1.2.3


From aac0bb661af8fae38d3ce08d6078cb4d9dfcb575 Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Fri, 12 Jan 2024 14:10:11 -0500
Subject: Initial integration of new buffering scheme into framework

It isn't working right now (lotsa test failures), but we're to the
debugging phase now.
---
 include/framework/interface/Query.h     | 2 +-
 include/framework/interface/Scheduler.h | 4 ----
 2 files changed, 1 insertion(+), 5 deletions(-)

(limited to 'include/framework/interface')

diff --git a/include/framework/interface/Query.h b/include/framework/interface/Query.h
index 8b92c45..ca742c3 100644
--- a/include/framework/interface/Query.h
+++ b/include/framework/interface/Query.h
@@ -23,7 +23,7 @@ concept QueryInterface = requires(Q q, void *p, std::vector<void*> &s) {
     {Q::get_query_state(p, p)} -> std::convertible_to<void*>;
     {Q::get_buffer_query_state(p, p)} -> std::convertible_to<void *>;
     */
-    {Q::process_query_states(p, s, s)};
+    {Q::process_query_states(p, s, p)};
     /*
     {Q::query(s, p, p)} -> std::convertible_to<std::vector<Wrapped<R>>>;
     {Q::buffer_query(p, p)} -> std::convertible_to<std::vector<Wrapped<R>>>;
diff --git a/include/framework/interface/Scheduler.h b/include/framework/interface/Scheduler.h
index a8544a7..94afe6c 100644
--- a/include/framework/interface/Scheduler.h
+++ b/include/framework/interface/Scheduler.h
@@ -8,10 +8,6 @@
  */
 #pragma once
 
-#include <vector>
-#include <concepts>
-#include "framework/interface/Record.h"
-#include "util/types.h"
 #include "framework/scheduling/Task.h"
 
 template <typename S>
-- 
cgit v1.2.3


From 138c793b0a58577713d98c98bb140cf1d9c79bee Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Wed, 17 Jan 2024 18:22:00 -0500
Subject: Multiple concurrency bug fixes

A poorly organized commit with fixes for a variety of bugs that were
causing missing records. The core problems all appear to be fixed,
though there is an outstanding problem with tombstones not being
completely canceled. A very small number are appearing in the wrong
order during the static structure test.
---
 include/framework/interface/Shard.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'include/framework/interface')

diff --git a/include/framework/interface/Shard.h b/include/framework/interface/Shard.h
index 2357795..8c4db34 100644
--- a/include/framework/interface/Shard.h
+++ b/include/framework/interface/Shard.h
@@ -12,6 +12,7 @@
 
 #include "util/types.h"
 #include "framework/interface/Record.h"
+#include <vector>
 
 namespace de {
 
@@ -19,8 +20,8 @@ namespace de {
 //        determining a good way to handle additional template arguments 
 //        to get the Record type into play
 template <typename S>
-concept ShardInterface = requires(S s, S **spp, void *p, bool b, size_t i) {
-    {S(spp, i)};
+concept ShardInterface = requires(S s, std::vector<S*> spp, void *p, bool b, size_t i) {
+    {S(spp)};
     /*
     {S(mutable buffer)}
     {s.point_lookup(r, b) } -> std::convertible_to<void*>
-- 
cgit v1.2.3


From 38693c342558628c75e0ab0d23c32a95a499ed8b Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Fri, 19 Jan 2024 15:58:04 -0500
Subject: Initial rough-out of internal statistics tracker

Need to figure out the best way to do the detailed tracking in
a concurrent manner. I was thinking just an event log, with parsing
routines for extracting statistics. But that'll be pretty slow.
---
 include/framework/interface/Scheduler.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'include/framework/interface')

diff --git a/include/framework/interface/Scheduler.h b/include/framework/interface/Scheduler.h
index 94afe6c..451ddd2 100644
--- a/include/framework/interface/Scheduler.h
+++ b/include/framework/interface/Scheduler.h
@@ -13,6 +13,7 @@
 template <typename S>
 concept SchedulerInterface = requires(S s, size_t i, void *vp, de::Job j) {
     {S(i, i)};
-    {s.schedule_job(j, i, vp)} -> std::convertible_to<void>;
+    {s.schedule_job(j, i, vp, i)} -> std::convertible_to<void>;
     {s.shutdown()};
+    {s.print_statistics()};
 };
-- 
cgit v1.2.3


From 10b4425e842d10b7fbfa85978969ed4591d6b98e Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Wed, 7 Feb 2024 10:56:52 -0500
Subject: Fully implemented Query concept and adjusted queries to use it

---
 include/framework/interface/Query.h | 25 ++++++++++---------------
 1 file changed, 10 insertions(+), 15 deletions(-)

(limited to 'include/framework/interface')

diff --git a/include/framework/interface/Query.h b/include/framework/interface/Query.h
index ca742c3..8cf9660 100644
--- a/include/framework/interface/Query.h
+++ b/include/framework/interface/Query.h
@@ -8,31 +8,26 @@
  */
 #pragma once
 
-#include <vector>
+#include "framework/QueryRequirements.h"
 #include <concepts>
 
-#include "util/types.h"
-
+namespace de{
 // FIXME: The interface is not completely specified yet, as it is pending
 //        determining a good way to handle additional template arguments 
 //        to get the Shard and Record types into play
-template <typename Q>
-concept QueryInterface = requires(Q q, void *p, std::vector<void*> &s) {
-
-    /*
-    {Q::get_query_state(p, p)} -> std::convertible_to<void*>;
-    {Q::get_buffer_query_state(p, p)} -> std::convertible_to<void *>;
-    */
+template <typename Q, typename R, typename S>
+concept QueryInterface = requires(void *p, S *sh, std::vector<void*> &s, std::vector<std::vector<Wrapped<R>>> &rv, BufferView<R> *bv) {
+    {Q::get_query_state(sh, p)} -> std::convertible_to<void*>;
+    {Q::get_buffer_query_state(bv, p)} -> std::convertible_to<void *>;
     {Q::process_query_states(p, s, p)};
-    /*
-    {Q::query(s, p, p)} -> std::convertible_to<std::vector<Wrapped<R>>>;
+    {Q::query(sh, p, p)} -> std::convertible_to<std::vector<Wrapped<R>>>;
     {Q::buffer_query(p, p)} -> std::convertible_to<std::vector<Wrapped<R>>>;
     {Q::merge(rv, p)} -> std::convertible_to<std::vector<R>>;
-    */
 
-    {Q::delete_query_state(std::declval<void*>())} -> std::same_as<void>;
-    {Q::delete_buffer_query_state(std::declval<void*>())} -> std::same_as<void>;
+    {Q::delete_query_state(p)} -> std::same_as<void>;
+    {Q::delete_buffer_query_state(p)} -> std::same_as<void>;
 
     {Q::EARLY_ABORT} -> std::convertible_to<bool>;
     {Q::SKIP_DELETE_FILTER} -> std::convertible_to<bool>;
 };
+}
-- 
cgit v1.2.3


From 2c5d549b3618b9ea72e6eece4cb4f3da5a6811a8 Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Wed, 7 Feb 2024 13:42:34 -0500
Subject: Fully realized shard concept interface

---
 include/framework/interface/Shard.h | 25 +++++++++----------------
 1 file changed, 9 insertions(+), 16 deletions(-)

(limited to 'include/framework/interface')

diff --git a/include/framework/interface/Shard.h b/include/framework/interface/Shard.h
index 8c4db34..c4a9180 100644
--- a/include/framework/interface/Shard.h
+++ b/include/framework/interface/Shard.h
@@ -8,25 +8,17 @@
  */
 #pragma once
 
-#include <concepts>
-
-#include "util/types.h"
-#include "framework/interface/Record.h"
-#include <vector>
+#include "framework/ShardRequirements.h"
 
 namespace de {
 
-// FIXME: The interface is not completely specified yet, as it is pending
-//        determining a good way to handle additional template arguments 
-//        to get the Record type into play
-template <typename S>
-concept ShardInterface = requires(S s, std::vector<S*> spp, void *p, bool b, size_t i) {
+template <typename S, typename R>
+concept ShardInterface = RecordInterface<R> && requires(S s, std::vector<S*> spp, void *p, bool b, size_t i, BufferView<R> bv, R r) {
     {S(spp)};
-    /*
-    {S(mutable buffer)}
-    {s.point_lookup(r, b) } -> std::convertible_to<void*>
-    */
-    {s.get_data()} -> std::convertible_to<void*>;
+    {S(std::move(bv))};
+
+    {s.point_lookup(r, b) } -> std::same_as<Wrapped<R>*>;
+    {s.get_data()} -> std::same_as<Wrapped<R>*>;
 
     {s.get_record_count()} -> std::convertible_to<size_t>;
     {s.get_tombstone_count()} -> std::convertible_to<size_t>;
@@ -35,9 +27,10 @@ concept ShardInterface = requires(S s, std::vector<S*> spp, void *p, bool b, siz
 };
 
 template <typename S, typename R>
-concept SortedShardInterface = ShardInterface<S> && requires(S s, R r, R *rp) {
+concept SortedShardInterface = ShardInterface<S, R> && requires(S s, R r, R *rp, size_t i) {
     {s.lower_bound(r)} -> std::convertible_to<size_t>;
     {s.upper_bound(r)} -> std::convertible_to<size_t>;
+    {s.get_record_at(i)} -> std::same_as<Wrapped<R>*>;
 };
 
 }
-- 
cgit v1.2.3


From 711769574e647839677739192698e400529efe75 Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Thu, 8 Feb 2024 16:38:44 -0500
Subject: Updated VPTree to new shard/query interfaces

---
 include/framework/interface/Record.h | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

(limited to 'include/framework/interface')

diff --git a/include/framework/interface/Record.h b/include/framework/interface/Record.h
index 457078d..29df4b6 100644
--- a/include/framework/interface/Record.h
+++ b/include/framework/interface/Record.h
@@ -212,4 +212,23 @@ struct RecordHash {
     }
 };
 
+template <typename R>
+class DistCmpMax {
+public:
+    DistCmpMax(R *baseline) : P(baseline) {}
+
+    inline bool operator()(const R *a, const R *b) requires WrappedInterface<R> {
+        return a->rec.calc_distance(P->rec) > b->rec.calc_distance(P->rec); 
+    }
+
+    inline bool operator()(const R *a, const R *b) requires (!WrappedInterface<R>){
+        return a->calc_distance(*P) > b->calc_distance(*P); 
+    }
+
+private:
+    R *P;
+};
+
+
+
 }
-- 
cgit v1.2.3


From 402fc269c0aaa671d84a6d15918735ad4b90e6b2 Mon Sep 17 00:00:00 2001
From: Douglas Rumbaugh <dbr4@psu.edu>
Date: Fri, 9 Feb 2024 12:30:21 -0500
Subject: Comment updates/fixes

---
 include/framework/interface/Query.h  | 5 +----
 include/framework/interface/Record.h | 7 ++-----
 2 files changed, 3 insertions(+), 9 deletions(-)

(limited to 'include/framework/interface')

diff --git a/include/framework/interface/Query.h b/include/framework/interface/Query.h
index 8cf9660..3d487f0 100644
--- a/include/framework/interface/Query.h
+++ b/include/framework/interface/Query.h
@@ -9,12 +9,9 @@
 #pragma once
 
 #include "framework/QueryRequirements.h"
-#include <concepts>
 
 namespace de{
-// FIXME: The interface is not completely specified yet, as it is pending
-//        determining a good way to handle additional template arguments 
-//        to get the Shard and Record types into play
+
 template <typename Q, typename R, typename S>
 concept QueryInterface = requires(void *p, S *sh, std::vector<void*> &s, std::vector<std::vector<Wrapped<R>>> &rv, BufferView<R> *bv) {
     {Q::get_query_state(sh, p)} -> std::convertible_to<void*>;
diff --git a/include/framework/interface/Record.h b/include/framework/interface/Record.h
index 29df4b6..5b9f307 100644
--- a/include/framework/interface/Record.h
+++ b/include/framework/interface/Record.h
@@ -138,7 +138,7 @@ struct CosinePoint{
         return true;
     }
 
-    // lexicographic order
+    /* lexicographic order */
     inline bool operator<(const CosinePoint& other) const {
         for (size_t i=0; i<D; i++) {
             if (data[i] < other.data[i]) {
@@ -182,7 +182,7 @@ struct EuclidPoint{
         return true;
     }
 
-    // lexicographic order
+    /* lexicographic order */
     inline bool operator<(const EuclidPoint& other) const {
         for (size_t i=0; i<D; i++) {
             if (data[i] < other.data[i]) {
@@ -228,7 +228,4 @@ public:
 private:
     R *P;
 };
-
-
-
 }
-- 
cgit v1.2.3