Integrated persistent table with Topology, Replaced Index with persistent...

Integrated persistent table with Topology, Replaced Index with persistent B+-Tree, Added various iterators

src/CMakeLists.txt

@@ -269,6 +269,7 @@ set(core_sources
   dsl/Topology.cpp
   dsl/Dataflow.cpp
   dsl/PFabricContext.cpp
+  table/TableInfo.cpp
   ${ZEROMQ_SOURCES}
   # we need this as dependency to download the sources from github
   ${THIRD_PARTY_DIR}/json
@@ -297,11 +298,6 @@ if(USE_NVML_TABLE)
     nvm/PTableInfo.cpp
     ${THIRD_PARTY_DIR}/nvml
   )
-else()
-  set(core_sources
-    ${core_sources}
-    nvm/VTableInfo.cpp
-  )
 endif()
 
 add_library(pfabric_core SHARED

src/nvm/PString.hpp

+/*
+Taken from: https://pmem.io/2017/01/23/cpp-strings.html
+*/
+
+#ifndef PString_hpp_
+#define PString_hpp_
+
+#include "nvml/include/libpmemobj/tx_base.h"
+#include "nvml/include/libpmemobj++/make_persistent.hpp"
+#include "nvml/include/libpmemobj++/make_persistent_array.hpp"
+#include "nvml/include/libpmemobj++/persistent_ptr.hpp"
+
+using nvml::obj::delete_persistent;
+using nvml::obj::make_persistent;
+using nvml::obj::persistent_ptr;
+
+#define SSO_CHARS 15
+#define SSO_SIZE (SSO_CHARS + 1)
+
+class PString {
+public:
+  char* data() const { return str ? str.get() : const_cast<char*>(sso); }
+  void reset();
+  void set(std::string* value);
+private:
+  char sso[SSO_SIZE];
+  persistent_ptr<char[]> str;
+};
+
+void PString::reset() {
+  pmemobj_tx_add_range_direct(sso, 1);
+  sso[0] = 0;
+  if (str) delete_persistent<char[]>(str, strlen(str.get()) + 1);
+}
+
+void PString::set(std::string* value) {
+  unsigned long length = value->length();
+  if (length <= SSO_CHARS) {
+    if (str) {
+      delete_persistent<char[]>(str, strlen(str.get()) + 1);
+      str = nullptr;
+    }
+    pmemobj_tx_add_range_direct(sso, SSO_SIZE);
+    strcpy(sso, value->c_str());
+  } else {
+    if (str) delete_persistent<char[]>(str, strlen(str.get()) + 1);
+    str = make_persistent<char[]>(length + 1);
+    strcpy(str.get(), value->c_str());
+  }
+}
+
+#endif

src/nvm/PTableInfo.cpp

@@ -47,24 +47,4 @@ int PTableInfo::findColumnByName(const std::string& colName) const {
     if (mColumns->at(i).getName() == colName) return (int)i;
   }
   return -1;
-}
-
-std::ostream& operator<<(std::ostream& os, pfabric::nvm::ColumnInfo::ColumnType ct) {
-  using namespace pfabric::nvm;
-
-  switch (ct) {
-    case ColumnInfo::Void_Type:
-      os << "";
-      break;
-    case ColumnInfo::Int_Type:
-      os << "int";
-      break;
-    case ColumnInfo::Double_Type:
-      os << "double";
-      break;
-    case ColumnInfo::String_Type:
-      os << "std::string";
-      break;
-  }
-  return os;
-}
+}
\ No newline at end of file

src/nvm/PTableInfo.hpp

@@ -9,6 +9,8 @@
 #include <utility>
 #include <vector>
 
+#include "table/TableInfo.hpp"
+
 #include "nvml/include/libpmemobj++/allocator.hpp"
 #include "nvml/include/libpmemobj++/detail/persistent_ptr_base.hpp"
 #include "nvml/include/libpmemobj++/make_persistent.hpp"
@@ -29,13 +31,11 @@ using nvml::obj::transaction;
 
 namespace pfabric { namespace nvm {
 
-class ColumnInfo {
+class PColumnInfo {
 public:
-  enum ColumnType { Void_Type, Int_Type, Double_Type, String_Type };
-
-  ColumnInfo(pool_base pop) : ColumnInfo(pop, "", ColumnType::Void_Type) {}
+  PColumnInfo(pool_base pop) : PColumnInfo(pop, "", ColumnInfo::ColumnType::Void_Type) {}
 
-  ColumnInfo(pool_base pop, const std::string& n, ColumnType ct) : mColType(ct) {
+  PColumnInfo(pool_base pop, const std::string& n, ColumnInfo::ColumnType ct) : mColType(ct) {
     transaction::exec_tx(pop, [&] {
       mColName = make_persistent<char[]>(n.length() +1);
       strcpy(mColName.get(), n.c_str());
@@ -44,84 +44,46 @@ public:
 
   const std::string getName() const { return mColName.get(); }
 
-  const ColumnType getType() const { return mColType.get_ro(); }
-
-  bool operator==(const ColumnInfo& other) const {
-    return (strcmp(mColName.get(), other.mColName.get()) == 0 && mColType.get_ro() == other.mColType.get_ro());
-  }
-
-  bool operator<(const ColumnInfo& other) const {
-    return strcmp(mColName.get(), other.mColName.get()) <= 0;
-  }
+  const ColumnInfo::ColumnType getType() const { return mColType.get_ro(); }
 
 private:
   persistent_ptr<char[]> mColName;
-  p<ColumnType> mColType;
+  p<ColumnInfo::ColumnType> mColType;
 };
 
 typedef std::initializer_list<std::pair<std::string, ColumnInfo::ColumnType>> ColumnInitList;
 
-/** TODO: For later, template implementation
-template< bool isFixedSize >
-struct IsFixedSize {
-  static const bool IS_FIXED_SIZE = isFixedSize;
-};
-
-template < ColumnInfo::ColumnType ColType >
-struct ColTypeTraits;
-
-template <>
-struct ColTypeTraits< ColumnInfo::Int_Type > : IsFixedSize<true> {
-  static const std::size_t COLUMN_SIZE = sizeof(int32_t);
-};
-
-template <>
-struct ColTypeTraits< ColumnInfo::Double_Type > : IsFixedSize<true> {
-  static const std::size_t COLUMN_SIZE = sizeof(double);
-};
-
-template <>
-struct ColTypeTraits< ColumnInfo::String_Type > : IsFixedSize<false>{};
-
-using Offset = std::size_t;
-
-template< ColumnInfo::ColumnType ColType, typename Col >
-typename std::enable_if< not ColTypeTraits< ColType >::IS_FIXED_SIZE, Offset >::type getSize(const  Col& t ) {
-  //TODO: runtime size calculation
-}
-
-template< ColumnInfo::ColumnType ColType, typename Col >
-typename std::enable_if< ColTypeTraits< ColType >::IS_FIXED_SIZE, Offset >::type getSize(const Col& t ) {
-  using ColTraits = ColTypeTraits<ColType>;
-  return ColTraits::COLUMN_SIZE;
-}
-
-*/
-
 class PTableInfo {
  public:
-  typedef persistent_ptr<PTableInfo> TableInfoPtr;
-  typedef std::vector<ColumnInfo, nvml::obj::allocator<ColumnInfo>> ColumnVector;
+  typedef persistent_ptr<PTableInfo> PTableInfoPtr;
+  typedef std::vector<PColumnInfo, nvml::obj::allocator<PColumnInfo>> ColumnVector;
   typedef ColumnVector::const_iterator ColumnIterator;
 
   PTableInfo(){}
 
-  PTableInfo(const std::string& name, ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type)
-    : mName(name.c_str()), mKeyType(keyType) {}
+  PTableInfo(const TableInfo& _tInfo, ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type)
+    : mName(_tInfo.tableName().c_str()), mKeyType(keyType){
+    auto pop = pool_by_vptr(this);
+    transaction::exec_tx(pop, [&] {
+      mColumns = make_persistent<ColumnVector>();
+      for (const auto &c : _tInfo)
+        mColumns->push_back(PColumnInfo(pop, c.getName(), c.getType()));
+    });
+  }
 
   PTableInfo(const std::string& name, ColumnInitList columns,
-      ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type) :
+             ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type) :
       mName(name.c_str()), mKeyType(keyType) {
     auto pop = pool_by_vptr(this);
     transaction::exec_tx(pop, [&] {
       mColumns = make_persistent<ColumnVector>();
       for (const auto &c : columns)
-        mColumns->push_back(ColumnInfo(pop, c.first, c.second));
+        mColumns->push_back(PColumnInfo(pop, c.first, c.second));
     });
   }
 
   PTableInfo(const std::string& name, const ColumnVector& columns,
-        ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type) :
+             ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type) :
         mName(name.c_str()), mKeyType(keyType) {
     auto pop = pool_by_vptr(this);
     transaction::exec_tx(pop, [&] {
@@ -139,7 +101,7 @@ class PTableInfo {
 
   int findColumnByName(const std::string& colName) const;
 
-  const ColumnInfo& columnInfo(int pos) const { return mColumns->at(pos); }
+  const PColumnInfo& columnInfo(int pos) const { return mColumns->at(pos); }
 
   const std::size_t numColumns() const { return mColumns->size(); }
 
@@ -151,9 +113,9 @@ class PTableInfo {
     });
    }
 
-  TableInfoPtr makeShared() const {
+  PTableInfoPtr makeShared() const {
     auto pop = pool_by_vptr(this);
-    TableInfoPtr tInfo_ptr = nullptr;
+    PTableInfoPtr tInfo_ptr = nullptr;
     transaction::exec_tx(pop, [&] {
        tInfo_ptr = make_persistent<PTableInfo>(std::string(mName.get()), *mColumns, mKeyType.get_ro());
     });
@@ -170,16 +132,14 @@ class PTableInfo {
 };
 
 }
-typedef persistent_ptr<nvm::PTableInfo> TableInfoPtr;
+typedef persistent_ptr<nvm::PTableInfo> PTableInfoPtr;
 
 } /* namespace pfabric::nvm */
 
-std::ostream& operator<<(std::ostream& os, pfabric::nvm::ColumnInfo::ColumnType ct);
-
 namespace std {
   template <>
-  struct hash<pfabric::nvm::ColumnInfo> {
-    std::size_t operator()(const pfabric::nvm::ColumnInfo& c) const {
+  struct hash<pfabric::nvm::PColumnInfo> {
+    std::size_t operator()(const pfabric::nvm::PColumnInfo& c) const {
       return std::hash<std::string>()(c.getName());
     }
   };

src/nvm/PTuple.hpp

@@ -128,7 +128,7 @@ struct get_helper {
  *****************************************************************************/
 template<std::size_t ID>
 struct get_helper<std::string, ID> {
-  static char (&( apply(persistent_ptr<NVM_Block> block, const uint16_t *offsets)))[] {
+  static std::string apply(persistent_ptr<NVM_Block> block, const uint16_t *offsets) {
     return reinterpret_cast<char (&)[]>(block->at(offsets[ID]));
   }
 };
@@ -142,9 +142,9 @@ struct get_helper<std::string, ID> {
  *   the index of the requested attribute
  *****************************************************************************/
 template<std::size_t ID>
-struct get_helper<int32_t, ID> {
-  static int32_t& apply(persistent_ptr<NVM_Block> block, const uint16_t *offsets) {
-    return reinterpret_cast<int32_t&>(block->at(offsets[ID]));
+struct get_helper<int, ID> {
+  static int apply(persistent_ptr<NVM_Block> block, const uint16_t *offsets) {
+    return reinterpret_cast<int&>(block->at(offsets[ID]));
   }
 };
 
@@ -158,11 +158,59 @@ struct get_helper<int32_t, ID> {
  *****************************************************************************/
 template<std::size_t ID>
 struct get_helper<double, ID> {
-  static double& apply(persistent_ptr<NVM_Block> block, const uint16_t *offsets) {
+  static double apply(persistent_ptr<NVM_Block> block, const uint16_t *offsets) {
     return reinterpret_cast<double&>(block->at(offsets[ID]));
   }
 };
 
+/**************************************************************************//**
+ * \brief getAll_helper is a helper function to add all attributes of a PTuple
+ *        to a passed TuplePtr instance.
+ *
+ * \tparam Tuple
+ *   the underlying and desired Tuple type of the PTuple
+ * \tparam CurrentIndex
+ *   the index of the attribute to set next
+ *****************************************************************************/
+template<class Tuple, std::size_t CurrentIndex>
+struct getAll_helper;
+
+/**************************************************************************//**
+ * \brief General overload for setting Tuples with more than 1 element to add.
+ *
+ * This specialization will add the remaining elements first and then set the
+ * current attribute value.
+ *
+ * \tparam Tuple
+ *   the underlying and desired Tuple type of the PTuple
+ * \tparam CurrentIndex
+ *   the index of the attribute to add next
+ *****************************************************************************/
+template<class Tuple, std::size_t CurrentIndex>
+struct getAll_helper {
+  static void apply(SmartPtr<Tuple> tptr, persistent_ptr<NVM_Block> block, const uint16_t* const offsets) {
+    getAll_helper<Tuple, CurrentIndex - 1>::apply(tptr, block, offsets);
+    auto val = get_helper<typename Tuple::template getAttributeType<CurrentIndex-1>::type, CurrentIndex - 1>::apply(block, offsets);
+    tptr->template setAttribute<CurrentIndex-1>(val);
+  }
+};
+
+/**************************************************************************//**
+ * \brief Specialization for setting the first attribute.
+ *
+ * This specialization will just set the first attribute value.
+ *
+ * \tparam Tuple
+ *    the underlying tuple type having one element
+ *****************************************************************************/
+template<class Tuple>
+struct getAll_helper<Tuple, 1> {
+  static void apply(SmartPtr<Tuple> tptr, persistent_ptr<NVM_Block> block, const uint16_t* const offsets) {
+    auto val = get_helper<typename Tuple::template getAttributeType<0>::type, 0>::apply(block, offsets);
+    tptr->template setAttribute<0>(val);
+  }
+};
+
 /**************************************************************************//**
  * \brief PTuplePrinter is a helper function to print a persistent tuple of any
  *        size.
@@ -194,7 +242,8 @@ template<class Tuple, std::size_t CurrentIndex>
 struct PTuplePrinter {
   static void print(std::ostream& os, persistent_ptr<NVM_Block> block, const uint16_t* offsets) {
     PTuplePrinter<Tuple, CurrentIndex - 1>::print(os, block, offsets);
-    os << "," << get_helper<typename Tuple::template getAttributeType<CurrentIndex-1>::type, CurrentIndex - 1>::apply(block, offsets);
+    auto val = get_helper<typename Tuple::template getAttributeType<CurrentIndex-1>::type, CurrentIndex - 1>::apply(block, offsets);
+    os << "," << val;
   }
 };
 
@@ -227,7 +276,6 @@ struct PTuplePrinter<Tuple, 0> {
   }
 };
 
-
 } /* end namespace detail */
 
 /**************************************************************************//**
@@ -260,6 +308,7 @@ struct PTuplePrinter<Tuple, 0> {
 template<class Tuple>
 class PTuple {
 public:
+
   /************************************************************************//**
    * \brief the number of attributes for this tuple type.
    ***************************************************************************/
@@ -291,6 +340,8 @@ public:
       block(_block), offsets(_offsets) {
   }
 
+  PTuple() : block(nullptr), offsets(std::array<uint16_t, Tuple::NUM_ATTRIBUTES>()) {}
+
   /************************************************************************//**
    * \brief Get a specific attribute value from the persistent tuple.
    *
@@ -300,8 +351,10 @@ public:
    *   a reference to the persistent tuple's attribute with the requested \c ID
    ***************************************************************************/
   template<std::size_t ID>
-  inline auto getAttribute() {
-    return detail::get_helper<typename getAttributeType<ID>::type, ID>::apply(block, offsets.get_ro().data());
+  typename getAttributeType< ID >::type& getAttribute() {
+    auto val = new typename getAttributeType<ID>::type;
+    *val = detail::get_helper<typename getAttributeType<ID>::type, ID>::apply(block, offsets.get_ro().data());
+    return *val;
   }
 
   /************************************************************************//**
@@ -326,7 +379,7 @@ public:
    *   a reference to the persistent tuple's attribute with the requested \c ID
    ***************************************************************************/
   template<std::size_t ID>
-  inline auto getAttribute() const {
+  const typename getAttributeType< ID >::type& getAttribute() const {
     return detail::get_helper<typename getAttributeType<ID>::type, ID>::apply(block, offsets.get_ro().data());
   }
 
@@ -353,15 +406,31 @@ public:
     detail::PTuplePrinter<Tuple, NUM_ATTRIBUTES>::print(os, block, offsets.get_ro().data());
   }
 
+  /************************************************************************//**
+   * \brief Create a new Tuple from this PTuple and return a pointer to it.
+   *
+   * \return
+   *   a smart pointer to the newly created Tuple
+   ***************************************************************************/
+  SmartPtr<Tuple> createTuple() const {
+    typename Tuple::Base tp{};
+    SmartPtr<Tuple> tptr(new Tuple(tp));
+    detail::getAll_helper<Tuple, NUM_ATTRIBUTES>::apply(tptr, block, offsets.get_ro().data());
+    return tptr;
+  }
+
 private:
 
   persistent_ptr<NVM_Block> block;
   p<std::array<uint16_t, NUM_ATTRIBUTES>> offsets;
 
-}; /* class PTuplePtr */
+}; /* class PTuple */
 
 } /* end namespace nvm */
 
+template<class Tuple>
+using PTuplePtr = persistent_ptr<nvm::PTuple<Tuple>>;
+
 /**************************************************************************//**
  * \brief Get a specific attribute reference from the PTuple.
  *

src/nvm/ctree_map_persistent.hpp

-/*
- * Copyright 2016, Intel Corporation
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *
- *     * Neither the name of the copyright holder nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef EXAMPLES_CTREE_MAP_PERSISTENT_HPP
-#define EXAMPLES_CTREE_MAP_PERSISTENT_HPP
-#include <cstdint>
-#include <functional>
-#include <stdlib.h>
-#include "ex_common.h"
-
-#include "nvml/include/libpmemobj++/make_persistent.hpp"
-#include "nvml/include/libpmemobj++/make_persistent_atomic.hpp"
-#include "nvml/include/libpmemobj++/p.hpp"
-#include "nvml/include/libpmemobj++/persistent_ptr.hpp"
-#include "nvml/include/libpmemobj++/pool.hpp"
-#include "nvml/include/libpmemobj++/transaction.hpp"
-#include "nvml/include/libpmemobj++/utils.hpp"
-
-#define BIT_IS_SET(n, i) (!!((n) & (1ULL << (i))))
-
-namespace nvobj = nvml::obj;
-
-namespace examples
-{
-
-/**
- * C++ implementation of a persistent ctree.
- *
- * Based on the volatile version. This version was implemented to show how much
- * effort is needed to convert a volatile structure into a persistent one using
- * C++ obj bindings. All API functions are atomic in respect to persistency.
- */
-template <typename K, typename T>
-class ctree_map_p {
-public:
-	/** Convenience typedef for the key type. */
-	typedef K key_type;
-
-	/** Convenience typedef for the value type. */
-	typedef nvobj::persistent_ptr<T> value_type;
-
-	/** Convenience typedef for the callback function. */
-	typedef std::function<int(key_type, value_type, void *)> callback;
-
-	/**
-	 * Default constructor.
-	 */
-	ctree_map_p()
-	{
-		auto pop = nvobj::pool_by_vptr(this);
-
-		nvobj::transaction::exec_tx(pop, [&] {
-			this->root = nvobj::make_persistent<entry>();
-		});
-	}
-
-	/**
-	 * Insert or update the given value under the given key.
-	 *
-	 * The map takes ownership of the value.
-	 *
-	 * @param key The key to insert under.
-	 * @param value The value to be inserted.
-	 *
-	 * @return 0 on success, negative values on error.
-	 */
-	int
-	insert(key_type key, value_type value)
-	{
-		auto dest_entry = root;
-		while (dest_entry->inode != nullptr) {
-			auto n = dest_entry->inode;
-			dest_entry = n->entries[BIT_IS_SET(key, n->diff)];
-		}
-
-		entry e(key, value);
-		auto pop = nvobj::pool_by_vptr(this);
-		nvobj::transaction::exec_tx(pop, [&] {
-			if (dest_entry->key == 0 || dest_entry->key == key) {
-				nvobj::delete_persistent<T>(dest_entry->value);
-				*dest_entry = e;
-			} else {
-				insert_leaf(&e, find_crit_bit(dest_entry->key,
-							      key));
-			}
-		});
-
-		return 0;
-	}
-
-	/**
-	 * Allocating insert.
-	 *
-	 * Creates a new value_type instance and inserts it into the tree.
-	 *
-	 * @param key The key to insert under.
-	 * @param args variadic template parameter for object construction
-	 *	arguments.
-	 *
-	 * @return 0 on success, negative values on error.
-	 */
-	template <typename... Args>
-	int
-	insert_new(key_type key, const Args &... args)
-	{
-		auto pop = nvobj::pool_by_vptr(this);
-		nvobj::transaction::exec_tx(pop, [&] {
-			return insert(key, nvobj::make_persistent<T>(args...));
-		});
-
-		return -1;
-	}
-
-	/**
-	 * Remove a value from the tree.
-	 *
-	 * The tree no longer owns the value.
-	 *
-	 * @param key The key for which the value will be removed.
-	 *
-	 * @return The value if it is in the tree, nullptr otherwise.
-	 */
-	value_type
-	remove(key_type key)
-	{
-		nvobj::persistent_ptr<entry> parent = nullptr;
-		auto leaf = get_leaf(key, &parent);
-
-		if (leaf == nullptr)
-			return nullptr;
-
-		auto ret = leaf->value;
-
-		auto pop = nvobj::pool_by_vptr(this);
-		nvobj::transaction::exec_tx(pop, [&] {
-			if (parent == nullptr) {
-				leaf->key = 0;
-				leaf->value = nullptr;
-			} else {
-				auto n = parent->inode;
-				*parent = *(
-					n->entries[parent->inode->entries[0]
-							   ->key == leaf->key]);
-
-				/* cleanup entries and the unnecessary node */
-				nvobj::delete_persistent<entry>(n->entries[0]);
-				nvobj::delete_persistent<entry>(n->entries[1]);