Commit 0c8e1607 authored by Philipp Götze's avatar Philipp Götze
Browse files

Integrated persistent table with Topology, Replaced Index with persistent...

Integrated persistent table with Topology, Replaced Index with persistent B+-Tree, Added various iterators
parent d9042fdb
......@@ -269,6 +269,7 @@ set(core_sources
dsl/Topology.cpp
dsl/Dataflow.cpp
dsl/PFabricContext.cpp
table/TableInfo.cpp
${ZEROMQ_SOURCES}
# we need this as dependency to download the sources from github
${THIRD_PARTY_DIR}/json
......@@ -297,11 +298,6 @@ if(USE_NVML_TABLE)
nvm/PTableInfo.cpp
${THIRD_PARTY_DIR}/nvml
)
else()
set(core_sources
${core_sources}
nvm/VTableInfo.cpp
)
endif()
add_library(pfabric_core SHARED
......
This diff is collapsed.
This diff is collapsed.
/*
Taken from: https://pmem.io/2017/01/23/cpp-strings.html
*/
#ifndef PString_hpp_
#define PString_hpp_
#include "nvml/include/libpmemobj/tx_base.h"
#include "nvml/include/libpmemobj++/make_persistent.hpp"
#include "nvml/include/libpmemobj++/make_persistent_array.hpp"
#include "nvml/include/libpmemobj++/persistent_ptr.hpp"
using nvml::obj::delete_persistent;
using nvml::obj::make_persistent;
using nvml::obj::persistent_ptr;
#define SSO_CHARS 15
#define SSO_SIZE (SSO_CHARS + 1)
class PString {
public:
char* data() const { return str ? str.get() : const_cast<char*>(sso); }
void reset();
void set(std::string* value);
private:
char sso[SSO_SIZE];
persistent_ptr<char[]> str;
};
void PString::reset() {
pmemobj_tx_add_range_direct(sso, 1);
sso[0] = 0;
if (str) delete_persistent<char[]>(str, strlen(str.get()) + 1);
}
void PString::set(std::string* value) {
unsigned long length = value->length();
if (length <= SSO_CHARS) {
if (str) {
delete_persistent<char[]>(str, strlen(str.get()) + 1);
str = nullptr;
}
pmemobj_tx_add_range_direct(sso, SSO_SIZE);
strcpy(sso, value->c_str());
} else {
if (str) delete_persistent<char[]>(str, strlen(str.get()) + 1);
str = make_persistent<char[]>(length + 1);
strcpy(str.get(), value->c_str());
}
}
#endif
......@@ -47,24 +47,4 @@ int PTableInfo::findColumnByName(const std::string& colName) const {
if (mColumns->at(i).getName() == colName) return (int)i;
}
return -1;
}
std::ostream& operator<<(std::ostream& os, pfabric::nvm::ColumnInfo::ColumnType ct) {
using namespace pfabric::nvm;
switch (ct) {
case ColumnInfo::Void_Type:
os << "";
break;
case ColumnInfo::Int_Type:
os << "int";
break;
case ColumnInfo::Double_Type:
os << "double";
break;
case ColumnInfo::String_Type:
os << "std::string";
break;
}
return os;
}
}
\ No newline at end of file
......@@ -9,6 +9,8 @@
#include <utility>
#include <vector>
#include "table/TableInfo.hpp"
#include "nvml/include/libpmemobj++/allocator.hpp"
#include "nvml/include/libpmemobj++/detail/persistent_ptr_base.hpp"
#include "nvml/include/libpmemobj++/make_persistent.hpp"
......@@ -29,13 +31,11 @@ using nvml::obj::transaction;
namespace pfabric { namespace nvm {
class ColumnInfo {
class PColumnInfo {
public:
enum ColumnType { Void_Type, Int_Type, Double_Type, String_Type };
ColumnInfo(pool_base pop) : ColumnInfo(pop, "", ColumnType::Void_Type) {}
PColumnInfo(pool_base pop) : PColumnInfo(pop, "", ColumnInfo::ColumnType::Void_Type) {}
ColumnInfo(pool_base pop, const std::string& n, ColumnType ct) : mColType(ct) {
PColumnInfo(pool_base pop, const std::string& n, ColumnInfo::ColumnType ct) : mColType(ct) {
transaction::exec_tx(pop, [&] {
mColName = make_persistent<char[]>(n.length() +1);
strcpy(mColName.get(), n.c_str());
......@@ -44,84 +44,46 @@ public:
const std::string getName() const { return mColName.get(); }
const ColumnType getType() const { return mColType.get_ro(); }
bool operator==(const ColumnInfo& other) const {
return (strcmp(mColName.get(), other.mColName.get()) == 0 && mColType.get_ro() == other.mColType.get_ro());
}
bool operator<(const ColumnInfo& other) const {
return strcmp(mColName.get(), other.mColName.get()) <= 0;
}
const ColumnInfo::ColumnType getType() const { return mColType.get_ro(); }
private:
persistent_ptr<char[]> mColName;
p<ColumnType> mColType;
p<ColumnInfo::ColumnType> mColType;
};
typedef std::initializer_list<std::pair<std::string, ColumnInfo::ColumnType>> ColumnInitList;
/** TODO: For later, template implementation
template< bool isFixedSize >
struct IsFixedSize {
static const bool IS_FIXED_SIZE = isFixedSize;
};
template < ColumnInfo::ColumnType ColType >
struct ColTypeTraits;
template <>
struct ColTypeTraits< ColumnInfo::Int_Type > : IsFixedSize<true> {
static const std::size_t COLUMN_SIZE = sizeof(int32_t);
};
template <>
struct ColTypeTraits< ColumnInfo::Double_Type > : IsFixedSize<true> {
static const std::size_t COLUMN_SIZE = sizeof(double);
};
template <>
struct ColTypeTraits< ColumnInfo::String_Type > : IsFixedSize<false>{};
using Offset = std::size_t;
template< ColumnInfo::ColumnType ColType, typename Col >
typename std::enable_if< not ColTypeTraits< ColType >::IS_FIXED_SIZE, Offset >::type getSize(const Col& t ) {
//TODO: runtime size calculation
}
template< ColumnInfo::ColumnType ColType, typename Col >
typename std::enable_if< ColTypeTraits< ColType >::IS_FIXED_SIZE, Offset >::type getSize(const Col& t ) {
using ColTraits = ColTypeTraits<ColType>;
return ColTraits::COLUMN_SIZE;
}
*/
class PTableInfo {
public:
typedef persistent_ptr<PTableInfo> TableInfoPtr;
typedef std::vector<ColumnInfo, nvml::obj::allocator<ColumnInfo>> ColumnVector;
typedef persistent_ptr<PTableInfo> PTableInfoPtr;
typedef std::vector<PColumnInfo, nvml::obj::allocator<PColumnInfo>> ColumnVector;
typedef ColumnVector::const_iterator ColumnIterator;
PTableInfo(){}
PTableInfo(const std::string& name, ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type)
: mName(name.c_str()), mKeyType(keyType) {}
PTableInfo(const TableInfo& _tInfo, ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type)
: mName(_tInfo.tableName().c_str()), mKeyType(keyType){
auto pop = pool_by_vptr(this);
transaction::exec_tx(pop, [&] {
mColumns = make_persistent<ColumnVector>();
for (const auto &c : _tInfo)
mColumns->push_back(PColumnInfo(pop, c.getName(), c.getType()));
});
}
PTableInfo(const std::string& name, ColumnInitList columns,
ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type) :
ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type) :
mName(name.c_str()), mKeyType(keyType) {
auto pop = pool_by_vptr(this);
transaction::exec_tx(pop, [&] {
mColumns = make_persistent<ColumnVector>();
for (const auto &c : columns)
mColumns->push_back(ColumnInfo(pop, c.first, c.second));
mColumns->push_back(PColumnInfo(pop, c.first, c.second));
});
}
PTableInfo(const std::string& name, const ColumnVector& columns,
ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type) :
ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type) :
mName(name.c_str()), mKeyType(keyType) {
auto pop = pool_by_vptr(this);
transaction::exec_tx(pop, [&] {
......@@ -139,7 +101,7 @@ class PTableInfo {
int findColumnByName(const std::string& colName) const;
const ColumnInfo& columnInfo(int pos) const { return mColumns->at(pos); }
const PColumnInfo& columnInfo(int pos) const { return mColumns->at(pos); }
const std::size_t numColumns() const { return mColumns->size(); }
......@@ -151,9 +113,9 @@ class PTableInfo {
});
}
TableInfoPtr makeShared() const {
PTableInfoPtr makeShared() const {
auto pop = pool_by_vptr(this);
TableInfoPtr tInfo_ptr = nullptr;
PTableInfoPtr tInfo_ptr = nullptr;
transaction::exec_tx(pop, [&] {
tInfo_ptr = make_persistent<PTableInfo>(std::string(mName.get()), *mColumns, mKeyType.get_ro());
});
......@@ -170,16 +132,14 @@ class PTableInfo {
};
}
typedef persistent_ptr<nvm::PTableInfo> TableInfoPtr;
typedef persistent_ptr<nvm::PTableInfo> PTableInfoPtr;
} /* namespace pfabric::nvm */
std::ostream& operator<<(std::ostream& os, pfabric::nvm::ColumnInfo::ColumnType ct);
namespace std {
template <>
struct hash<pfabric::nvm::ColumnInfo> {
std::size_t operator()(const pfabric::nvm::ColumnInfo& c) const {
struct hash<pfabric::nvm::PColumnInfo> {
std::size_t operator()(const pfabric::nvm::PColumnInfo& c) const {
return std::hash<std::string>()(c.getName());
}
};
......
......@@ -128,7 +128,7 @@ struct get_helper {
*****************************************************************************/
template<std::size_t ID>
struct get_helper<std::string, ID> {
static char (&( apply(persistent_ptr<NVM_Block> block, const uint16_t *offsets)))[] {
static std::string apply(persistent_ptr<NVM_Block> block, const uint16_t *offsets) {
return reinterpret_cast<char (&)[]>(block->at(offsets[ID]));
}
};
......@@ -142,9 +142,9 @@ struct get_helper<std::string, ID> {
* the index of the requested attribute
*****************************************************************************/
template<std::size_t ID>
struct get_helper<int32_t, ID> {
static int32_t& apply(persistent_ptr<NVM_Block> block, const uint16_t *offsets) {
return reinterpret_cast<int32_t&>(block->at(offsets[ID]));
struct get_helper<int, ID> {
static int apply(persistent_ptr<NVM_Block> block, const uint16_t *offsets) {
return reinterpret_cast<int&>(block->at(offsets[ID]));
}
};
......@@ -158,11 +158,59 @@ struct get_helper<int32_t, ID> {
*****************************************************************************/
template<std::size_t ID>
struct get_helper<double, ID> {
static double& apply(persistent_ptr<NVM_Block> block, const uint16_t *offsets) {
static double apply(persistent_ptr<NVM_Block> block, const uint16_t *offsets) {
return reinterpret_cast<double&>(block->at(offsets[ID]));
}
};
/**************************************************************************//**
* \brief getAll_helper is a helper function to add all attributes of a PTuple
* to a passed TuplePtr instance.
*
* \tparam Tuple
* the underlying and desired Tuple type of the PTuple
* \tparam CurrentIndex
* the index of the attribute to set next
*****************************************************************************/
template<class Tuple, std::size_t CurrentIndex>
struct getAll_helper;
/**************************************************************************//**
* \brief General overload for setting Tuples with more than 1 element to add.
*
* This specialization will add the remaining elements first and then set the
* current attribute value.
*
* \tparam Tuple
* the underlying and desired Tuple type of the PTuple
* \tparam CurrentIndex
* the index of the attribute to add next
*****************************************************************************/
template<class Tuple, std::size_t CurrentIndex>
struct getAll_helper {
static void apply(SmartPtr<Tuple> tptr, persistent_ptr<NVM_Block> block, const uint16_t* const offsets) {
getAll_helper<Tuple, CurrentIndex - 1>::apply(tptr, block, offsets);
auto val = get_helper<typename Tuple::template getAttributeType<CurrentIndex-1>::type, CurrentIndex - 1>::apply(block, offsets);
tptr->template setAttribute<CurrentIndex-1>(val);
}
};
/**************************************************************************//**
* \brief Specialization for setting the first attribute.
*
* This specialization will just set the first attribute value.
*
* \tparam Tuple
* the underlying tuple type having one element
*****************************************************************************/
template<class Tuple>
struct getAll_helper<Tuple, 1> {
static void apply(SmartPtr<Tuple> tptr, persistent_ptr<NVM_Block> block, const uint16_t* const offsets) {
auto val = get_helper<typename Tuple::template getAttributeType<0>::type, 0>::apply(block, offsets);
tptr->template setAttribute<0>(val);
}
};
/**************************************************************************//**
* \brief PTuplePrinter is a helper function to print a persistent tuple of any
* size.
......@@ -194,7 +242,8 @@ template<class Tuple, std::size_t CurrentIndex>
struct PTuplePrinter {
static void print(std::ostream& os, persistent_ptr<NVM_Block> block, const uint16_t* offsets) {
PTuplePrinter<Tuple, CurrentIndex - 1>::print(os, block, offsets);
os << "," << get_helper<typename Tuple::template getAttributeType<CurrentIndex-1>::type, CurrentIndex - 1>::apply(block, offsets);
auto val = get_helper<typename Tuple::template getAttributeType<CurrentIndex-1>::type, CurrentIndex - 1>::apply(block, offsets);
os << "," << val;
}
};
......@@ -227,7 +276,6 @@ struct PTuplePrinter<Tuple, 0> {
}
};
} /* end namespace detail */
/**************************************************************************//**
......@@ -260,6 +308,7 @@ struct PTuplePrinter<Tuple, 0> {
template<class Tuple>
class PTuple {
public:
/************************************************************************//**
* \brief the number of attributes for this tuple type.
***************************************************************************/
......@@ -291,6 +340,8 @@ public:
block(_block), offsets(_offsets) {
}
PTuple() : block(nullptr), offsets(std::array<uint16_t, Tuple::NUM_ATTRIBUTES>()) {}
/************************************************************************//**
* \brief Get a specific attribute value from the persistent tuple.
*
......@@ -300,8 +351,10 @@ public:
* a reference to the persistent tuple's attribute with the requested \c ID
***************************************************************************/
template<std::size_t ID>
inline auto getAttribute() {
return detail::get_helper<typename getAttributeType<ID>::type, ID>::apply(block, offsets.get_ro().data());
typename getAttributeType< ID >::type& getAttribute() {
auto val = new typename getAttributeType<ID>::type;
*val = detail::get_helper<typename getAttributeType<ID>::type, ID>::apply(block, offsets.get_ro().data());
return *val;
}
/************************************************************************//**
......@@ -326,7 +379,7 @@ public:
* a reference to the persistent tuple's attribute with the requested \c ID
***************************************************************************/
template<std::size_t ID>
inline auto getAttribute() const {
const typename getAttributeType< ID >::type& getAttribute() const {
return detail::get_helper<typename getAttributeType<ID>::type, ID>::apply(block, offsets.get_ro().data());
}
......@@ -353,15 +406,31 @@ public:
detail::PTuplePrinter<Tuple, NUM_ATTRIBUTES>::print(os, block, offsets.get_ro().data());
}
/************************************************************************//**
* \brief Create a new Tuple from this PTuple and return a pointer to it.
*
* \return
* a smart pointer to the newly created Tuple
***************************************************************************/
SmartPtr<Tuple> createTuple() const {
typename Tuple::Base tp{};
SmartPtr<Tuple> tptr(new Tuple(tp));
detail::getAll_helper<Tuple, NUM_ATTRIBUTES>::apply(tptr, block, offsets.get_ro().data());
return tptr;
}
private:
persistent_ptr<NVM_Block> block;
p<std::array<uint16_t, NUM_ATTRIBUTES>> offsets;
}; /* class PTuplePtr */
}; /* class PTuple */
} /* end namespace nvm */
template<class Tuple>
using PTuplePtr = persistent_ptr<nvm::PTuple<Tuple>>;
/**************************************************************************//**
* \brief Get a specific attribute reference from the PTuple.
*
......
/*
* Copyright 2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* * Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef EXAMPLES_CTREE_MAP_PERSISTENT_HPP
#define EXAMPLES_CTREE_MAP_PERSISTENT_HPP
#include <cstdint>
#include <functional>
#include <stdlib.h>
#include "ex_common.h"
#include "nvml/include/libpmemobj++/make_persistent.hpp"
#include "nvml/include/libpmemobj++/make_persistent_atomic.hpp"
#include "nvml/include/libpmemobj++/p.hpp"
#include "nvml/include/libpmemobj++/persistent_ptr.hpp"
#include "nvml/include/libpmemobj++/pool.hpp"
#include "nvml/include/libpmemobj++/transaction.hpp"
#include "nvml/include/libpmemobj++/utils.hpp"
#define BIT_IS_SET(n, i) (!!((n) & (1ULL << (i))))
namespace nvobj = nvml::obj;
namespace examples
{
/**
* C++ implementation of a persistent ctree.
*
* Based on the volatile version. This version was implemented to show how much
* effort is needed to convert a volatile structure into a persistent one using
* C++ obj bindings. All API functions are atomic in respect to persistency.
*/
template <typename K, typename T>
class ctree_map_p {
public:
/** Convenience typedef for the key type. */
typedef K key_type;
/** Convenience typedef for the value type. */
typedef nvobj::persistent_ptr<T> value_type;
/** Convenience typedef for the callback function. */
typedef std::function<int(key_type, value_type, void *)> callback;
/**
* Default constructor.
*/
ctree_map_p()
{
auto pop = nvobj::pool_by_vptr(this);
nvobj::transaction::exec_tx(pop, [&] {
this->root = nvobj::make_persistent<entry>();
});
}
/**
* Insert or update the given value under the given key.
*
* The map takes ownership of the value.
*
* @param key The key to insert under.
* @param value The value to be inserted.
*
* @return 0 on success, negative values on error.
*/
int
insert(key_type key, value_type value)
{
auto dest_entry = root;
while (dest_entry->inode != nullptr) {
auto n = dest_entry->inode;
dest_entry = n->entries[BIT_IS_SET(key, n->diff)];
}
entry e(key, value);
auto pop = nvobj::pool_by_vptr(this);
nvobj::transaction::exec_tx(pop, [&] {
if (dest_entry->key == 0 || dest_entry->key == key) {
nvobj::delete_persistent<T>(dest_entry->value);
*dest_entry = e;
} else {
insert_leaf(&e, find_crit_bit(dest_entry->key,
key));
}
});
return 0;
}
/**
* Allocating insert.
*
* Creates a new value_type instance and inserts it into the tree.
*
* @param key The key to insert under.
* @param args variadic template parameter for object construction
* arguments.
*
* @return 0 on success, negative values on error.
*/
template <typename... Args>
int
insert_new(key_type key, const Args &... args)
{
auto pop = nvobj::pool_by_vptr(this);
nvobj::transaction::exec_tx(pop, [&] {
return insert(key, nvobj::make_persistent<T>(args...));
});
return -1;
}
/**
* Remove a value from the tree.
*
* The tree no longer owns the value.
*
* @param key The key for which the value will be removed.
*
* @return The value if it is in the tree, nullptr otherwise.
*/
value_type
remove(key_type key)
{
nvobj::persistent_ptr<entry> parent = nullptr;
auto leaf = get_leaf(key, &parent);
if (leaf == nullptr)
return nullptr;
auto ret = leaf->value;
auto pop = nvobj::pool_by_vptr(this);
nvobj::transaction::exec_tx(pop, [&] {
if (parent == nullptr) {
leaf->key = 0;
leaf->value = nullptr;
} else {
auto n = parent->inode;
*parent = *(
n->entries[parent->inode->entries[0]
->key == leaf->key]);
/* cleanup entries and the unnecessary node */
nvobj::delete_persistent<entry>(n->entries[0]);
nvobj::delete_persistent<entry>(n->entries[1]);