Commit e2d7a4fd authored by Philipp Götze's avatar Philipp Götze
Browse files

Cleaned some implementation details + and some fixes

parent 3f4e53b3
......@@ -28,13 +28,13 @@ option(USE_BOOST_SPIRIT_PARSER
# If switched to off, it will not be downloaded, saving initial building time.
option(USE_ROCKSDB_TABLE
"use RocksDB for implementing persistent tables"
OFF
OFF
)
# Use Non-Volatile Memory Library for implementing tables
option(USE_NVML_TABLE
"use nvml for implementing persistent memory tables"
ON
ON
)
# Build only pipefabric libraries (cep and core)
......@@ -55,7 +55,7 @@ option(BUILD_TEST_CASES
#Build google benchmark library
option(BUILD_GOOGLE_BENCH
"build google benchmark"
ON
OFF
)
# Build benchmark test
......@@ -75,7 +75,7 @@ endif()
#CMAKE_FORCE_CXX_COMPILER(icpc "Intel C++ Compiler")
# C++ compiler flags
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -Wall -Wno-deprecated -g -O3 -Wsign-compare")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -Wall -Wno-deprecated -g -O2 -Wsign-compare")
if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-local-typedefs -Wno-#pragma-messages")
elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
......@@ -278,37 +278,43 @@ set(core_sources
${THIRD_PARTY_DIR}/catch
)
if(USE_ROCKSDB_TABLE)
if(BUILD_GOOGLE_BENCH)
set(core_sources
${core_sources}
${THIRD_PARTY_DIR}/rocksdb
${THIRD_PARTY_DIR}/benchmark
)
endif()
if(BUILD_GOOGLE_BENCH)
if(USE_ROCKSDB_TABLE)
set(core_sources
${core_sources}
${THIRD_PARTY_DIR}/benchmark
)
${THIRD_PARTY_DIR}/rocksdb)
add_library(pfabric_core SHARED
${core_sources})
target_link_libraries(pfabric_core
${BOOST_LIBRARIES}
${ZEROMQ_LIBRARIES}
${ROCKSDB_LIB})
add_executable(rock_test rock_test.cpp)
target_link_libraries(rock_test pfabric_core ${ROCKSDB_LIB})
endif()
if(USE_NVML_TABLE)
set(core_sources
${core_sources}
nvm/PTableInfo.cpp
${THIRD_PARTY_DIR}/nvml
)
endif()
add_library(pfabric_core SHARED
${THIRD_PARTY_DIR}/nvml)
add_library(pfabric_core SHARED
${core_sources}
)
)
target_link_libraries(pfabric_core
${BOOST_LIBRARIES}
${ZEROMQ_LIBRARIES}
${NVML_LIBRARIES})
target_link_libraries(pfabric_core
${BOOST_LIBRARIES}
${ZEROMQ_LIBRARIES}
${NVML_LIBRARIES}
)
add_executable(nvm_test nvm_test.cpp)
target_link_libraries(nvm_test pfabric_core)
endif()
#-----------------------------------------------------------------------------------------
#
......
......@@ -23,8 +23,6 @@
#include "dsl/PFabricContext.hpp"
#include "dsl/Topology.hpp"
#include "table/BaseTable.hpp"
#include "table/TableInfo.hpp"
using namespace pfabric;
......
......@@ -35,50 +35,57 @@
#include "nvml/include/libpmemobj++/transaction.hpp"
#include "nvml/include/libpmemobj++/utils.hpp"
namespace pfabric { namespace nvm {
namespace pfabric {
namespace nvm {
using nvml::obj::allocator;
using nvml::obj::delete_persistent;
using nvml::obj::make_persistent;
using nvml::obj::p;
using nvml::obj::persistent_ptr;
using nvml::obj::pool_by_vptr;
using nvml::obj::transaction;
using nvml::obj::allocator;
using nvml::obj::delete_persistent;
using nvml::obj::make_persistent;
using nvml::obj::p;
using nvml::obj::persistent_ptr;
using nvml::obj::pool_by_vptr;
using nvml::obj::transaction;
/**************************************************************************//**
* \brief Info structure about the BDCC meta data.
*
* It is used in persistent tables to store the BDCC meta data and statistics.
*****************************************************************************/
struct BDCCInfo {
typedef std::unordered_map<uint16_t, uint16_t> ColumnBitsMap; //<mapping from column id to number of bits
typedef std::vector<std::pair<uint16_t,uint16_t>, nvml::obj::allocator<std::pair<uint16_t, uint16_t>>> pColumnBitsMap;
class BDCCInfo {
using pColumnBitsMap = const std::vector<std::pair<uint16_t, uint16_t>, nvml::obj::allocator<std::pair<uint16_t, uint16_t>>>;
public:
using ColumnBitsMap = const std::unordered_map<uint16_t, uint16_t>; //<mapping from column id to number of bits
explicit BDCCInfo(const ColumnBitsMap &_bitMap) :
bitMap(_bitMap.cbegin(), _bitMap.cend()),
numberOfBins(std::accumulate(_bitMap.begin(), _bitMap.end(), 0,
[](const size_t sum, decltype(*_bitMap.begin()) p) { return sum + p.second; })) {}
explicit BDCCInfo(const ColumnBitsMap &_bitMap) {
auto pop = pool_by_vptr(this);
transaction::exec_tx(pop, [&] {
//bitMap = make_persistent<pColumnBitsMap>();
for (const auto& c : _bitMap) {
bitMap.insert(bitMap.cbegin(), std::make_pair(c.first, c.second));
const pColumnBitsMap::const_iterator find(uint16_t item) const {
for (auto it = bitMap.cbegin(); it != bitMap.cend(); it++) {
if (it->first == item) return it;
}
return bitMap.cend();
}
numBins = std::accumulate(_bitMap.begin(), _bitMap.end(), 0,
[](const size_t sum, decltype(*_bitMap.begin()) p) { return sum + p.second; });
});
}
pColumnBitsMap::const_iterator find(uint16_t item) {
for (auto it = bitMap.cbegin(); it!= bitMap.cend(); it++) {
if (it->first == item) return it;
}
return bitMap.cend();
}
const size_t numColumns() const {
return bitMap.size();
}
pColumnBitsMap bitMap;
p<size_t> numBins;
//std::map<uint32_t, std::size_t> histogram;
const size_t numBins() const {
return numberOfBins.get_ro();
}
};/* struct BDCCInfo */
const pColumnBitsMap::const_iterator cend() const {
return bitMap.cend();
}
// private:
const pColumnBitsMap bitMap;
p<const size_t> numberOfBins;
//std::map<uint32_t, std::size_t> histogram;
};/* struct BDCCInfo */
}} /* namespace pfabric::nvm */
}
} /* namespace pfabric::nvm */
#endif /* PTuple_hpp_ */
/*
* Copyright (c) 2014-17 The PipeFabric team,
* All Rights Reserved.
*
* This file is part of the PipeFabric package.
*
* PipeFabric is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License (GPL) as
* published by the Free Software Foundation; either version 2 of
* the License, or (at your option) any later version.
*
* This package is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; see the file LICENSE.
* If not you can find the GPL at http://www.gnu.org/copyleft/gpl.html
*/
#ifndef NVM_Block_hpp_
#define NVM_Block_hpp_
#include <array>
#include "nvml/include/libpmemobj++/make_persistent.hpp"
#include "nvml/include/libpmemobj++/persistent_ptr.hpp"
namespace pfabric {
namespace nvm {
using nvml::obj::persistent_ptr;
using nvml::obj::make_persistent;
using nvml::obj::delete_persistent;
/** Positions in NVM_Block */
const int gDDCRangePos1 = 0;
const int gDDCRangePos2 = 4;
const int gCountPos = 8;
const int gFreeSpacePos = 12;
const int gSmaOffsetPos = 14;
const int gDataOffsetPos = 16;
/** Sizes/Lengths in NVM_Block */
const int gFixedHeaderSize = 14;
const int gDDCValueSize = 4;
const int gAttrOffsetSize = 4;
const int gOffsetSize = 2;
/** The size of a single block in persistent memory */
static constexpr uint16_t gBlockSize = 1 << 15; // 32KB
/**
* \brief This type represents a byte array used for persistent structures.
*
* A BDCC_Block is a PAX oriented data block with the following structure for 32KB:
* <ddc_range><ddc_cnt><sma_offset_0><data_offset_0> ...<sma_offset_n><data_offset_n>
* <sma_min_0><sma_max_0><data_vector_0> ... <sma_min_n><sma_max_n><data_vector_n>
* 0 ddc_range -> long (x2) - 8 Byte
* 8 ddc_cnt -> long - 4 Byte
* 12 free_space -> unsigned short
* for each attribute:
* 14 sma_offset_x -> unsigned short - 2 Byte (depends on block size)
* 16 data_offset_x -> unsigned short
* ...
*
* for each attribute (int, double):
* . sma_min_x -> size of attributes data type
* . sma_max_x -> size of attributes data type
* . data_vector -> size of attributes data type * ddc_cnt
* ...
*
* for each attribute (string - data starts at the end of the minipage):
* . sma_min_offset_x -> unsigned short
* . sma_max_offset_x -> unsigned short
* . data_offset_vector -> unsigned short * ddc_cnt
* . ...
* . data -> size of all strings + ddc_cnt (Nul termination)
*/
using BDCC_Block = typename std::array<uint8_t, gBlockSize>;
template<typename KeyType>
struct DataNode {
using KeyVector = std::array<KeyType, 8192>; // <KeyType, nvml::obj::allocator<KeyType>>;
DataNode() : next(nullptr), block(nullptr) {
keys = make_persistent<KeyVector>();
}
DataNode(BDCC_Block _block) : next(nullptr) {
block = make_persistent<BDCC_Block>(_block);
keys = make_persistent<KeyVector>();
}
persistent_ptr<struct DataNode> next;
persistent_ptr<BDCC_Block> block;
persistent_ptr<KeyVector> keys;
void clear() {
if (next) {
//delete_persistent<struct DataNode>(next);
next = nullptr;
}
if (block) {
delete_persistent<BDCC_Block>(block);
block = nullptr;
}
delete_persistent<struct DataNode>(this);
}
};
} /* end namespace nvm */
} /* end namespace pfabric */
#endif /* NVM_Block_hpp_ */
This diff is collapsed.
This diff is collapsed.
......@@ -14,39 +14,45 @@ using nvml::obj::delete_persistent;
using nvml::obj::make_persistent;
using nvml::obj::persistent_ptr;
namespace pfabric { namespace nvm {
#define SSO_CHARS 15
#define SSO_SIZE (SSO_CHARS + 1)
class PString {
public:
char* data() const { return str ? str.get() : const_cast<char*>(sso); }
void reset();
void set(std::string* value);
private:
char sso[SSO_SIZE];
persistent_ptr<char[]> str;
};
void PString::reset() {
pmemobj_tx_add_range_direct(sso, 1);
sso[0] = 0;
if (str) delete_persistent<char[]>(str, strlen(str.get()) + 1);
}
void PString::set(std::string* value) {
unsigned long length = value->length();
if (length <= SSO_CHARS) {
if (str) {
delete_persistent<char[]>(str, strlen(str.get()) + 1);
str = nullptr;
class PString {
public:
char *data() const { return str ? str.get() : const_cast<char *>(sso); }
void reset();
void set(std::string *value);
private:
char sso[SSO_SIZE];
persistent_ptr<char[]> str;
};
inline void PString::reset() {
pmemobj_tx_add_range_direct(sso, 1);
sso[0] = 0;
if (str) delete_persistent<char[]>(str, strlen(str.get()) + 1);
}
inline void PString::set(std::string *value) {
unsigned long length = value->length();
if (length <= SSO_CHARS) {
if (str) {
delete_persistent<char[]>(str, strlen(str.get()) + 1);
str = nullptr;
}
pmemobj_tx_add_range_direct(sso, SSO_SIZE);
strcpy(sso, value->c_str());
} else {
if (str) delete_persistent<char[]>(str, strlen(str.get()) + 1);
str = make_persistent<char[]>(length + 1);
strcpy(str.get(), value->c_str());
}
}
pmemobj_tx_add_range_direct(sso, SSO_SIZE);
strcpy(sso, value->c_str());
} else {
if (str) delete_persistent<char[]>(str, strlen(str.get()) + 1);
str = make_persistent<char[]>(length + 1);
strcpy(str.get(), value->c_str());
}
}
}} /* namespace pfabric::nvm */
#endif
This diff is collapsed.
......@@ -10,6 +10,7 @@
#include <vector>
#include "table/TableInfo.hpp"
#include "nvm/PString.hpp"
#include "nvml/include/libpmemobj++/allocator.hpp"
#include "nvml/include/libpmemobj++/detail/persistent_ptr_base.hpp"
......@@ -51,20 +52,20 @@ private:
p<ColumnInfo::ColumnType> mColType;
};
typedef std::initializer_list<std::pair<std::string, ColumnInfo::ColumnType>> ColumnInitList;
using ColumnInitList = std::initializer_list<std::pair<std::string, ColumnInfo::ColumnType>>;
class PTableInfo {
public:
typedef persistent_ptr<PTableInfo> PTableInfoPtr;
typedef std::vector<PColumnInfo, nvml::obj::allocator<PColumnInfo>> ColumnVector;
typedef ColumnVector::const_iterator ColumnIterator;
using PTableInfoPtr = persistent_ptr<PTableInfo>;
using ColumnVector = std::vector<PColumnInfo, nvml::obj::allocator<PColumnInfo>>;
using ColumnIterator = ColumnVector::const_iterator;
PTableInfo(){}
PTableInfo(const TableInfo& _tInfo, ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type)
: mName(_tInfo.tableName().c_str()), mKeyType(keyType){
PTableInfo(const TableInfo& _tInfo, ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type) : mKeyType(keyType){
auto pop = pool_by_vptr(this);
transaction::exec_tx(pop, [&] {
mName.set(const_cast<std::string*>(& _tInfo.tableName()));
mColumns = make_persistent<ColumnVector>();
for (const auto &c : _tInfo)
mColumns->push_back(PColumnInfo(pop, c.getName(), c.getType()));
......@@ -72,10 +73,10 @@ class PTableInfo {
}
PTableInfo(const std::string& name, ColumnInitList columns,
ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type) :
mName(name.c_str()), mKeyType(keyType) {
ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type) : mKeyType(keyType) {
auto pop = pool_by_vptr(this);
transaction::exec_tx(pop, [&] {
mName.set(const_cast<std::string*>(&name));
mColumns = make_persistent<ColumnVector>();
for (const auto &c : columns)
mColumns->push_back(PColumnInfo(pop, c.first, c.second));
......@@ -83,15 +84,15 @@ class PTableInfo {
}
PTableInfo(const std::string& name, const ColumnVector& columns,
ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type) :
mName(name.c_str()), mKeyType(keyType) {
ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type) : mKeyType(keyType) {
auto pop = pool_by_vptr(this);
transaction::exec_tx(pop, [&] {
mName.set(const_cast<std::string*>(&name));
mColumns = make_persistent<ColumnVector>(columns.begin(), columns.end());
});
}
const std::string tableName() const { return std::string(mName.get()); }
const std::string tableName() const { return std::string(mName.data()); }
std::string typeSignature() const;
......@@ -117,7 +118,7 @@ class PTableInfo {
auto pop = pool_by_vptr(this);
PTableInfoPtr tInfo_ptr = nullptr;
transaction::exec_tx(pop, [&] {
tInfo_ptr = make_persistent<PTableInfo>(std::string(mName.get()), *mColumns, mKeyType.get_ro());
tInfo_ptr = make_persistent<PTableInfo>(std::string(mName.data()), *mColumns, mKeyType.get_ro());
});
return tInfo_ptr;
}
......@@ -125,14 +126,14 @@ class PTableInfo {
ColumnIterator begin() const { return mColumns->begin(); }
ColumnIterator end() const { return mColumns->end(); }
private:
persistent_ptr<const char[]> mName;
// private:
PString mName;
persistent_ptr<ColumnVector> mColumns;
p<ColumnInfo::ColumnType> mKeyType;
};
}
typedef persistent_ptr<nvm::PTableInfo> PTableInfoPtr;
using PTableInfoPtr = persistent_ptr<nvm::PTableInfo>;
} /* namespace pfabric::nvm */
......
......@@ -30,6 +30,7 @@
#include "core/PFabricTypes.hpp"
#include "core/Tuple.hpp"
#include "nvm/DataNode.hpp"
#include "nvml/include/libpmemobj++/persistent_ptr.hpp"
......@@ -40,53 +41,6 @@ namespace pfabric {
namespace nvm {
//TODO: Find a more suitable position for these constants
/* Positions in NVM_Block */
const int gDDCRangePos1 = 0;
const int gDDCRangePos2 = 4;
const int gCountPos = 8;
const int gFreeSpacePos = 12;
const int gSmaOffsetPos = 14;
const int gDataOffsetPos = 16;
/* Sizes/Lengths in NVM_Block */
const int gFixedHeaderSize = 14;
const int gDDCValueSize = 4;
const int gAttrOffsetSize = 4;
const int gOffsetSize = 2;
/** The size of a single block in persistent memory */
static constexpr uint16_t gBlockSize = 1 << 15; // 32KB
/**
* \brief This type represents a byte array used for persistent structures.
*
* A NVM_Block is a PAX oriented data block with the following structure for 32KB:
* <ddc_range><ddc_cnt><sma_offset_0><data_offset_0> ...<sma_offset_n><data_offset_n>
* <sma_min_0><sma_max_0><data_vector_0> ... <sma_min_n><sma_max_n><data_vector_n>
* 0 ddc_range -> long (x2) - 8 Byte
* 8 ddc_cnt -> long - 4 Byte
* 12 free_space -> unsigned short
* for each attribute:
* 14 sma_offset_x -> unsigned short - 2 Byte (depends on block size)
* 16 data_offset_x -> unsigned short
* ...
*
* for each attribute (int, double):
* . sma_min_x -> size of attributes data type
* . sma_max_x -> size of attributes data type
* . data_vector -> size of attributes data type * ddc_cnt
* ...
*
* for each attribute (string - data starts at the end of the minipage):
* . sma_min_offset_x -> unsigned short
* . sma_max_offset_x -> unsigned short
* . data_offset_vector -> unsigned short * ddc_cnt
* . ...
* . data -> size of all strings + ddc_cnt (Nul termination)
*/
typedef typename std::array<uint8_t, gBlockSize> NVM_Block;
namespace detail {
/**************************************************************************//**
......@@ -110,7 +64,7 @@ struct get_helper;
*****************************************************************************/
template<typename T, std::size_t ID>
struct get_helper {
static T apply(persistent_ptr<NVM_Block> block, const uint16_t *offsets) {
static T apply(persistent_ptr<BDCC_Block> block, const uint16_t *offsets) {
T val;
uint8_t* ptr = reinterpret_cast<uint8_t*>(&val);
std::copy(block->begin() + offsets[ID], block->begin() + offsets[ID] + sizeof(T), ptr);
......@@ -128,7 +82,7 @@ struct get_helper {
*****************************************************************************/
template<std::size_t ID>
struct get_helper<std::string, ID> {
static std::string apply(persistent_ptr<NVM_Block> block, const uint16_t *offsets) {
static std::string apply(persistent_ptr<BDCC_Block> block, const uint16_t *offsets) {
return reinterpret_cast<char (&)[]>(block->at(offsets[ID]));
}
};
......@@ -143,7 +97,7 @@ struct get_helper<std::string, ID> {
*****************************************************************************/
template<std::size_t ID>
struct get_helper<int, ID> {
static int apply(persistent_ptr<NVM_Block> block, const uint16_t *offsets) {
static int apply(persistent_ptr<BDCC_Block> block, const uint16_t *offsets) {
return reinterpret_cast<int&>(block->at(offsets[ID]));
}
};
......@@ -158,7 +112,7 @@ struct get_helper<int, ID> {
*****************************************************************************/
template<std::size_t ID>
struct get_helper<double, ID> {
static double apply(persistent_ptr<NVM_Block> block, const uint16_t *offsets) {
static double apply(persistent_ptr<BDCC_Block> block, const uint16_t *offsets) {
return reinterpret_cast<double&>(block->at(offsets[ID]));
}
};
......@@ -188,7 +142,7 @@ struct getAll_helper;
*****************************************************************************/
template<class Tuple, std::size_t CurrentIndex>
struct getAll_helper {
static void apply(SmartPtr<Tuple> tptr, persistent_ptr<NVM_Block> block, const uint16_t* const offsets) {
static void apply(SmartPtr<Tuple> tptr, persistent_ptr<BDCC_Block> block, const uint16_t* const offsets) {
getAll_helper<Tuple, CurrentIndex - 1>::apply(tptr, block, offsets);
auto val = get_helper<typename Tuple::template getAttributeType<CurrentIndex-1>::type, CurrentIndex - 1>::apply(block, offsets);
tptr->template setAttribute<CurrentIndex-1>(val);
......@@ -205,7 +159,7 @@ struct getAll_helper {
*****************************************************************************/
template<class Tuple>
struct getAll_helper<Tuple, 1> {
static void apply(SmartPtr<Tuple> tptr, persistent_ptr<NVM_Block> block, const uint16_t* const offsets) {
static void apply(SmartPtr<Tuple> tptr, persistent_ptr<BDCC_Block> block, const uint16_t* const offsets) {
auto val = get_helper<typename Tuple::template getAttributeType<0>::type, 0>::apply(block, offsets);
tptr->template setAttribute<0>(val);
}
......@@ -240,7 +194,7 @@ struct PTuplePrinter;
*****************************************************************************/
template<class Tuple, std::size_t CurrentIndex>
struct PTuplePrinter {
static void print(std::ostream& os, persistent_ptr<NVM_Block> block, const uint16_t* offsets) {
static void print(std::ostream& os, persistent_ptr<BDCC_Block> block, const uint16_t* offsets) {
PTuplePrinter<Tuple, CurrentIndex - 1>::print(os, block, offsets);
auto val = get_helper<typename Tuple::template getAttributeType<CurrentIndex-1>::type, CurrentIndex - 1>::apply(block, offsets);
os << "," << val;
......@@ -257,7 +211,7 @@ struct PTuplePrinter {
*****************************************************************************/
template<class Tuple>
struct PTuplePrinter<Tuple, 1> {
static void print(std::ostream& os, persistent_ptr<NVM_Block> block, const uint16_t* offsets) {
static void print(std::ostream& os, persistent_ptr<BDCC_Block> block, const uint16_t* offsets) {
os << get_helper<typename Tuple::template getAttributeType<0>::type, 0>::apply(block, offsets);
}