Commit 12a94e67 authored by Philipp Götze's avatar Philipp Götze
Browse files

Backup

parent 88d2aa18
......@@ -56,7 +56,7 @@ option(USE_NVML_TABLE
#CMAKE_FORCE_CXX_COMPILER(icpc "Intel C++ Compiler")
# C++ compiler flags
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -Wall -Wno-deprecated -g -O3 -Wsign-compare")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -Wall -Wno-deprecated -g -O0 -Wsign-compare")
if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-local-typedefs -Wno-#pragma-messages")
elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
......@@ -127,10 +127,10 @@ if (USE_NVML_TABLE)
message(STATUS "using NVML based persistent table")
add_definitions(-DUSE_NVML_TABLE)
set (NVML_LIBRARIES
"${THIRD_PARTY_DIR}/nvml/lib/libpmemblk.a"
"${THIRD_PARTY_DIR}/nvml/lib/libpmemlog.a"
# "${THIRD_PARTY_DIR}/nvml/lib/libpmemblk.a"
# "${THIRD_PARTY_DIR}/nvml/lib/libpmemlog.a"
"${THIRD_PARTY_DIR}/nvml/lib/libpmemobj.a"
"${THIRD_PARTY_DIR}/nvml/lib/libpmempool.a"
# "${THIRD_PARTY_DIR}/nvml/lib/libpmempool.a"
"${THIRD_PARTY_DIR}/nvml/lib/libpmem.a"
# "${THIRD_PARTY_DIR}/nvml/lib/libvmem.a"
# "${THIRD_PARTY_DIR}/nvml/lib/libvmmalloc.a"
......@@ -220,7 +220,7 @@ else()
set (ZEROMQ_SOURCES "")
endif (ZEROMQ_FOUND)
add_library(pfabric_core SHARED
set(core_sources
core/TimestampHelper.cpp
core/Punctuation.cpp
qop/TextFileSource.cpp
......@@ -230,21 +230,41 @@ add_library(pfabric_core SHARED
dsl/Topology.cpp
dsl/Dataflow.cpp
dsl/PFabricContext.cpp
table/TableInfo.cpp
${ZEROMQ_SOURCES}
# we need this as dependency to download the sources from github
${THIRD_PARTY_DIR}/json
${THIRD_PARTY_DIR}/fmt
${THIRD_PARTY_DIR}/SimpleWeb
${THIRD_PARTY_DIR}/rocksdb
${THIRD_PARTY_DIR}/nvml
${THIRD_PARTY_DIR}/catch
)
if(USE_ROCKSDB_TABLE)
set(core_sources
${core_sources}
${THIRD_PARTY_DIR}/rocksdb
)
endif()
if(USE_NVML_TABLE)
set(core_sources
${core_sources}
nvm/PTableInfo.cpp
${THIRD_PARTY_DIR}/nvml
)
else()
set(core_sources
${core_sources}
nvm/VTableInfo.cpp
)
endif(USE_NVML_TABLE)
add_library(pfabric_core SHARED
${core_sources}
)
target_link_libraries(pfabric_core
${BOOST_LIBRARIES}
${ZEROMQ_LIBRARIES}
${BOOST_LIBRARIES}
${ZEROMQ_LIBRARIES}
)
#-----------------------------------------------------------------------------------------
......@@ -271,7 +291,7 @@ add_subdirectory(qcomp)
#
# Demo project
#
add_subdirectory(demo)
#add_subdirectory(demo)
#-----------------------------------------------------------------------------------------
#
......
......@@ -18,8 +18,13 @@
* along with this program; see the file LICENSE.
* If not you can find the GPL at http://www.gnu.org/copyleft/gpl.html
*/
#include <string>
#include "PFabricContext.hpp"
#include <sstream>
#include "dsl/PFabricContext.hpp"
#include "dsl/Topology.hpp"
#include "table/BaseTable.hpp"
#include "table/TableInfo.hpp"
using namespace pfabric;
......@@ -40,8 +45,10 @@ TableInfoPtr PFabricContext::getTableInfo(const std::string& tblName) {
return it->second->tableInfo();
}
else {
// TODO: shouldn't we throw an exception here???
std::cout << "table not found: '" << tblName << "' : " << mTableSet.size() << std::endl;
return std::shared_ptr<TableInfo>();
// std::cout << "table not found: '" << tblName << "' : " << mTableSet.size() << std::endl;
//return std::shared_ptr<TableInfo>();
std::stringstream errMsg;
errMsg << "table not found: '" << tblName << "' : " << mTableSet.size() << '\n';
throw TableException(errMsg.str().c_str());
}
}
......@@ -21,13 +21,21 @@
#ifndef PFabricContext_hpp_
#define PFabricContext_hpp_
#include <string>
#include <iostream>
#include <map>
#include <memory>
#include <string>
#include "core/PFabricTypes.hpp"
#include "table/Table.hpp"
#include "dsl/Dataflow.hpp"
#include "qop/Queue.hpp"
#include "dsl/Topology.hpp"
#include "table/Table.hpp"
#include "table/TableException.hpp"
#include "table/TableInfo.hpp"
namespace pfabric {
class Topology;
} /* namespace pfabric */
namespace pfabric {
......@@ -168,7 +176,7 @@ public:
}
private:
typedef std::shared_ptr<BaseTable> BaseTablePtr;
using BaseTablePtr = typename std::shared_ptr<BaseTable>;
std::map<std::string, BaseTablePtr> mTableSet; //< a dictionary collecting all existing tables
std::map<std::string, Dataflow::BaseOpPtr> mStreamSet; //< a dictionary collecting all named streams
......
......@@ -22,7 +22,11 @@
#ifndef BDCCInfo_hpp_
#define BDCCInfo_hpp_
namespace pfabric {
#include <nvm/PTableInfo.hpp>
#include <map>
#include <unordered_map>
namespace pfabric { namespace nvm {
/**************************************************************************//**
* \brief Info structure about the BDCC meta data.
......@@ -42,6 +46,6 @@ struct BDCCInfo {
};/* struct BDCCInfo */
} /* namespace pfabric */
}} /* namespace pfabric::nvm */
#endif /* PTuple_hpp_ */
#include "PTableInfo.hpp"
#include <sstream>
using namespace pfabric::nvm;
std::string PTableInfo::generateTypeDef() const {
std::ostringstream os;
bool first = true;
os << "TuplePtr<Tuple<";
for (auto& col : *mColumns) {
if (first)
first = false;
else
os << ", ";
os << col.getType();
}
os << ">>";
return os.str();
}
std::string PTableInfo::typeSignature() const {
std::ostringstream os;
os << "[";
for (auto& col : *mColumns) {
switch (col.getType()) {
case ColumnInfo::Void_Type:
os << "V";
break;
case ColumnInfo::Int_Type:
os << "i";
break;
case ColumnInfo::Double_Type:
os << "d";
break;
case ColumnInfo::String_Type:
os << "S";
break;
}
}
os << "]";
return os.str();
}
int PTableInfo::findColumnByName(const std::string& colName) const {
for (std::size_t i = 0; i < (*mColumns).size(); i++) {
if (mColumns->at(i).getName() == colName) return (int)i;
}
return -1;
}
std::ostream& operator<<(std::ostream& os, pfabric::nvm::ColumnInfo::ColumnType ct) {
using namespace pfabric::nvm;
switch (ct) {
case ColumnInfo::Void_Type:
os << "";
break;
case ColumnInfo::Int_Type:
os << "int";
break;
case ColumnInfo::Double_Type:
os << "double";
break;
case ColumnInfo::String_Type:
os << "std::string";
break;
}
return os;
}
#ifndef PTableInfo_hpp_
#define PTableInfo_hpp_
#include <iostream>
#include <memory>
#include <string>
#include <array>
#include <vector>
#include "nvml/include/libpmemobj++/allocator.hpp"
#include "nvml/include/libpmemobj++/detail/persistent_ptr_base.hpp"
#include "nvml/include/libpmemobj++/make_persistent.hpp"
#include "nvml/include/libpmemobj++/p.hpp"
#include "nvml/include/libpmemobj++/persistent_ptr.hpp"
#include "nvml/include/libpmemobj++/transaction.hpp"
#include "nvml/include/libpmemobj++/utils.hpp"
using nvml::obj::delete_persistent;
using nvml::obj::make_persistent;
using nvml::obj::p;
using nvml::obj::persistent_ptr;
using nvml::obj::pool_by_vptr;
using nvml::obj::transaction;
namespace pfabric { namespace nvm {
class ColumnInfo {
public:
enum ColumnType { Void_Type, Int_Type, Double_Type, String_Type };
ColumnInfo() : mColName(""), mColType(Void_Type) {}
ColumnInfo(const std::string& n, ColumnType ct) : mColName(n.c_str()), mColType(ct) {}
const std::string getName() const { return mColName.get(); }
const ColumnType getType() const { return mColType.get_ro(); }
bool operator==(const ColumnInfo& other) const {
return (strcmp(mColName.get(), other.mColName.get()) == 0 && mColType.get_ro() == other.mColType.get_ro());
}
bool operator<(const ColumnInfo& other) const {
return strcmp(mColName.get(), other.mColName.get()) <= 0;
}
private:
persistent_ptr<const char[]> mColName;
p<ColumnType> mColType;
};
/** TODO: For later, template implementation
template< bool isFixedSize >
struct IsFixedSize {
static const bool IS_FIXED_SIZE = isFixedSize;
};
template < ColumnInfo::ColumnType ColType >
struct ColTypeTraits;
template <>
struct ColTypeTraits< ColumnInfo::Int_Type > : IsFixedSize<true> {
static const std::size_t COLUMN_SIZE = sizeof(int32_t);
};
template <>
struct ColTypeTraits< ColumnInfo::Double_Type > : IsFixedSize<true> {
static const std::size_t COLUMN_SIZE = sizeof(double);
};
template <>
struct ColTypeTraits< ColumnInfo::String_Type > : IsFixedSize<false>{};
using Offset = std::size_t;
template< ColumnInfo::ColumnType ColType, typename Col >
typename std::enable_if< not ColTypeTraits< ColType >::IS_FIXED_SIZE, Offset >::type getSize(const Col& t ) {
//TODO: runtime size calculation
}
template< ColumnInfo::ColumnType ColType, typename Col >
typename std::enable_if< ColTypeTraits< ColType >::IS_FIXED_SIZE, Offset >::type getSize(const Col& t ) {
using ColTraits = ColTypeTraits<ColType>;
return ColTraits::COLUMN_SIZE;
}
*/
class PTableInfo {
public:
typedef persistent_ptr<PTableInfo> TableInfoPtr;
typedef std::vector<ColumnInfo, nvml::obj::allocator<ColumnInfo>> ColumnVector;
typedef ColumnVector::const_iterator ColumnIterator;
PTableInfo(){}
PTableInfo(const std::string& name, ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type)
: mName(name.c_str()), mKeyType(keyType) {}
PTableInfo(const std::string& name, std::initializer_list<ColumnInfo> columns,
ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type) :
mName(name.c_str()), mKeyType(keyType) {
auto pop = pool_by_vptr(this);
transaction::exec_tx(pop, [&] {
mColumns = make_persistent<ColumnVector>(columns);
});
}
PTableInfo(const std::string& name, const ColumnVector& columns,
ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type) :
mName(name.c_str()), mKeyType(keyType) {
auto pop = pool_by_vptr(this);
transaction::exec_tx(pop, [&] {
mColumns = make_persistent<ColumnVector>(columns.begin(), columns.end());
});
}
const std::string tableName() const { return std::string(mName.get()); }
std::string typeSignature() const;
std::string generateTypeDef() const;
ColumnInfo::ColumnType typeOfKey() const { return mKeyType.get_ro(); }
int findColumnByName(const std::string& colName) const;
const ColumnInfo& columnInfo(int pos) const { return mColumns->at(pos); }
const std::size_t numColumns() const { return mColumns->size(); }
void setColumns(const ColumnVector vec) {
auto pop = pool_by_vptr(this);
transaction::exec_tx(pop, [&] {
delete_persistent<ColumnVector>(mColumns);
mColumns = make_persistent<ColumnVector>(vec.cbegin(), vec.cend());
});
}
TableInfoPtr makeShared() const {
auto pop = pool_by_vptr(this);
TableInfoPtr tInfo_ptr = nullptr;
transaction::exec_tx(pop, [&] {
tInfo_ptr = make_persistent<PTableInfo>(std::string(mName.get()), *mColumns, mKeyType.get_ro());
});
return tInfo_ptr;
}
ColumnIterator begin() const { return mColumns->begin(); }
ColumnIterator end() const { return mColumns->end(); }
private:
persistent_ptr<const char[]> mName;
persistent_ptr<ColumnVector> mColumns;
p<ColumnInfo::ColumnType> mKeyType;
};
}
typedef persistent_ptr<nvm::PTableInfo> TableInfoPtr;
} /* namespace pfabric::nvm */
std::ostream& operator<<(std::ostream& os, pfabric::nvm::ColumnInfo::ColumnType ct);
namespace std {
template <>
struct hash<pfabric::nvm::ColumnInfo> {
std::size_t operator()(const pfabric::nvm::ColumnInfo& c) const {
return std::hash<std::string>()(c.getName());
}
};
}
#endif
......@@ -28,12 +28,13 @@
#include <cstdint>
#include <string>
#include <core/PFabricTypes.hpp>
#include <core/Tuple.hpp>
#include "core/PFabricTypes.hpp"
#include "core/Tuple.hpp"
#include <libpmemobj++/persistent_ptr.hpp>
#include "nvml/include/libpmemobj++/persistent_ptr.hpp"
using nvml::obj::persistent_ptr;
using nvml::obj::p;
namespace pfabric {
......@@ -300,7 +301,7 @@ public:
***************************************************************************/
template<std::size_t ID>
inline auto getAttribute() {
return detail::get_helper<typename getAttributeType<ID>::type, ID>::apply(block, offsets.data());
return detail::get_helper<typename getAttributeType<ID>::type, ID>::apply(block, offsets.get_ro().data());
}
/************************************************************************//**
......@@ -313,7 +314,7 @@ public:
***************************************************************************/
template<std::size_t ID>
inline auto get() {
return detail::get_helper<typename getAttributeType<ID>::type, ID>::apply(block, offsets.data());
return detail::get_helper<typename getAttributeType<ID>::type, ID>::apply(block, offsets.get_ro().data());
}
/************************************************************************//**
......@@ -326,7 +327,7 @@ public:
***************************************************************************/
template<std::size_t ID>
inline auto getAttribute() const {
return detail::get_helper<typename getAttributeType<ID>::type, ID>::apply(block, offsets.data());
return detail::get_helper<typename getAttributeType<ID>::type, ID>::apply(block, offsets.get_ro().data());
}
/************************************************************************//**
......@@ -339,7 +340,7 @@ public:
***************************************************************************/
template<std::size_t ID>
inline auto get() const {
return detail::get_helper<typename getAttributeType<ID>::type, ID>::apply(block, offsets.data());
return detail::get_helper<typename getAttributeType<ID>::type, ID>::apply(block, offsets.get_ro().data());
}
/************************************************************************//**
......@@ -349,13 +350,13 @@ public:
* the output stream to print the tuple
***************************************************************************/
void print(std::ostream& os) {
detail::PTuplePrinter<Tuple, NUM_ATTRIBUTES>::print(os, block, offsets.data());
detail::PTuplePrinter<Tuple, NUM_ATTRIBUTES>::print(os, block, offsets.get_ro().data());
}
private:
persistent_ptr<NVM_Block> block;
std::array<uint16_t, NUM_ATTRIBUTES> offsets;
p<std::array<uint16_t, NUM_ATTRIBUTES>> offsets;
}; /* class PTuplePtr */
......
/*
* Copyright 2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* * Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef EXAMPLES_CTREE_MAP_PERSISTENT_HPP
#define EXAMPLES_CTREE_MAP_PERSISTENT_HPP
#include <cstdint>
#include <functional>
#include <stdlib.h>
#include "ex_common.h"
#include "nvml/include/libpmemobj++/make_persistent.hpp"
#include "nvml/include/libpmemobj++/make_persistent_atomic.hpp"
#include "nvml/include/libpmemobj++/p.hpp"
#include "nvml/include/libpmemobj++/persistent_ptr.hpp"
#include "nvml/include/libpmemobj++/pool.hpp"
#include "nvml/include/libpmemobj++/transaction.hpp"
#include "nvml/include/libpmemobj++/utils.hpp"
#define BIT_IS_SET(n, i) (!!((n) & (1ULL << (i))))
namespace nvobj = nvml::obj;
namespace examples
{
/**
* C++ implementation of a persistent ctree.
*
* Based on the volatile version. This version was implemented to show how much
* effort is needed to convert a volatile structure into a persistent one using
* C++ obj bindings. All API functions are atomic in respect to persistency.
*/
template <typename K, typename T>
class ctree_map_p {
public:
/** Convenience typedef for the key type. */
typedef K key_type;
/** Convenience typedef for the value type. */
typedef nvobj::persistent_ptr<T> value_type;
/** Convenience typedef for the callback function. */
typedef std::function<int(key_type, value_type, void *)> callback;
/**
* Default constructor.
*/
ctree_map_p()
{
auto pop = nvobj::pool_by_vptr(this);
nvobj::transaction::exec_tx(pop, [&] {
this->root = nvobj::make_persistent<entry>();
});
}
/**
* Insert or update the given value under the given key.
*
* The map takes ownership of the value.
*
* @param key The key to insert under.
* @param value The value to be inserted.
*
* @return 0 on success, negative values on error.
*/
int
insert(key_type key, value_type value)
{
auto dest_entry = root;
while (dest_entry->inode != nullptr) {
auto n = dest_entry->inode;
dest_entry = n->entries[BIT_IS_SET(key, n->diff)];
}
entry e(key, value);
auto pop = nvobj::pool_by_vptr(this);
nvobj::transaction::exec_tx(pop, [&] {
if (dest_entry->key == 0 || dest_entry->key == key) {
nvobj::delete_persistent<T>(dest_entry->value);
*dest_entry = e;
} else {
insert_leaf(&e, find_crit_bit(dest_entry->key,
key));
}
});
return 0;
}
/**
* Allocating insert.
*
* Creates a new value_type instance and inserts it into the tree.
*
* @param key The key to insert under.
* @param args variadic template parameter for object construction
* arguments.
*
* @return 0 on success, negative values on error.
*/
template <typename... Args>
int
insert_new(key_type key, const Args &... args)
{
auto pop = nvobj::pool_by_vptr(this);
nvobj::transaction::exec_tx(pop, [&] {
return insert(key, nvobj::make_persistent<T>(args...));