Commit 14cff085 authored by Philipp Götze's avatar Philipp Götze
Browse files

Removed most heuristic sleeps

parent 569c94a8
Pipeline #184 failed with stages
in 46 minutes and 29 seconds
......@@ -19,7 +19,7 @@ endmacro(build_executable)
macro(build_catch_test arg)
include_directories("${PROJECT_SOURCE_DIR}/test")
add_executable( ${arg} "${arg}.cpp" $<TARGET_OBJECTS:TestMain>)
target_link_libraries( ${arg}
target_link_libraries( ${arg}
pfabric_core
${Boost_SYSTEM_LIBRARY}
${ROCKSDB_LIB}
......
......@@ -17,40 +17,40 @@
* along with PipeFabric. If not, see <http://www.gnu.org/licenses/>.
*/
#include "Dataflow.hpp"
using namespace pfabric;
Dataflow::BaseOpIterator Dataflow::addPublisher(Dataflow::BaseOpPtr op) {
publishers.push_back(op);
auto iter = publishers.end();
return --iter;
}
Dataflow::BaseOpIterator Dataflow::addPublisherList(const Dataflow::BaseOpList& lst) {
publishers.insert(publishers.end(), lst.begin(), lst.end());
auto iter = publishers.end();
std::advance(iter, -lst.size());
return iter;
}
void Dataflow::addSink(Dataflow::BaseOpPtr op) { sinks.push_back(op); }
/**
* @brief Returns the operator at the end of the publisher list.
*
* Returns the operator which acts as the publisher for the next
* added operator.
*
* @return
* the last operator in the publisher list
*/
#include "Dataflow.hpp"
using namespace pfabric;
Dataflow::BaseOpIterator Dataflow::addPublisher(Dataflow::BaseOpPtr op) {
publishers.push_back(op);
auto iter = publishers.end();
return --iter;
}
Dataflow::BaseOpIterator Dataflow::addPublisherList(const Dataflow::BaseOpList& lst) {
publishers.insert(publishers.end(), lst.begin(), lst.end());
auto iter = publishers.end();
std::advance(iter, -lst.size());
return iter;
}
void Dataflow::addSink(Dataflow::BaseOpPtr op) { sinks.push_back(op); }
/**
* @brief Returns the operator at the end of the publisher list.
*
* Returns the operator which acts as the publisher for the next
* added operator.
*
* @return
* the last operator in the publisher list
*/
Dataflow::BaseOpPtr Dataflow::getPublisher() { return publishers.back(); }
Dataflow::BaseOpIterator Dataflow::getPublishers(unsigned int num) {
auto iter = publishers.end();
std::advance(iter, -num);
return iter;
}
Dataflow::BaseOpIterator Dataflow::getPublishers(unsigned int num) {
auto iter = publishers.end();
std::advance(iter, -num);
return iter;
}
std::size_t Dataflow::size() const { return publishers.size(); }
std::size_t Dataflow::size() const { return publishers.size(); }
......@@ -17,10 +17,7 @@
* along with PipeFabric. If not, see <http://www.gnu.org/licenses/>.
*/
#include <boost/thread.hpp>
#include <boost/chrono.hpp>
#include "Topology.hpp"
#include "qop/ZMQSource.hpp"
using namespace pfabric;
......@@ -74,14 +71,21 @@ void Topology::prepare() {
}
}
void Topology::wait() {
void Topology::wait(const std::chrono::milliseconds &dur) {
if (!asyncStarted)
return;
std::lock_guard<std::mutex> guard(mMutex);
// let's wait until the function finished
for(auto &f : startupFutures)
f.get();
f.wait();
//TODO: wait for EndOfStream Punctuations on all sinks
//TODO: what about merging streams or no actual sinks?
std::unique_lock<std::mutex> lk(mCv_m);
const auto now = std::chrono::system_clock::now();
if (mCv.wait_until(lk, now + dur) == std::cv_status::timeout) {
//Timeout!
}
}
void Topology::runEvery(unsigned long secs) {
......@@ -159,14 +163,16 @@ Pipe<TStringPtr> Topology::newStreamFromREST(unsigned int port,
return Pipe<TStringPtr>(dataflow, dataflow->addPublisher(op));
}
Pipe<TStringPtr> Topology::newAsciiStreamFromZMQ(const std::string& path,
Pipe<TStringPtr> Topology::newAsciiStreamFromZMQ(const std::string& path, const std::string& syncPath,
ZMQParams::SourceType stype) {
auto op = std::make_shared<ZMQSource<TStringPtr> >(path, stype);
auto op = std::make_shared<ZMQSource<TStringPtr> >(path, syncPath, stype);
registerStartupFunction(std::bind(&ZMQSource<TStringPtr>::start, op.get()));
return Pipe<TStringPtr>(dataflow, dataflow->addPublisher(op));
}
Pipe<TBufPtr> Topology::newBinaryStreamFromZMQ(const std::string& path,
Pipe<TBufPtr> Topology::newBinaryStreamFromZMQ(const std::string& path, const std::string& syncPath,
ZMQParams::SourceType stype) {
auto op = std::make_shared<ZMQSource<TBufPtr> >(path, stype);
auto op = std::make_shared<ZMQSource<TBufPtr> >(path, syncPath, stype);
registerStartupFunction(std::bind(&ZMQSource<TBufPtr>::start, op.get()));
return Pipe<TBufPtr>(dataflow, dataflow->addPublisher(op));
}
......@@ -20,12 +20,15 @@
#ifndef Topology_hpp_
#define Topology_hpp_
#include <chrono>
#include <condition_variable>
#include <string>
#include <list>
#include <vector>
#include <future>
#include <mutex>
#include <chrono>
#include <boost/chrono.hpp>
#include <boost/thread.hpp>
#include "core/Tuple.hpp"
......@@ -102,6 +105,8 @@ namespace pfabric {
std::vector<std::future<unsigned long> > startupFutures; //< futures for the startup functions
std::vector<boost::thread> wakeupTimers; //< interruptible threads for runEvery queries
std::mutex mMutex; //< mutex for accessing startupFutures
std::condition_variable mCv; //< condition variable to check if sinks have received EndOfStream
std::mutex mCv_m;
DataflowPtr dataflow;
......@@ -174,7 +179,7 @@ namespace pfabric {
* If the topology was started asynchronously the call of wait()
* blocks until the execution stopped.
*/
void wait();
void wait(const std::chrono::milliseconds &dur = 500ms);
/**
* @brief Creates a pipe from a TextFileSource as input.
......@@ -289,9 +294,11 @@ namespace pfabric {
* a new pipe where ZMQSource acts as a producer.
*/
Pipe<TStringPtr> newAsciiStreamFromZMQ(const std::string& path,
const std::string& syncPath = "",
ZMQParams::SourceType stype = ZMQParams::SubscriberSource);
Pipe<TBufPtr> newBinaryStreamFromZMQ(const std::string& path,
const std::string& syncPath = "",
ZMQParams::SourceType stype = ZMQParams::SubscriberSource);
/**
......
......@@ -86,8 +86,7 @@ TEST_CASE("Controlling stream processing by a barrier", "[Barrier]") {
// set counter to 10 and send tuples 1, 2, 3, 4, 11, 12:
// => only tuples 1, 2, 3, 4 should arrive
mockup->start();
std::this_thread::sleep_for(std::chrono::milliseconds(100));
mockup->wait();
REQUIRE(mockup->numTuplesProcessed() == 4);
// now set counter to 13:
......@@ -95,7 +94,7 @@ TEST_CASE("Controlling stream processing by a barrier", "[Barrier]") {
mockup->addExpected({makeTuplePtr(11), makeTuplePtr(12)});
counter.set(13);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
mockup->wait();
REQUIRE(mockup->numTuplesProcessed() == 6);
// set counter to 25:
......@@ -103,6 +102,6 @@ TEST_CASE("Controlling stream processing by a barrier", "[Barrier]") {
mockup->addExpected({makeTuplePtr(20), makeTuplePtr(21), makeTuplePtr(22)});
counter.set(25);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
mockup->wait();
REQUIRE(mockup->numTuplesProcessed() == 9);
}
......@@ -55,8 +55,8 @@ TEST_CASE("Stream from matrix", "[FromMatrixTest]")
for(auto &tuple : inputs) {
matrix->insert(tuple);
}
using namespace std::chrono_literals;
std::this_thread::sleep_for(2s);
mockup->wait(2000ms);
REQUIRE(mockup->numTuplesProcessed() == size);
}
......@@ -66,8 +66,7 @@ TEST_CASE("Producing a data stream from inserts into a table", "[FromTable]") {
testTable->insert(i, *tp);
}
std::this_thread::sleep_for(std::chrono::milliseconds(100));
mockup->wait();
REQUIRE(mockup->numTuplesProcessed() == 10);
testTable->drop();
}
......@@ -97,9 +97,7 @@ TEST_CASE("Partitioning a data stream and merging the results.",
CREATE_DATA_LINK(merge, mockup);
mockup->start();
using namespace std::chrono_literals;
std::this_thread::sleep_for(std::chrono::milliseconds(100));
mockup->wait();
REQUIRE(mockup->numTuplesProcessed() == numTuples / 2);
}
......@@ -56,8 +56,7 @@ TEST_CASE("Decoupling producer and consumer via a queue", "[Queue]") {
CREATE_DATA_LINK(ch, mockup)
mockup->start();
std::this_thread::sleep_for(std::chrono::milliseconds(100));
mockup->wait();
REQUIRE(mockup->numTuplesProcessed() == expected.size());
}
......@@ -17,65 +17,47 @@
* along with PipeFabric. If not, see <http://www.gnu.org/licenses/>.
*/
#include <boost/filesystem.hpp>
#include <iostream>
#include <future>
// using namespace boost::filesystem;
#include "catch.hpp"
#include "fmt/format.h"
#include "SimpleWeb/client_http.hpp"
#include "core/Tuple.hpp"
#include "qop/RESTSource.hpp"
#include "qop/DataSink.hpp"
#include "qop/OperatorMacros.hpp"
#include "StreamMockup.hpp"
using namespace pfabric;
using namespace ns_types;
class TestConsumer : public SynchronizedDataSink<TStringPtr> {
public:
PFABRIC_SYNC_SINK_TYPEDEFS(TStringPtr)
TestConsumer() : tupleNum(0) {}
BIND_INPUT_CHANNEL_DEFAULT(InputDataChannel, TestConsumer, processDataElement);
BIND_INPUT_CHANNEL_DEFAULT(InputPunctuationChannel, TestConsumer, processPunctuation);
void processPunctuation(const PunctuationPtr& punctuation) {}
void processDataElement( const TStringPtr& data, const bool outdated ) {
std::string input (data->getAttribute<0>().begin_, data->getAttribute<0>().size_);
std::string expected = fmt::format("(\"key\": \"{0}\",\"value\": \"Always the same\")", tupleNum);
using HttpClient = SimpleWeb::Client<SimpleWeb::HTTP>;
REQUIRE(input == expected);
tupleNum++;
TEST_CASE("Receiving data via REST", "[RESTSource]" ) {
constexpr auto numTuples = 1000;
std::vector<TStringPtr> expected;
std::vector<std::string> stringVec;
for (int i = 0; i < numTuples; i++) {
auto param_string = fmt::format("(\"key\": \"{0}\",\"value\": \"Always the same\")", i);
stringVec.push_back(std::move(param_string));
expected.push_back(makeTuplePtr(StringRef(stringVec[i].c_str(), stringVec[i].size())));
}
private:
int tupleNum;
};
typedef SimpleWeb::Client<SimpleWeb::HTTP> HttpClient;
TEST_CASE("Receiving data via REST", "[RESTSource]" ) {
auto restSource = std::make_shared<RESTSource>(8099, "^/publish$", RESTSource::POST_METHOD);
auto consumer = std::make_shared<TestConsumer>();
CREATE_LINK(restSource, consumer);
auto mockup = std::make_shared<StreamMockup<TStringPtr, TStringPtr> >(expected, expected);
CREATE_LINK(restSource, mockup);
// note we have to start the REST server asynchronously
auto handle = std::async(std::launch::async, [&](std::shared_ptr<RESTSource> src){ src->start(); }, restSource);
/// NOTE: we have to start the REST server asynchronously
auto handle = std::async(std::launch::async, [&restSource, &stringVec](){
restSource->start();
});
std::this_thread::sleep_for(std::chrono::milliseconds(100));
HttpClient client("localhost:8099");
for (int i = 0; i < 100; i++) {
std::string param_string = fmt::format("(\"key\": \"{0}\",\"value\": \"Always the same\")", i);
auto res = client.request("POST", "/publish", param_string);
for (int i = 0; i < numTuples; i++) {
auto res = client.request("POST", "/publish", stringVec[i]);
}
restSource->stop();
handle.get();
REQUIRE(mockup->numTuplesProcessed() == numTuples);
}
......@@ -25,9 +25,11 @@
#include <boost/algorithm/string/find_iterator.hpp>
#include <boost/algorithm/string.hpp>
#include <vector>
#include <mutex>
#include <chrono>
#include <condition_variable>
#include <fstream>
#include <mutex>
#include <vector>
#include "core/Tuple.hpp"
#include "qop/Map.hpp"
......@@ -36,157 +38,171 @@
#include "qop/OperatorMacros.hpp"
using namespace std::chrono_literals;
namespace pfabric {
struct MockupHelper {
template <typename StreamElement>
static void readTuplesFromStream(std::ifstream& in, std::vector<StreamElement>& vec) {
std::string line;
std::vector<std::string> data(StreamElementTraits<StreamElement>::NUM_ATTRIBUTES);
while (getline(in, line)) {
int i = 0;
for (boost::split_iterator<std::string::iterator> it =
boost::make_split_iterator(line, boost::first_finder(",", boost::is_iequal()));
it != boost::split_iterator<std::string::iterator>(); it++) {
data[i++] = boost::copy_range<std::string>(*it);
}
auto tp = StreamElementTraits<StreamElement>::create(data);
vec.push_back(tp);
}
}
template <typename StreamElement>
static void readTuplesFromStream(std::ifstream& in, std::vector<StreamElement>& vec) {
std::string line;
std::vector<std::string> data(StreamElementTraits<StreamElement>::NUM_ATTRIBUTES);
while (getline(in, line)) {
int i = 0;
for (boost::split_iterator<std::string::iterator> it =
boost::make_split_iterator(line, boost::first_finder(",", boost::is_iequal()));
it != boost::split_iterator<std::string::iterator>(); it++) {
data[i++] = boost::copy_range<std::string>(*it);
}
auto tp = StreamElementTraits<StreamElement>::create(data);
vec.push_back(tp);
}
}
};
template<
typename InputStreamElement,
typename OutputStreamElement>
class StreamMockup :
public DataSource< InputStreamElement >,
public SynchronizedDataSink< OutputStreamElement >
public DataSource< InputStreamElement >,
public SynchronizedDataSink< OutputStreamElement >
{
public:
typedef DataSource< InputStreamElement > SourceBase;
typedef typename SourceBase::OutputDataChannel OutputDataChannel;
typedef typename SourceBase::OutputPunctuationChannel OutputPunctuationChannel;
typedef typename SourceBase::OutputDataElementTraits OutputDataElementTraits;
typedef SynchronizedDataSink< OutputStreamElement > SinkBase;
typedef typename SinkBase::InputDataChannel InputDataChannel;
typedef typename SinkBase::InputPunctuationChannel InputPunctuationChannel;
typedef typename SinkBase::InputDataElementTraits InputDataElementTraits;
typedef std::function<bool(const OutputStreamElement& lhs, const OutputStreamElement& rhs)> CompareFunc;
/**
*
* @param input
* @param expected
* @param ordered
* @param cFunc
*/
StreamMockup(const std::vector<InputStreamElement>& input,
const std::vector<OutputStreamElement>& expected, bool ordered = true, CompareFunc cFunc = nullptr) :
inputTuples(input), expectedTuples(expected), tuplesProcessed(0), compareOrdered(ordered),
compareFunc(cFunc) {
if (!compareOrdered)
BOOST_ASSERT_MSG(compareFunc != nullptr, "no comparison predicated given.");
}
StreamMockup(const std::string& inputStream, const std::string& expectedStream)
: tuplesProcessed(0), compareOrdered(true) {
auto inputFile = std::string(TEST_DATA_DIRECTORY) + inputStream;
std::ifstream input(inputFile);
REQUIRE(input.is_open());
auto expectedFile = TEST_DATA_DIRECTORY + expectedStream;
std::ifstream expected(expectedFile);
REQUIRE(expected.is_open());
MockupHelper::readTuplesFromStream<InputStreamElement>(input, inputTuples);
MockupHelper::readTuplesFromStream<OutputStreamElement>(expected, expectedTuples);
}
void start() {
const bool outdated = false;
for (auto tp : inputTuples) {
this->getOutputDataChannel().publish( tp, outdated );
}
}
int numTuplesProcessed() const {
int res = 0;
{
std::lock_guard<std::mutex> guard(mMtx);
res = tuplesProcessed;
}
return res;
}
void addExpected(const std::vector<OutputStreamElement>& expected) {
std::lock_guard<std::mutex> guard(mMtx);
expectedTuples.insert(expectedTuples.end(), expected.begin(), expected.end());
}
/**
* @brief Bind the callback for the data channel.
*/
BIND_INPUT_CHANNEL_DEFAULT( InputDataChannel, StreamMockup, processDataElement );
/**
* @brief Bind the callback for the punctuation channel.
*/
BIND_INPUT_CHANNEL_DEFAULT( InputPunctuationChannel, StreamMockup, processPunctuation );
typedef DataSource< InputStreamElement > SourceBase;
typedef typename SourceBase::OutputDataChannel OutputDataChannel;
typedef typename SourceBase::OutputPunctuationChannel OutputPunctuationChannel;
typedef typename SourceBase::OutputDataElementTraits OutputDataElementTraits;
typedef SynchronizedDataSink< OutputStreamElement > SinkBase;
typedef typename SinkBase::InputDataChannel InputDataChannel;
typedef typename SinkBase::InputPunctuationChannel InputPunctuationChannel;
typedef typename SinkBase::InputDataElementTraits InputDataElementTraits;
typedef std::function<bool(const OutputStreamElement& lhs, const OutputStreamElement& rhs)> CompareFunc;
/**
*
* @param input
* @param expected
* @param ordered
* @param cFunc
*/
StreamMockup(const std::vector<InputStreamElement>& input,
const std::vector<OutputStreamElement>& expected, bool ordered = true, CompareFunc cFunc = nullptr) :
inputTuples(input), expectedTuples(expected), tuplesProcessed(0), compareOrdered(ordered),
compareFunc(cFunc) {
if (!compareOrdered)
BOOST_ASSERT_MSG(compareFunc != nullptr, "no comparison predicated given.");
}
StreamMockup(const std::string& inputStream, const std::string& expectedStream)
: tuplesProcessed(0), compareOrdered(true) {
auto inputFile = std::string(TEST_DATA_DIRECTORY) + inputStream;
std::ifstream input(inputFile);
REQUIRE(input.is_open());
auto expectedFile = TEST_DATA_DIRECTORY + expectedStream;
std::ifstream expected(expectedFile);
REQUIRE(expected.is_open());
MockupHelper::readTuplesFromStream<InputStreamElement>(input, inputTuples);
MockupHelper::readTuplesFromStream<OutputStreamElement>(expected, expectedTuples);
}
void start() {
const bool outdated = false;
auto i = 1;
for (auto tp : inputTuples) {
this->getOutputDataChannel().publish( tp, outdated );
}
}
void wait(const std::chrono::milliseconds dur = 1000ms) {
std::unique_lock<std::mutex> lk(mCv_m);
const auto now = std::chrono::system_clock::now();
if (mCv.wait_until(lk, now + dur) == std::cv_status::timeout) {
std::cerr << "Mockup - waiting timed out.\n";
}
}
int numTuplesProcessed() const {
int res = 0;
{
std::lock_guard<std::mutex> guard(mMtx);
res = tuplesProcessed;
}
return res;
}
void addExpected(const std::vector<OutputStreamElement>& expected) {
std::lock_guard<std::mutex> guard(mMtx);
expectedTuples.insert(expectedTuples.end(), expected.begin(), expected.end());
}
/**
* @brief Bind the callback for the data channel.
*/
BIND_INPUT_CHANNEL_DEFAULT( InputDataChannel, StreamMockup, processDataElement );
/**
* @brief Bind the callback for the punctuation channel.
*/
BIND_INPUT_CHANNEL_DEFAULT( InputPunctuationChannel, StreamMockup, processPunctuation );
private:
void processDataElement( const OutputStreamElement& data, const bool outdated ) {
std::lock_guard<std::mutex> guard(mMtx);
// std::cout << "StreamMockup::processDataElement: " << data << std::endl;
REQUIRE(unsigned(tuplesProcessed) < expectedTuples.size());
if (compareOrdered) {
// If we can compare tuples in their order of arrival everything is easy:
// simply compare the current tuple with the expected tuple at the
// same position.
REQUIRE(data->data() == expectedTuples[tuplesProcessed]->data());
// TODO: handle Null values
for (auto i = 0u; i < OutputDataElementTraits::NUM_ATTRIBUTES; i++) {
REQUIRE(data->isNull(i) == expectedTuples[tuplesProcessed]->isNull(i));
}
tuplesProcessed++;
}
else {
// Otherwise, more more is needed: first, we store the incoming tuple.
processedTuples.push_back(data);
if (++tuplesProcessed == expectedTuples.size()) {
// if we got enough tuples ...
REQUIRE(processedTuples.size() == expectedTuples.size());
// we sort both the processed and the expected tuples using the given
// comparison function
std::sort(processedTuples.begin(), processedTuples.end(), compareFunc);
std::sort(expectedTuples.begin(), expectedTuples.end(), compareFunc);
// the difference of these two vectors should now be empty
std::vector<OutputStreamElement> res;
std::set_difference(processedTuples.begin(), processedTuples.end(),
expectedTuples.begin(), expectedTuples.end(), std::back_inserter(res), compareFunc);
REQUIRE(res.empty());
}
}