Commit 14cff085 authored by Philipp Götze's avatar Philipp Götze
Browse files

Removed most heuristic sleeps

parent 569c94a8
Pipeline #184 failed with stages
in 46 minutes and 29 seconds
......@@ -17,26 +17,26 @@
* along with PipeFabric. If not, see <http://www.gnu.org/licenses/>.
*/
#include "Dataflow.hpp"
#include "Dataflow.hpp"
using namespace pfabric;
using namespace pfabric;
Dataflow::BaseOpIterator Dataflow::addPublisher(Dataflow::BaseOpPtr op) {
Dataflow::BaseOpIterator Dataflow::addPublisher(Dataflow::BaseOpPtr op) {
publishers.push_back(op);
auto iter = publishers.end();
return --iter;
}
}
Dataflow::BaseOpIterator Dataflow::addPublisherList(const Dataflow::BaseOpList& lst) {
Dataflow::BaseOpIterator Dataflow::addPublisherList(const Dataflow::BaseOpList& lst) {
publishers.insert(publishers.end(), lst.begin(), lst.end());
auto iter = publishers.end();
std::advance(iter, -lst.size());
return iter;
}
}
void Dataflow::addSink(Dataflow::BaseOpPtr op) { sinks.push_back(op); }
void Dataflow::addSink(Dataflow::BaseOpPtr op) { sinks.push_back(op); }
/**
/**
* @brief Returns the operator at the end of the publisher list.
*
* Returns the operator which acts as the publisher for the next
......@@ -47,10 +47,10 @@
*/
Dataflow::BaseOpPtr Dataflow::getPublisher() { return publishers.back(); }
Dataflow::BaseOpIterator Dataflow::getPublishers(unsigned int num) {
Dataflow::BaseOpIterator Dataflow::getPublishers(unsigned int num) {
auto iter = publishers.end();
std::advance(iter, -num);
return iter;
}
}
std::size_t Dataflow::size() const { return publishers.size(); }
std::size_t Dataflow::size() const { return publishers.size(); }
......@@ -17,10 +17,7 @@
* along with PipeFabric. If not, see <http://www.gnu.org/licenses/>.
*/
#include <boost/thread.hpp>
#include <boost/chrono.hpp>
#include "Topology.hpp"
#include "qop/ZMQSource.hpp"
using namespace pfabric;
......@@ -74,14 +71,21 @@ void Topology::prepare() {
}
}
void Topology::wait() {
void Topology::wait(const std::chrono::milliseconds &dur) {
if (!asyncStarted)
return;
std::lock_guard<std::mutex> guard(mMutex);
// let's wait until the function finished
for(auto &f : startupFutures)
f.get();
f.wait();
//TODO: wait for EndOfStream Punctuations on all sinks
//TODO: what about merging streams or no actual sinks?
std::unique_lock<std::mutex> lk(mCv_m);
const auto now = std::chrono::system_clock::now();
if (mCv.wait_until(lk, now + dur) == std::cv_status::timeout) {
//Timeout!
}
}
void Topology::runEvery(unsigned long secs) {
......@@ -159,14 +163,16 @@ Pipe<TStringPtr> Topology::newStreamFromREST(unsigned int port,
return Pipe<TStringPtr>(dataflow, dataflow->addPublisher(op));
}
Pipe<TStringPtr> Topology::newAsciiStreamFromZMQ(const std::string& path,
Pipe<TStringPtr> Topology::newAsciiStreamFromZMQ(const std::string& path, const std::string& syncPath,
ZMQParams::SourceType stype) {
auto op = std::make_shared<ZMQSource<TStringPtr> >(path, stype);
auto op = std::make_shared<ZMQSource<TStringPtr> >(path, syncPath, stype);
registerStartupFunction(std::bind(&ZMQSource<TStringPtr>::start, op.get()));
return Pipe<TStringPtr>(dataflow, dataflow->addPublisher(op));
}
Pipe<TBufPtr> Topology::newBinaryStreamFromZMQ(const std::string& path,
Pipe<TBufPtr> Topology::newBinaryStreamFromZMQ(const std::string& path, const std::string& syncPath,
ZMQParams::SourceType stype) {
auto op = std::make_shared<ZMQSource<TBufPtr> >(path, stype);
auto op = std::make_shared<ZMQSource<TBufPtr> >(path, syncPath, stype);
registerStartupFunction(std::bind(&ZMQSource<TBufPtr>::start, op.get()));
return Pipe<TBufPtr>(dataflow, dataflow->addPublisher(op));
}
......@@ -20,12 +20,15 @@
#ifndef Topology_hpp_
#define Topology_hpp_
#include <chrono>
#include <condition_variable>
#include <string>
#include <list>
#include <vector>
#include <future>
#include <mutex>
#include <chrono>
#include <boost/chrono.hpp>
#include <boost/thread.hpp>
#include "core/Tuple.hpp"
......@@ -102,6 +105,8 @@ namespace pfabric {
std::vector<std::future<unsigned long> > startupFutures; //< futures for the startup functions
std::vector<boost::thread> wakeupTimers; //< interruptible threads for runEvery queries
std::mutex mMutex; //< mutex for accessing startupFutures
std::condition_variable mCv; //< condition variable to check if sinks have received EndOfStream
std::mutex mCv_m;
DataflowPtr dataflow;
......@@ -174,7 +179,7 @@ namespace pfabric {
* If the topology was started asynchronously the call of wait()
* blocks until the execution stopped.
*/
void wait();
void wait(const std::chrono::milliseconds &dur = 500ms);
/**
* @brief Creates a pipe from a TextFileSource as input.
......@@ -289,9 +294,11 @@ namespace pfabric {
* a new pipe where ZMQSource acts as a producer.
*/
Pipe<TStringPtr> newAsciiStreamFromZMQ(const std::string& path,
const std::string& syncPath = "",
ZMQParams::SourceType stype = ZMQParams::SubscriberSource);
Pipe<TBufPtr> newBinaryStreamFromZMQ(const std::string& path,
const std::string& syncPath = "",
ZMQParams::SourceType stype = ZMQParams::SubscriberSource);
/**
......
......@@ -86,8 +86,7 @@ TEST_CASE("Controlling stream processing by a barrier", "[Barrier]") {
// set counter to 10 and send tuples 1, 2, 3, 4, 11, 12:
// => only tuples 1, 2, 3, 4 should arrive
mockup->start();
std::this_thread::sleep_for(std::chrono::milliseconds(100));
mockup->wait();
REQUIRE(mockup->numTuplesProcessed() == 4);
// now set counter to 13:
......@@ -95,7 +94,7 @@ TEST_CASE("Controlling stream processing by a barrier", "[Barrier]") {
mockup->addExpected({makeTuplePtr(11), makeTuplePtr(12)});
counter.set(13);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
mockup->wait();
REQUIRE(mockup->numTuplesProcessed() == 6);
// set counter to 25:
......@@ -103,6 +102,6 @@ TEST_CASE("Controlling stream processing by a barrier", "[Barrier]") {
mockup->addExpected({makeTuplePtr(20), makeTuplePtr(21), makeTuplePtr(22)});
counter.set(25);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
mockup->wait();
REQUIRE(mockup->numTuplesProcessed() == 9);
}
......@@ -55,8 +55,8 @@ TEST_CASE("Stream from matrix", "[FromMatrixTest]")
for(auto &tuple : inputs) {
matrix->insert(tuple);
}
using namespace std::chrono_literals;
std::this_thread::sleep_for(2s);
mockup->wait(2000ms);
REQUIRE(mockup->numTuplesProcessed() == size);
}
......@@ -66,8 +66,7 @@ TEST_CASE("Producing a data stream from inserts into a table", "[FromTable]") {
testTable->insert(i, *tp);
}
std::this_thread::sleep_for(std::chrono::milliseconds(100));
mockup->wait();
REQUIRE(mockup->numTuplesProcessed() == 10);
testTable->drop();
}
......@@ -97,9 +97,7 @@ TEST_CASE("Partitioning a data stream and merging the results.",
CREATE_DATA_LINK(merge, mockup);
mockup->start();
using namespace std::chrono_literals;
std::this_thread::sleep_for(std::chrono::milliseconds(100));
mockup->wait();
REQUIRE(mockup->numTuplesProcessed() == numTuples / 2);
}
......@@ -56,8 +56,7 @@ TEST_CASE("Decoupling producer and consumer via a queue", "[Queue]") {
CREATE_DATA_LINK(ch, mockup)
mockup->start();
std::this_thread::sleep_for(std::chrono::milliseconds(100));
mockup->wait();
REQUIRE(mockup->numTuplesProcessed() == expected.size());
}
......@@ -17,65 +17,47 @@
* along with PipeFabric. If not, see <http://www.gnu.org/licenses/>.
*/
#include <boost/filesystem.hpp>
#include <iostream>
#include <future>
// using namespace boost::filesystem;
#include "catch.hpp"
#include "fmt/format.h"
#include "SimpleWeb/client_http.hpp"
#include "core/Tuple.hpp"
#include "qop/RESTSource.hpp"
#include "qop/DataSink.hpp"
#include "qop/OperatorMacros.hpp"
#include "StreamMockup.hpp"
using namespace pfabric;
using namespace ns_types;
class TestConsumer : public SynchronizedDataSink<TStringPtr> {
public:
PFABRIC_SYNC_SINK_TYPEDEFS(TStringPtr)
TestConsumer() : tupleNum(0) {}
BIND_INPUT_CHANNEL_DEFAULT(InputDataChannel, TestConsumer, processDataElement);
BIND_INPUT_CHANNEL_DEFAULT(InputPunctuationChannel, TestConsumer, processPunctuation);
void processPunctuation(const PunctuationPtr& punctuation) {}
void processDataElement( const TStringPtr& data, const bool outdated ) {
std::string input (data->getAttribute<0>().begin_, data->getAttribute<0>().size_);
std::string expected = fmt::format("(\"key\": \"{0}\",\"value\": \"Always the same\")", tupleNum);
using HttpClient = SimpleWeb::Client<SimpleWeb::HTTP>;
REQUIRE(input == expected);
tupleNum++;
TEST_CASE("Receiving data via REST", "[RESTSource]" ) {
constexpr auto numTuples = 1000;
std::vector<TStringPtr> expected;
std::vector<std::string> stringVec;
for (int i = 0; i < numTuples; i++) {
auto param_string = fmt::format("(\"key\": \"{0}\",\"value\": \"Always the same\")", i);
stringVec.push_back(std::move(param_string));
expected.push_back(makeTuplePtr(StringRef(stringVec[i].c_str(), stringVec[i].size())));
}
private:
int tupleNum;
};
typedef SimpleWeb::Client<SimpleWeb::HTTP> HttpClient;
TEST_CASE("Receiving data via REST", "[RESTSource]" ) {
auto restSource = std::make_shared<RESTSource>(8099, "^/publish$", RESTSource::POST_METHOD);
auto consumer = std::make_shared<TestConsumer>();
CREATE_LINK(restSource, consumer);
auto mockup = std::make_shared<StreamMockup<TStringPtr, TStringPtr> >(expected, expected);
CREATE_LINK(restSource, mockup);
// note we have to start the REST server asynchronously
auto handle = std::async(std::launch::async, [&](std::shared_ptr<RESTSource> src){ src->start(); }, restSource);
/// NOTE: we have to start the REST server asynchronously
auto handle = std::async(std::launch::async, [&restSource, &stringVec](){
restSource->start();
});
std::this_thread::sleep_for(std::chrono::milliseconds(100));
HttpClient client("localhost:8099");
for (int i = 0; i < 100; i++) {
std::string param_string = fmt::format("(\"key\": \"{0}\",\"value\": \"Always the same\")", i);
auto res = client.request("POST", "/publish", param_string);
for (int i = 0; i < numTuples; i++) {
auto res = client.request("POST", "/publish", stringVec[i]);
}
restSource->stop();
handle.get();
REQUIRE(mockup->numTuplesProcessed() == numTuples);
}
......@@ -25,9 +25,11 @@
#include <boost/algorithm/string/find_iterator.hpp>
#include <boost/algorithm/string.hpp>
#include <vector>
#include <mutex>
#include <chrono>
#include <condition_variable>
#include <fstream>
#include <mutex>
#include <vector>
#include "core/Tuple.hpp"
#include "qop/Map.hpp"
......@@ -36,6 +38,8 @@
#include "qop/OperatorMacros.hpp"
using namespace std::chrono_literals;
namespace pfabric {
struct MockupHelper {
......@@ -109,11 +113,20 @@ public:
void start() {
const bool outdated = false;
auto i = 1;
for (auto tp : inputTuples) {
this->getOutputDataChannel().publish( tp, outdated );
}
}
void wait(const std::chrono::milliseconds dur = 1000ms) {
std::unique_lock<std::mutex> lk(mCv_m);
const auto now = std::chrono::system_clock::now();
if (mCv.wait_until(lk, now + dur) == std::cv_status::timeout) {
std::cerr << "Mockup - waiting timed out.\n";
}
}
int numTuplesProcessed() const {
int res = 0;
{
......@@ -142,7 +155,7 @@ private:
void processDataElement( const OutputStreamElement& data, const bool outdated ) {
std::lock_guard<std::mutex> guard(mMtx);
// std::cout << "StreamMockup::processDataElement: " << data << std::endl;
//std::cout << "StreamMockup::processDataElement: " << data << std::endl;
REQUIRE(unsigned(tuplesProcessed) < expectedTuples.size());
if (compareOrdered) {
// If we can compare tuples in their order of arrival everything is easy:
......@@ -154,8 +167,7 @@ private:
REQUIRE(data->isNull(i) == expectedTuples[tuplesProcessed]->isNull(i));
}
tuplesProcessed++;
}
else {
} else {
// Otherwise, more more is needed: first, we store the incoming tuple.
processedTuples.push_back(data);
......@@ -173,7 +185,9 @@ private:
expectedTuples.begin(), expectedTuples.end(), std::back_inserter(res), compareFunc);
REQUIRE(res.empty());
}
}
if (tuplesProcessed == expectedTuples.size()) {
mCv.notify_all();
}
}
......@@ -187,6 +201,8 @@ private:
bool compareOrdered;
CompareFunc compareFunc;
mutable std::mutex mMtx;
std::condition_variable mCv;
std::mutex mCv_m;
};
}
......
......@@ -92,9 +92,8 @@ TEST_CASE("Building and running a topology with partitioned aggregation",
tuplesProcessed++;
});
t.start(false);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
t.start();
t.wait();
REQUIRE(results.size() == num);
}
......
......@@ -130,13 +130,11 @@ void TopologyPartitionedWhereBeforeMapTest(benchmark::State& state) {
})
.merge();
t.start();
t.wait();
t.start(false);
//BAD: Takes far too long because of iteration number
//wait for results - stop timer
//state.PauseTiming();
//std::this_thread::sleep_for(std::chrono::milliseconds(100));
//state.ResumeTiming();
}
}
......@@ -217,7 +215,6 @@ void TopologyPartitionedGroupByTest(benchmark::State& state) {
//BAD: Takes far too long because of iteration number
//wait for results - stop timer
//state.PauseTiming();
//std::this_thread::sleep_for(std::chrono::milliseconds(100));
//state.ResumeTiming();
}
}
......@@ -253,7 +250,6 @@ void TopologyPartitionedJoinTest(benchmark::State& state) {
//BAD: Takes far too long because of iteration number
//wait for results - stop timer
//state.PauseTiming();
//std::this_thread::sleep_for(std::chrono::milliseconds(100));
//state.ResumeTiming();
}
}
......
......@@ -102,8 +102,6 @@ TEST_CASE("Building and running a topology with simple unpartitioned grouping",
t.start(false);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
REQUIRE(results.size() == num);
for (auto i=0u; i<num; i++) {
......@@ -159,8 +157,6 @@ TEST_CASE("Building and running a topology with unpartitioned grouping",
t.start(false);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
REQUIRE(results.size() == num);
for (auto i=0u; i<num; i++) {
......@@ -219,16 +215,14 @@ TEST_CASE("Building and running a topology with partitioned grouping",
std::vector<double> tmpVec;
tmpVec.push_back(get<0>(tp));
tmpVec.push_back(get<1>(tp));
results.push_back(tmpVec);
}
tuplesProcessed++;
});
//.print(std::cout);
t.start(false);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
t.start();
t.wait();
REQUIRE(results.size() == num);
......
......@@ -57,7 +57,6 @@ TEST_CASE("Building and running a topology with ScaleJoin (3 instances)", "[Scal
t.prepare();
t.start(false);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
REQUIRE(results == num);
......@@ -165,9 +164,8 @@ TEST_CASE("Building and running a topology with a join on one partitioned stream
results.push_back(tmp_vec);
});
t.start(false);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
t.start();
t.wait();
REQUIRE(results.size() == num*10);
......@@ -207,9 +205,8 @@ TEST_CASE("Building and running a topology with a join on another partitioned st
results.push_back(tmp_vec);
});
t.start(false);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
t.start();
t.wait();
REQUIRE(results.size() == num*10);
......@@ -248,9 +245,8 @@ TEST_CASE("Building and running a topology with a join on another partitioned st
results.push_back(tmp_vec);
});
t.start(false);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
t.start();
t.wait();
REQUIRE(results.size() == num*10);
......@@ -291,9 +287,8 @@ TEST_CASE("Building and running a topology with a join on two partitioned stream
results.push_back(tmp_vec);
});
t.start(false);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
t.start();
t.wait();
REQUIRE(results.size() == num*10);
......
......@@ -63,45 +63,48 @@ TEST_CASE("Building and running a simple topology", "[Topology]") {
t.start();
t.wait();
REQUIRE(strm.str() == expected);
}
TEST_CASE("Building and running a topology with ZMQ", "[Topology]") {
typedef TuplePtr<int, int> T1;
zmq::context_t context (1);
zmq::socket_t publisher (context, ZMQ_PUB);
publisher.bind("tcp://*:5678");
std::stringstream strm;
Topology t;
auto s = t.newAsciiStreamFromZMQ("tcp://localhost:5678")
.extract<T1>(',')
.print(strm);
zmq::context_t context(1);
zmq::socket_t publisher(context, ZMQ_PUB);
publisher.bind("tcp://*:7890");
zmq::socket_t syncservice(context, ZMQ_REP);
syncservice.bind("tcp://*:7891");
t.start(false);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
auto handle = std::async(std::launch::async, [&publisher](){
auto handle = std::async(std::launch::async, [&publisher, &syncservice](){
std::vector<std::string> input = {
"0,10", "1,11", "2,12", "3,13", "4,14", "5,15"
};
zmq::message_t message;
syncservice.recv(message, zmq::recv_flags::none);
for(const std::string &s : input) {
zmq::message_t request (4);
memcpy (request.data (), s.c_str(), 4);
zmq::message_t request(4);
memcpy(request.data(), s.c_str(), 4);
publisher.send(request, zmq::send_flags::none);
}
});
handle.get();
std::this_thread::sleep_for(std::chrono::milliseconds(100));
Topology t;
std::stringstream strm;
auto s = t.newAsciiStreamFromZMQ("tcp://localhost:7890", "tcp://localhost:7891")
.extract<T1>(',')
.print(strm);
std::string expected = "0,10\n1,11\n2,12\n3,13\n4,14\n5,15\n";
t.start();
handle.wait();
t.wait();
std::string expected = "0,10\n1,11\n2,12\n3,13\n4,14\n5,15\n";
REQUIRE(strm.str() == expected);
syncservice.close();
publisher.close();
}
TEST_CASE("Building and running a topology with ToTable", "[Topology]") {
......@@ -155,8 +158,7 @@ TEST_CASE("Building and running a topology with partitioning", "[Topology]") {
});
t.start(false);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
t.wait();
REQUIRE(results.size() == 500);
......@@ -221,9 +223,8 @@ TEST_CASE("Building and running a topology with batcher", "[Topology]") {
})
;
t2.start(false);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
t2.start();