jgrewe/dataservice
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

remove conversion queue1

Browse Source
This commit is contained in:
Jan Grewe 2018-10-02 17:55:00 +02:00
parent 639f0fb55b
commit 8be4d6ae6a
10 changed files with 168 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
DataService.pro
View File

@ -48,7 +48,8 @@ HEADERS += \
conversionqueue.h \
chart.h \
chartview.h \
multichartview.h
multichartview.h \
arrayinfo.h
FORMS += \
mainwindow.ui \

83
arrayinfo.h Normal file
View File

@ -0,0 +1,83 @@
#ifndef ARRAYINFO_H
#define ARRAYINFO_H
struct ArrayInfo {
ArrayInfo(const nix::DataArray &array) {
if (!array)
std::cerr << "shiiii\n";
arrayid = array.id();
dataExtent = array.dataExtent();
xDim = guessBestXdim(array);
nix::Dimension d = array.getDimension(xDim);
dimType = d.dimensionType();
xlabel = "";
if (dimType == nix::DimensionType::Sample) {
nix::SampledDimension sd = d.asSampledDimension();
xoffset = sd.offset() ? *sd.offset() : 0.0;
sampleinterval = sd.samplingInterval();
xmax = dataExtent[xDim - 1] * sampleinterval + xoffset;
xlabel = sd.label() ? *sd.label() : "";
xlabel += sd.unit() ? " [" + *sd.unit() + "]" : "";
} else if (dimType == nix::DimensionType::Range) {
nix::RangeDimension rd = d.asRangeDimension();
tics = rd.ticks();
xoffset = tics.front();
xmax = tics.back();
sampleinterval = 0.0;
xlabel = rd.label() ? *rd.label() : "";
xlabel += rd.unit() ? " [" + *rd.unit() + "]" : "";
} else { // SetDimension
xoffset = 0.0;
xmax = dataExtent[xDim - 1];
sampleinterval = 1.0;
}
name = array.name();
ylabel = array.label() ? *array.label() : "";
ylabel += array.unit() ? " [" + *array.unit() + "]" : "";
}
std::string arrayid;
std::string name;
std::string ylabel;
std::string xlabel;
int xDim;
nix::DimensionType dimType;
double xoffset, xmax, sampleinterval;
std::vector<double> tics;
nix::NDSize dataExtent;
int guessBestXdim(const nix::DataArray &array) const {
if(dataExtent.size() == 0) {
throw nix::IncompatibleDimensions("Array has dataExtent().size 0.", "guess_best_xdim");
}
if(dataExtent.size() > 2) {
throw nix::IncompatibleDimensions("Array has more than two dimensions.", "guess_best_xdim");
}
if(dataExtent.size() == 1) {
return 1;
} else {
nix::DimensionType d_1 = array.getDimension(1).dimensionType();
nix::DimensionType d_2 = array.getDimension(2).dimensionType();
if(d_1 == nix::DimensionType::Sample) {
return 1;
} else if (d_2 == nix::DimensionType::Sample) {
return 2;
} else {
if(d_1 == nix::DimensionType::Set && d_2 == nix::DimensionType::Range) {
return 2;
} else if (d_1 == nix::DimensionType::Range && d_2 == nix::DimensionType::Set){
return 1;
} else {
std::cerr << "How did you manage to end up in Lineplotter with 2D Set Data? guess_best_xdims() failure?" << std::endl;
throw nix::IncompatibleDimensions("Array contains 2D set data.", "guess_best_xdim");
}
}
}
}
};
#endif // ARRAYINFO_H

13
dataloader.cpp
View File

@ -8,12 +8,13 @@ DataLoader::DataLoader()
DataResult1D DataLoader::doLoad(DataRequest &request) {
std::vector<double> x, y;
if (request.start + request.count > request.array.dataExtent())
request.count = request.array.dataExtent() - request.start;
x = request.array.getDimension(1).asSampledDimension().axis(request.count[0], request.start[0]);
request.array.getData(y, request.count, request.start);
std::cerr << x[0] << std::endl;
//TODO handle dim types
if (request.offset + request.count > request.array.dataExtent())
request.count = request.array.dataExtent() - request.offset;
x = request.array.getDimension(1).asSampledDimension().axis(request.count[0], request.offset[0]);
std::cerr << request.count << request.offset;
request.array.getData(y, request.count, request.offset);
std::cerr << "dataLoader done" << std::endl;
DataResult1D res(request.id, request.array.id(), x, y);
return res;
}

44
dataqueue.cpp
View File

@ -16,9 +16,24 @@ void DataQueue::run() {
while (!stop) {
mutex.lock();
if (requestQueue.size() > 0) {
DataResult1D data = loader->doLoad(requestQueue.front());
resultMap.insert(std::pair<std::string, DataResult1D>(data.id, data));
emit resultReady(QString::fromStdString(data.id));
DataRequest r = requestQueue.front();
if (r.valid) {
DataResult1D data = loader->doLoad(requestQueue.front());
double min = data.ydata[0];
double max = data.ydata[0];
for (size_t i = 0; i < data.xdata.size(); ++i) {
r.buffer->getBuffer()->push_back({data.xdata[i], data.ydata[i]});
if (data.ydata[i] < min)
min = data.ydata[i];
if (data.ydata[i] > max)
max = data.ydata[i];
}
//resultMap.insert(std::pair<std::string, DataResult1D>(data.id, data));
emit dataReady(QString::fromStdString(data.entityId), min, max);
}
else {
std::cerr << "request is invalid" << std::endl;
}
requestQueue.pop_front();
stop = requestQueue.size() == 0 || stopped;
}
@ -26,6 +41,7 @@ void DataQueue::run() {
}
}
/*
void DataQueue::requestData(const nix::DataArray &array, const nix::NDSize &count, const nix::NDSize &offset, std::string &requestId) {
requestId = nix::util::createId();
DataRequest dr(requestId, array, count, offset);
@ -40,23 +56,23 @@ void DataQueue::requestData(const nix::DataArray &array, const nix::NDSize &coun
mutex.unlock();
start();
}
*/
void DataQueue::requestData(const nix::DataArray &array, DataBuffer *buffer, qreal start, qreal end, std::string &requestId) {
if (end > buffer->rangeMaximum()) { // need to load new data
std::cerr << buffer->rangeMaximum() << " need to load new data! "<< std::endl;
void DataQueue::requestData(const nix::DataArray &array, DataBuffer *buffer, const ArrayInfo &info, qreal min, qreal max, std::string &requestId) {
min = min < info.xoffset ? info.xoffset : min;
if (min < buffer->rangeMinimum()) {
return; // don't do nothing...
}
if (max > buffer->rangeMaximum()) { // need to load new data
if (min >= buffer->rangeMaximum())
min = buffer->rangeMaximum() + info.sampleinterval;
/*DataRequest dr(requestId, array, count, offset);
requestId = nix::util::createId();
DataRequest dr(requestId, array, buffer, info, min, max);
mutex.lock();
std::deque<DataRequest>::iterator it;
for (it = requestQueue.begin(); it < requestQueue.end(); it++) {
std::cerr << it->array.id() << std::endl;
if (it->array.id() == array.id())
std::cerr << "updating request" <<std::endl;
}
requestQueue.push_back(dr);
mutex.unlock();
start();
*/
}
}

6
dataqueue.h
View File

@ -12,6 +12,7 @@
#include "dataloader.h"
#include "databuffer.h"
#include "arrayinfo.h"
#include "dataresult1d.h"
#include "datarequest.h"
@ -28,10 +29,11 @@ public:
signals:
void resultReady(QString requestId);
void dataReady(QString requestId, double min, double max);
public slots:
void requestData(const nix::DataArray &array, const nix::NDSize &count, const nix::NDSize &offset, std::string &requestId);
void requestData(const nix::DataArray &array, DataBuffer *buffer, qreal start, qreal end, std::string &requestId);
//void requestData(const nix::DataArray &array, const nix::NDSize &count, const nix::NDSize &offset, std::string &requestId);
void requestData(const nix::DataArray &array, DataBuffer *buffer, const ArrayInfo &info, qreal min, qreal max, std::string &requestId);
private:

22
datarequest.cpp
View File

@ -1,5 +1,21 @@
#include "datarequest.h"
DataRequest::DataRequest(const std::string &requestId, const nix::DataArray &array, const nix::NDSize &count, const nix::NDSize &start)
: id(requestId), array(array), count(count), start(start)
{}
DataRequest::DataRequest(const std::string &requestId, const nix::DataArray &array, DataBuffer *buf, const ArrayInfo &info, qreal start, qreal end)
: id(requestId), array(array), buffer(buf), info(info)
{
if (info.dataExtent.size() > 1) {
valid = false;
return;
}
if (info.dimType == nix::DimensionType::Sample) {
size_t s = nix::check::fits_in_size_t(start / info.sampleinterval, "Index does not fit in size_t on this system!!!");
size_t e = nix::check::fits_in_size_t((end - start)/ info.sampleinterval, "Index does not fit in size_t on this system!!!");
offset = nix::NDSize(1, s);
count = nix::NDSize(1, e);
if (offset + count > info.dataExtent)
count = info.dataExtent - offset;
}
std::cerr << "create new request " << info.arrayid << " " << offset << " " << count << std::endl;
}

11
datarequest.h
View File

@ -2,15 +2,22 @@
#define DATAREQUEST_H
#include <nix.hpp>
#include <iostream>
#include "databuffer.h"
#include "arrayinfo.h"
class DataRequest
{
public:
DataRequest(const std::string &requestId, const nix::DataArray &array, const nix::NDSize &count, const nix::NDSize &start);
DataRequest(const std::string &requestId, const nix::DataArray &array, DataBuffer *buf, const ArrayInfo &info, qreal start, qreal end);
bool valid;
std::string id;
nix::DataArray array;
nix::NDSize count, start;
DataBuffer *buffer;
ArrayInfo info;
nix::NDSize count, offset;
};
#endif // DATAREQUEST_H

7
mainwindow.cpp
View File

@ -26,14 +26,13 @@ MainWindow::MainWindow(QWidget *parent) :
dataQueue.start();
conversionQueue.start();
/*
file = nix::File::open("/home/grewe/zwischenlager/2017-08-17-af-invivo-1/2017-08-17-af-invivo-1.nix",
nix::FileMode::ReadOnly);
*/
/*
file = nix::File::open("/Users/jan/zwischenlager/threading_test/dataservice/data/2017-08-17-af-invivo-1.nix",
nix::FileMode::ReadOnly);
*/
nix::Block b = file.getBlock(0);
this->voltage = b.getDataArray("V-1");
this->eod = b.getDataArray("EOD");

21
multichartview.cpp
View File

@ -10,9 +10,10 @@ MultiChartView::MultiChartView(DataQueue *queue, ConversionQueue *cqueue, QWidge
ui->setupUi(this);
xRangeMax = 0.0;
xRangeMin = 0.0;
connect(dataQueue, SIGNAL(resultReady(QString)), this, SLOT(dataReady(QString)));
//connect(dataQueue, SIGNAL(resultReady(QString)), this, SLOT(dataReady(QString)));
connect(dataQueue, SIGNAL(dataReady(QString, double, double)), this, SLOT(dataConverted(QString, double, double)));
//connect(&dataQueue, SIGNAL(finished()), &dataQueue, SLOT(deleteLater()));
connect(conversionQueue, SIGNAL(dataConverted(QString,double,double)), this, SLOT(dataConverted(QString,double,double)));
//connect(conversionQueue, SIGNAL(dataConverted(QString,double,double)), this, SLOT(dataConverted(QString,double,double)));
labelFont = QFont("Sans", 10);
tickLabelFont = QFont("Sans", 7);
setContentsMargins(QMargins(0,0,0,0));
@ -42,7 +43,7 @@ void MultiChartView::addArray(const nix::DataArray &array) {
}
this->arrayMap.insert(std::pair<std::string, nix::DataArray>(array.id(), array));
ArrayInfo ai(array);
this->infoMap.insert(std::pair<std::string, ArrayInfo*>(ai.arrayid, &ai));
this->infoMap.insert(std::pair<std::string, ArrayInfo>(ai.arrayid, ai));
DataBuffer *buff = new DataBuffer(array);
this->data_buffer.insert(std::pair<std::string, DataBuffer*>(ai.arrayid, buff));
@ -107,16 +108,13 @@ ChartView* MultiChartView::createChartView(Chart *chart, const ArrayInfo &info)
void MultiChartView::loadData(const ArrayInfo &info) {
if (arrayMap.at(info.arrayid)) {
nix::NDSize count(1, 100000);
nix::NDSize start(1, 0);
std::string vId, eodId;
dataQueue->requestData(arrayMap.at(info.arrayid), count, start, vId);
std::string vId;
dataQueue->requestData(arrayMap.at(info.arrayid), data_buffer[info.arrayid], info, 0.0, 1.0, vId);
}
}
void MultiChartView::newDataRequest(const std::string &entity_id, qreal min, qreal max) {
std::cerr << "request new data from: "<< entity_id << " with max: "<< max << std::endl;
/*
nix::NDSize offset(1, data_buffer[entity_id]->getBuffer()->size());
nix::NDSize count(1, 100000);
@ -125,19 +123,18 @@ void MultiChartView::newDataRequest(const std::string &entity_id, qreal min, qre
*/
// dataQueue->requestData(arrayMap[entity_id], count, offset, request_id);
std::string request_id;
dataQueue->requestData(arrayMap[entity_id], data_buffer[entity_id], min, max, request_id);
dataQueue->requestData(arrayMap[entity_id], data_buffer[entity_id], *infoMap[entity_id], min, max, request_id);
}
void MultiChartView::dataReady(QString requestId){
std::cerr << "MCV::dataReady: " << requestId.toStdString() << std::endl;
// need to make sure, that only one task does the conversion on the same buffer at the same time.
DataResult1D data = dataQueue->getData(requestId);
if (data.id.empty()) {
return;
}
conversionQueue->requestConversion(data_buffer[data.entityId]->getBuffer(), data);
//conversionQueue->requestConversion(data_buffer[data.entityId]->getBuffer(), data);
}
void MultiChartView::dataConverted(QString entityId, double ymin, double ymax) {

83
multichartview.h
View File

@ -11,86 +11,7 @@
#include "conversionqueue.h"
#include "chart.h"
#include "chartview.h"
struct ArrayInfo {
ArrayInfo(const nix::DataArray &array) {
if (!array)
std::cerr << "shiiii\n";
arrayid = array.id();
dataExtent = array.dataExtent();
xDim = guessBestXdim(array);
nix::Dimension d = array.getDimension(xDim);
dimType = d.dimensionType();
xlabel = "";
if (dimType == nix::DimensionType::Sample) {
nix::SampledDimension sd = d.asSampledDimension();
xoffset = sd.offset() ? *sd.offset() : 0.0;
sampleinterval = sd.samplingInterval();
xmax = dataExtent[xDim - 1] * sampleinterval + xoffset;
xlabel = sd.label() ? *sd.label() : "";
xlabel += sd.unit() ? " [" + *sd.unit() + "]" : "";
} else if (dimType == nix::DimensionType::Range) {
nix::RangeDimension rd = d.asRangeDimension();
tics = rd.ticks();
xoffset = tics.front();
xmax = tics.back();
sampleinterval = 0.0;
xlabel = rd.label() ? *rd.label() : "";
xlabel += rd.unit() ? " [" + *rd.unit() + "]" : "";
} else { // SetDimension
xoffset = 0.0;
xmax = dataExtent[xDim - 1];
sampleinterval = 1.0;
}
name = array.name();
ylabel = array.label() ? *array.label() : "";
ylabel += array.unit() ? " [" + *array.unit() + "]" : "";
}
std::string arrayid;
std::string name;
std::string ylabel;
std::string xlabel;
int xDim;
nix::DimensionType dimType;
double xoffset, xmax, sampleinterval;
std::vector<double> tics;
nix::NDSize dataExtent;
int guessBestXdim(const nix::DataArray &array) const {
if(dataExtent.size() == 0) {
throw nix::IncompatibleDimensions("Array has dataExtent().size 0.", "guess_best_xdim");
}
if(dataExtent.size() > 2) {
throw nix::IncompatibleDimensions("Array has more than two dimensions.", "guess_best_xdim");
}
if(dataExtent.size() == 1) {
return 1;
} else {
nix::DimensionType d_1 = array.getDimension(1).dimensionType();
nix::DimensionType d_2 = array.getDimension(2).dimensionType();
if(d_1 == nix::DimensionType::Sample) {
return 1;
} else if (d_2 == nix::DimensionType::Sample) {
return 2;
} else {
if(d_1 == nix::DimensionType::Set && d_2 == nix::DimensionType::Range) {
return 2;
} else if (d_1 == nix::DimensionType::Range && d_2 == nix::DimensionType::Set){
return 1;
} else {
std::cerr << "How did you manage to end up in Lineplotter with 2D Set Data? guess_best_xdims() failure?" << std::endl;
throw nix::IncompatibleDimensions("Array contains 2D set data.", "guess_best_xdim");
}
}
}
}
};
#include "arrayinfo.h"
namespace Ui {
class MultiChartView;
@ -115,7 +36,7 @@ private:
std::map<std::string, nix::DataArray> arrayMap;
std::map<std::string, Chart*> chartMap;
std::map<std::string, ChartView*> viewMap;
std::map<std::string, ArrayInfo*> infoMap;
std::map<std::string, ArrayInfo> infoMap;
std::map<std::string, DataBuffer*> data_buffer;
QFont labelFont, tickLabelFont;