it('should save and load the stream aggregate', function () {
var aggr = {
name: 'featureSpaceWindow',
type: 'timeSeriesWinBufFeatureSpace',
store: 'Docs',
timestamp: 'Time',
featureSpace: {
type: "categorical",
source: "Docs",
field: "Text"
},
winsize: 1000
};
var sa = store.addStreamAggr(aggr);
store.push({ Time: '2015-06-10T14:13:32.0', Text: 'a' }); // 0
store.push({ Time: '2015-06-10T14:13:33.0', Text: 'b' }); // 1
store.push({ Time: '2015-06-10T14:14:34.0', Text: 'c' }); // 2
store.push({ Time: '2015-06-10T14:15:35.0', Text: 'd' }); // 3
store.push({ Time: '2015-06-10T14:15:36.0', Text: 'e' }); // 4
store.push({ Time: '2015-06-10T14:15:37.0', Text: 'f' }); // 5
var fout = qm.fs.openWrite('fsWinBuf.bin');
sa.save(fout).close();
var fin = qm.fs.openRead('fsWinBuf.bin');
sa.load(fin);
var featureSpace = sa.getFeatureSpace();
assert.equal(featureSpace.dim, 6);
});
files.forEach(function (file) {
var source = path.normalize(file);
var dest = source.replace(inFolder, outFolder);
// copy file one by one
if (qm.fs.exists(file)) {
if (!qm.fs.exists(path.dirname(dest))) qm.fs.mkdir(path.dirname(dest));
qm.fs.copy(source, dest);
}
});
describe('Serialization test', function () {
var batchSize = 1000;
var nbatches = 10;
var eps = .01;
var gk = new quants.Gk({
eps: eps
})
var maxRelErr = eps;
var targets = [];
for (var prob = 0; prob <= 1; prob += 0.001) {
targets.push(prob);
}
var vals = [];
for (var i = 0; i < batchSize; i++) {
vals.push(i);
}
for (var batchN = 0; batchN < nbatches; batchN++) {
// shuffle the array
for (var i = 0; i < batchSize; i++) {
var swapN = Math.floor(Math.random()*batchSize);
var temp = vals[i];
vals[i] = vals[swapN];
vals[swapN] = temp;
}
for (var i = 0; i < batchSize; i++) {
gk.insert(vals[i]);
}
var quantiles = gk.quantile(targets);
for (var targetN = 0; targetN < targets.length; targetN++) {
var prob = targets[targetN];
var quant = gk.quantile(prob);
assert.equal(quant, quantiles[targetN]);
assert(Math.floor((prob - maxRelErr)*batchSize) <= quant);
assert(Math.ceil((prob + maxRelErr)*batchSize) >= quant);
}
}
gk.save(qm.fs.openWrite('gk-orig.dat')).close();
var gk1 = new quants.Gk(qm.fs.openRead('gk-orig.dat'));
for (var cumProb = 0; cumProb <= 1; cumProb += 0.001) {
var quant_hat = gk1.quantile(cumProb);
assert(Math.floor((cumProb - maxRelErr)*batchSize) <= quant_hat);
assert(Math.ceil((cumProb + maxRelErr)*batchSize) >= quant_hat);
}
})
it('should save and load vector', function () {
var outRecordVector = new qm.RecordVector(base);
outRecordVector.push(store.newRecord({ Name: "Plato", Era: "Ancient philosophy" }));
outRecordVector.push(store.newRecord({ Name: "Immanuel Kant", Era: "18th-century philosophy" }));
var fout = qm.fs.openWrite('record_vector.bin');
outRecordVector.save(fout).close();
var fin = qm.fs.openRead('record_vector.bin');
var inRecordVector = new qm.RecordVector(base, fin);
assert.equal(inRecordVector.length, 2);
assert.equal(inRecordVector[0].Name, "Plato");
assert.equal(inRecordVector[0].Era, "Ancient philosophy");
assert.equal(inRecordVector[1].Name, "Immanuel Kant");
assert.equal(inRecordVector[1].Era, "18th-century philosophy");
assert.equal(inRecordVector[2], null);
})
it('should save vector', function () {
var recordVector = new qm.RecordVector(base);
recordVector.push(store.newRecord({ Name: "Plato", Era: "Ancient philosophy" }));
recordVector.push(store.newRecord({ Name: "Immanuel Kant", Era: "18th-century philosophy" }));
var fout = qm.fs.openWrite('record_vector.bin');
recordVector.save(fout).close();
})
saveState = function (buffers, dirName) {
// check if dirName exists, if not, create it
if (!qm.fs.exists(dirName)) qm.fs.mkdir(dirName);
// open file in write mode
var fout = new qm.fs.FOut(path.join(dirName, "buffers_model"));
// save each buffer aggregate
for (var property in buffers) {
if (buffers.hasOwnProperty(property)) {
var buffer = buffers[property];
buffer.save(fout);
}
}
fout.flush();
fout.close();
logger.info('Saved buffer model states')
};
saveState = function (avrgs, fields, dirName) {
// check if dirName exists, if not, create it
if (!qm.fs.exists(dirName)) qm.fs.mkdir(dirName);
// open file in write mode
var fout = new qm.fs.FOut(path.join(dirName, "averages_model"));
// iterate over all localizev averages, and save them one by one
for (var avrIdx in avrgs) {
for (var workIdx = 0; workIdx < 2; workIdx++) { // 2 models: working day or not
for (var hourIdx = 0; hourIdx < 24; hourIdx++) {
var avrgModel = avrgs[avrIdx].avrgs[workIdx][hourIdx];
avrgModel.save(fout);
}
}
}
fout.flush();
fout.close();
logger.info('Saved local average model states')
};
saveState = function (linregs, fields, horizons, dirName) {
// check if dirName exists, if not, create it
if (!qm.fs.exists(dirName)) qm.fs.mkdir(dirName);
// open file in write mode
var fout = new qm.fs.FOut(path.join(dirName, "linregs_model"));
// write all states to fout
for (var fieldIdx in fields) {
for (var horizonIdx in horizons) {
for (var workIdx = 0; workIdx < 2; workIdx++) {
for (var hourIdx = 0; hourIdx < 24; hourIdx++) {
var linreg = linregs[fieldIdx][horizonIdx][workIdx][hourIdx];
linreg.save(fout);
fout.writeJson({"updateCount": linreg.updateCount});
}
}
}
}
fout.flush();
fout.close();
logger.info('Saved regression model states')
};
it("should make test number 76", function () {
// import fs module
var fs = require('qminer').fs;
var la = require('qminer').la;
// create a new vector
var vec = new la.StrVector(['a', 'b', 'c']);
// open write stream
var fout = fs.openWrite('vec.dat');
// save vector and close write stream
vec.save(fout).close();
});
it("should make test number 70", function () {
// import fs module
var fs = require('qminer').fs;
var la = require('qminer').la;
// create an empty vector
var vec = new la.Vector();
// open a read stream
var fin = fs.openRead('vec.dat');
// load the matrix
vec.loadascii(fin);
});
it("should make test number 69", function () {
// import fs module
var fs = require('qminer').fs;
var la = require('qminer').la;
// create a new vector
var vec = new la.Vector([1, 2, 3]);
// open write stream
var fout = fs.openWrite('vec.dat');
// save matrix and close write stream
vec.saveascii(fout).close();
});
it("should make test number 45", function () {
// import the modules
var fs = require('qminer').fs;
var la = require('qminer').la;
// create an empty matrix
var mat = new la.SparseMatrix();
// open a read stream ('mat.dat' was previously created)
var fin = fs.openRead('mat.dat');
// load the matrix
mat.load(fin);
});
it("should make test number 44", function () {
// import the modules
var fs = require('qminer').fs;
var la = require('qminer').la;
// create a new sparse matrix
var mat = new la.SparseMatrix([[[0, 1]], [[0, 3], [1, 12]]]);
// open write stream
var fout = fs.openWrite('mat.dat');
// save matrix and close write stream
mat.save(fout).close();
});
it("should make test number 25", function () {
// import the modules
var fs = require('qminer').fs;
var la = require('qminer').la;
// create new matrix
var mat = new la.Matrix([[1, 2], [3, 4]]);
// open write stream
var fout = fs.openWrite('mat.dat');
// save matrix and close write stream
mat.save(fout).close();
});
exports.copyFolder = function (inFolder, outFolder) {
logger.debug("Copying ./" + path.basename(inFolder) + " to ./" + path.basename(outFolder) + " folder...");
// read all files in inFolder
var files = qm.fs.listFile(inFolder, null, true);
// create outFolder if it doesnet exist
if (!qm.fs.exists(outFolder)) qm.fs.mkdir(outFolder);
// copy files from inFloder to outFolder
files.forEach(function (file) {
var source = path.normalize(file);
var dest = source.replace(inFolder, outFolder);
// copy file one by one
if (qm.fs.exists(file)) {
if (!qm.fs.exists(path.dirname(dest))) qm.fs.mkdir(path.dirname(dest));
qm.fs.copy(source, dest);
}
});
logger.debug(files.length + " files copied.\n");
}
it("histogram_diff - complex - reset and reload", function () {
var qm = require('qminer');
// create a base with a simple store
// the store records results of clustering
var base = new qm.Base({
mode: "createClean",
schema: [
{
name: "Rpm",
fields: [
{ name: "ClusterId", type: "float" },
{ name: "Time", type: "datetime" }
]
}]
});
try {
var store = base.store('Rpm');
// create a new time series stream aggregator for the 'Rpm' store that takes the recorded cluster id
// and the timestamp from the 'Time' field. The size of the window is 4 weeks.
var timeser1 = {
name: 'TimeSeries1',
type: 'timeSeriesWinBuf',
store: 'Rpm',
timestamp: 'Time',
value: 'ClusterId',
winsize: 2 * 60 * 60 * 1000 // 2 hours
};
var timeSeries1 = base.store("Rpm").addStreamAggr(timeser1);
// add a histogram aggregator, that is connected with the 'TimeSeries1' aggregator
var aggrJson1 = {
name: 'Histogram1',
type: 'onlineHistogram',
store: 'Rpm',
inAggr: 'TimeSeries1',
lowerBound: 0,
upperBound: 5,
bins: 5,
addNegInf: false,
addPosInf: false
};
var hist1 = base.store("Rpm").addStreamAggr(aggrJson1);
// create a new time series stream aggregator for the 'Rpm' store that takes the recorded cluster id
// and the timestamp from the 'Time' field.
var timeser2 = {
name: 'TimeSeries2',
type: 'timeSeriesWinBuf',
store: 'Rpm',
timestamp: 'Time',
value: 'ClusterId',
winsize: 6 * 60 * 60 * 1000 // 6 hours
};
var timeSeries2 = base.store("Rpm").addStreamAggr(timeser2);
// add a histogram aggregator, that is connected with the 'TimeSeries1' aggregator
var aggrJson2 = {
name: 'Histogram2',
type: 'onlineHistogram',
store: 'Rpm',
inAggr: 'TimeSeries2',
lowerBound: 0,
upperBound: 5,
bins: 5,
addNegInf: false,
addPosInf: false
};
var hist2 = base.store("Rpm").addStreamAggr(aggrJson2);
// add diff aggregator that subtracts Histogram1 with 2h window from Histogram2 with 6h window
var aggrJson3 = {
name: 'DiffAggr',
type: 'onlineVecDiff',
storeX: 'Rpm',
storeY: 'Rpm',
inAggrX: 'Histogram2',
inAggrY: 'Histogram1'
}
var diff = store.addStreamAggr(aggrJson3);
////////////////////////////////////////////////////////////////////
// add some values
store.push({ Time: '2015-06-10T00:13:30.0', ClusterId: 0 });
store.push({ Time: '2015-06-10T00:14:30.0', ClusterId: 0 });
store.push({ Time: '2015-06-10T01:15:31.0', ClusterId: 1 });
store.push({ Time: '2015-06-10T02:16:30.0', ClusterId: 2 });
store.push({ Time: '2015-06-10T03:17:30.0', ClusterId: 0 });
store.push({ Time: '2015-06-10T04:18:30.0', ClusterId: 2 });
var tmp1 = hist1.saveJson();
//console.log(JSON.stringify(tmp1));
assert.equal(tmp1.counts.length, 5);
assert.equal(tmp1.counts[0], 1);
assert.equal(tmp1.counts[1], 0);
assert.equal(tmp1.counts[2], 1);
assert.equal(tmp1.counts[3], 0);
assert.equal(tmp1.counts[4], 0);
// just the last three
var tmp2 = hist2.saveJson();
//console.log(JSON.stringify(tmp2));
assert.equal(tmp2.counts[0], 3);
assert.equal(tmp2.counts[1], 1);
assert.equal(tmp2.counts[2], 2);
assert.equal(tmp2.counts[3], 0);
assert.equal(tmp2.counts[4], 0);
// difference
var tmp3 = diff.saveJson();
//console.log(JSON.stringify(tmp3));
assert.equal(tmp3.diff[0], 2);
assert.equal(tmp3.diff[1], 1);
assert.equal(tmp3.diff[2], 1);
assert.equal(tmp3.diff[3], 0);
assert.equal(tmp3.diff[4], 0);
store.resetStreamAggregates();
// add some values
store.push({ Time: '2015-06-10T00:13:30.0', ClusterId: 0 });
store.push({ Time: '2015-06-10T00:14:30.0', ClusterId: 0 });
store.push({ Time: '2015-06-10T01:15:31.0', ClusterId: 1 });
store.push({ Time: '2015-06-10T02:16:30.0', ClusterId: 2 });
store.push({ Time: '2015-06-10T03:17:30.0', ClusterId: 0 });
store.push({ Time: '2015-06-10T04:18:30.0', ClusterId: 2 });
var tmp1 = hist1.saveJson();
//console.log(JSON.stringify(tmp1));
assert.equal(tmp1.counts.length, 5);
assert.equal(tmp1.counts[0], 1);
assert.equal(tmp1.counts[1], 0);
assert.equal(tmp1.counts[2], 1);
assert.equal(tmp1.counts[3], 0);
assert.equal(tmp1.counts[4], 0);
// just the last three
var tmp2 = hist2.saveJson();
//console.log(JSON.stringify(tmp2));
assert.equal(tmp2.counts[0], 3);
assert.equal(tmp2.counts[1], 1);
assert.equal(tmp2.counts[2], 2);
assert.equal(tmp2.counts[3], 0);
assert.equal(tmp2.counts[4], 0);
// difference
var tmp3 = diff.saveJson();
//console.log(JSON.stringify(tmp3));
assert.equal(tmp3.diff[0], 2);
assert.equal(tmp3.diff[1], 1);
assert.equal(tmp3.diff[2], 1);
assert.equal(tmp3.diff[3], 0);
assert.equal(tmp3.diff[4], 0);
// show distribution for expected values
//console.log(diff);
/////////////////////////////////////////////////////////
var fout = qm.fs.openWrite("aggr.tmp");
hist1.save(fout);
timeSeries1.save(fout);
hist2.save(fout);
timeSeries2.save(fout);
diff.save(fout);
fout.close();
store.resetStreamAggregates();
var fin = qm.fs.openRead("aggr.tmp");
hist1.load(fin);
timeSeries1.load(fin);
hist2.load(fin);
timeSeries2.load(fin);
diff.load(fin);
fin.close();
var tmp1x = hist1.saveJson();
var tmp2x = hist2.saveJson();
var tmp3x = diff.saveJson();
assert.equal(JSON.stringify(tmp1), JSON.stringify(tmp1x));
assert.equal(JSON.stringify(tmp2), JSON.stringify(tmp2x));
assert.equal(JSON.stringify(tmp3), JSON.stringify(tmp3x));
} finally {
base.close();
}
});
describe('Serialization test', function () {
var batchSize = 1000;
var nbatches = 10;
var eps = .1;
var targetProb = 0.01;
var gkLow = new quants.BiasedGk({
eps: eps,
targetProb: targetProb
})
var gkHigh = new quants.BiasedGk({
eps: eps,
targetProb: 1 - targetProb
})
var isErrorInRangeLow = function (prob, quant) {
var p = Math.max(prob, targetProb);
var mxRelError = p*eps;
var mxError = Math.ceil(mxRelError*batchSize);
var error = Math.abs(quant - prob*batchSize);
return error <= mxError + 1e-9;
}
var isErrorInRangeHigh = function (prob, quant) {
var p = Math.max(1 - prob, 1 - targetProb);
var mxRelError = p*eps;
var mxError = Math.ceil(mxRelError*batchSize);
var error = Math.abs(quant - prob*batchSize);
return error <= mxError + 1e-9;
}
var targets = [];
for (var prob = 0; prob <= 1; prob += 0.001) {
targets.push(prob);
}
var vals = [];
for (var i = 0; i < batchSize; i++) {
vals.push(i);
}
for (var batchN = 0; batchN < nbatches; batchN++) {
// shuffle the array
for (var i = 0; i < batchSize; i++) {
var swapN = Math.floor(Math.random()*batchSize);
var temp = vals[i];
vals[i] = vals[swapN];
vals[swapN] = temp;
}
for (var i = 0; i < batchSize; i++) {
gkLow.insert(vals[i]);
gkHigh.insert(vals[i]);
}
var quantsLow = gkLow.quantile(targets);
var quantsHigh = gkHigh.quantile(targets);
for (var targetN = 0; targetN < targets.length; targetN++) {
var prob = targets[targetN];
var quantLow = gkLow.quantile(prob);
var quantHigh = gkHigh.quantile(prob);
assert.equal(quantLow, quantsLow[targetN]);
assert.equal(quantHigh, quantsHigh[targetN]);
assert(isErrorInRangeLow(prob, quantLow));
assert(isErrorInRangeHigh(prob, quantHigh));
}
}
gkLow.save(qm.fs.openWrite('gkLow-orig.dat')).close();
gkHigh.save(qm.fs.openWrite('gkHigh-orig.dat')).close();
var gkLow1 = new quants.BiasedGk(qm.fs.openRead('gkLow-orig.dat'));
var gkHigh1 = new quants.BiasedGk(qm.fs.openRead('gkHigh-orig.dat'));
for (var cumProb = 0; cumProb <= 1; cumProb += 0.001) {
var quantLow = gkLow1.quantile(cumProb);
var quantHigh = gkHigh1.quantile(cumProb);
assert(isErrorInRangeLow(cumProb, quantLow));
assert(isErrorInRangeHigh(cumProb, quantHigh));
}
var params = gkLow1.getParams();
assert(params.eps != null);
assert(params.targetProb != null);
assert(params.compression != null);
assert(params.useBands != null);
assert.equal(params.eps, 0.1);
assert.equal(params.targetProb, 0.01);
assert.equal(params.compression, "periodic");
assert.equal(params.useBands, true);
})