}, function(err){
synonyms = _.map(_.unique(synonyms), function(num){ return [num, wordCount.indexOf(num)] });
antonyms = _.map(_.unique(antonyms), function(num){ return [num, wordCount.indexOf(num)] });
synonyms = _.sortBy(synonyms, function(num){ return num[1] }).reverse()
antonyms = _.sortBy(antonyms, function(num){ return num[1] }).reverse()
wordnet.close()
synonyms = _.filter(synonyms, function(num){ return (num[0].indexOf("_")==-1) })
antonyms = _.filter(antonyms, function(num){ return (num[0].indexOf("_")==-1) })
// eliminate the numbers
synonyms = _.unique(_.flatten(_.map(synonyms, function(num){ return num[0] })))
antonyms = _.unique(_.flatten(_.map(antonyms, function(num){ return num[0] })))
console.vlog("DEBUGWORDNET: synonyms: "+JSON.stringify(synonyms, null, 4))
console.vlog("DEBUGWORDNET: antonyms: "+JSON.stringify(antonyms, null, 4))
callbackg(null, {
'synonyms':_.filter(synonyms, function(num){ return (num[0].indexOf("_")==-1) }),
'antonyms':_.filter(antonyms, function(num){ return (num[0].indexOf("_")==-1) })
})
})
run: function (root, path, settings, doc, callback) {
var module_cache = {};
for (var k in doc._duality_core_load) {
if (doc._duality_core_load[k] && doc._duality_core_load[k].load) {
// check that this package wants its exports wrapped by duality
if ('duality' in doc._duality_core_load[k].dependencies || {}) {
try {
load(module_cache, doc, doc._duality_core_load[k]);
}
catch (e) {
return callback(e);
}
}
}
}
// prepend required duality rewrites and flatten
doc.rewrites = _.flatten([
{from: '/modules.js', to: 'modules.js'},
{from: '/duality.js', to: 'duality.js'},
{from: '/_db/*', to: '../../*'},
{from: '/_db', to: '../..'}
].concat(doc.rewrites || []));
delete doc._duality_core_load;
callback(null, doc);
}
module.exports = function (config, callback) {
if (!intf) {
return callback(false);
}
function addIp(ip, callback) {
con.info('Adding local IP ' + ip + ' for forwards; if asked for password, give your local (sudo) password.');
var ifconfig = pspawn('sudo', ['ifconfig', intf, 'add', ip]);
ifconfig.on('exit', callback);
}
function addMissingIps(exitCode) {
if (exitCode === 0 && missing.length > 0) {
addIp(missing.shift(), addMissingIps);
} else {
callback(exitCode === 0);
}
}
var allForwards = _.flatten(_.values(config.forwards).concat(_.values(config.localForwards)));
var ips = [];
allForwards.forEach(function (f) {
var m = f.from.match(/^([0-9.]+):/);
ips.push(m[1]);
});
ips = _.uniq(ips);
var currentIps = _.pluck(os.networkInterfaces()[intf], 'address');
var missing = _.difference(ips, currentIps);
// Add any missing IP:s and finally call the callback.
addMissingIps(0);
};
async.mapSeries(fetched, cleanredis, function(err, bestli)
{
bestli = cleanposfromredis(_.flatten(bestli), withscores)
fetched = fetched.concat(bestli)
// fetched = _.unique(_.flatten(fetched))
next()
})
//Returns an array of interpretations. Each interpretation is a corresponding array of components.
function processComponents(components, colIdx) {
var component, langNodeInterps;
if(components.length === 0 || colIdx <= 0){//colIdx?
return [[]];
}
component = components.slice(-1)[0];
if('terminal' in component) {
return _.map(processComponents(components.slice(0, -1), colIdx - component.terminal.length), function(interpretation){
return interpretation.concat(component);
});
} else if('regex' in component) {
if(component.match){
return _.map(processComponents(components.slice(0, -1), colIdx - component.match.length), function(interpretation){
return interpretation.concat(component);
});
}
} else if('category' in component) {
langNodeInterps = _.filter(chart[colIdx], function(langNode) {
return (langNode.category === component.category) && (langNode.parseData.atComponent >= langNode.components.length);
});
if(langNodeInterps.length === 0 ) return [[]];
return _.flatten(_.map(langNodeInterps, function(langNodeInterp) {
//Might be causing stack overflow.
var returnInterp = langNodeInterp;//_.extend({}, langNodeInterp);//TODO: Probably not necessairy.
returnInterp.interpretations = processComponents(returnInterp.components, colIdx);
return _.map(processComponents(components.slice(0, -1), langNodeInterp.parseData.origin), function(interpretation){
//TODO: remove parseData here?
return interpretation.concat(returnInterp);
});
}), true);
} else {
throw "Unknown component type:\n" + JSON.stringify(component);
}
}
_.each(me.knows, function(person) {
// Chercher dans la base de donnée les messages de "person"
var messages = _.flatten(listMessages[person]);
_.each(messages, function(mess) {
personalMessage.push(mess);
});
});
function (callbackwhilst) {
// var len = 5
index += (index < 20 ? 2 : 5)
var mytrain = train.slice(0, index)
var mytrainex = (bars.isDialogue(mytrain) ? _.flatten(mytrain) : mytrain)
var mytestex = (bars.isDialogue(test) ? _.flatten(test) : test)
console.log(msg("DEBUGWORKER "+process["pid"]+": index=" + index +
" train_dialogue="+mytrain.length+" train_turns="+mytrainex.length+
" test_dialogue="+test.length +" test_turns="+mytestex.length+
" classifier="+classifier+ " fold="+fold))
stats = trainAndTest_hash(classifiers[classifier], mytrainex, mytestex, false)
var uniqueid = new Date().getTime()
console.log(msg("DEBUGWORKER "+process["pid"]+": traintime="+
stats['traintime']/1000 + " testtime="+
stats['testtime']/1000 + " classifier="+classifier +
" Accuracy="+stats['stats']['Accuracy']+ " fold="+fold))
var stats1 = {}
_.each(stats['stats'], function(value, key, list){
if ((key.indexOf("macro") != -1) || (key.indexOf("Accuracy") != -1 ) || (key.indexOf("micro") != -1))
stats1[key] = value
}, this)
var results = {
'classifier': classifier,
'fold': fold,
'trainsize': mytrain.length,
'stats': stats1,
'uniqueid': stats['id']
}
console.log(msg("DEBUG:"+JSON.stringify(results, null, 4)))
process.send(JSON.stringify(results))
callbackwhilst()
},
async.mapSeries([seed], quickfetch, function(err, bestlist1) {
// async.mapSeries(_.flatten(bestlist1), quickfetch, function(err, bestlist2) {
// async.mapSeries(_.flatten(bestlist2), quickfetch, function(err, bestlist3) {
// fetched = fetched.concat(bestlist1).concat(bestlist2).concat(bestlist3)
// fetched = fetched.concat(bestlist1).concat(bestlist2)
fetched = fetched.concat(bestlist1)
callback(null, _.unique(_.flatten(fetched)))
// })
// })
})
function output(results) {
var str;
if (program.count && cmd != 'lookup') {
var label = program.brief ? '' : _.flatten(['#', _.values(POS), 'Parsed\n']).join(' ');
str = (cmd == 'get' && (label + _.reduce(POS, function(memo, v){
return memo + ((results[v] && results[v].length) || 0) +" ";
},''))) + nWords;
} else {
str = sprint(results);
}
console.log(str);
}
RequestHandler.prototype.getTranslations = function(req, res) {
var params = {
select: req.param('select'),
query: req.param('q'),
onlyEmpty: req.param('onlyEmpty'),
namespace: req.param('namespace')
},
query = TranslationModel.find();
if(params.select) {
query.select(_.flatten([params.select]).join(' '));
}
if(params.query) {
query.or([
{'key': new RegExp(params.query, 'i')},
{'translations.value': new RegExp(params.query, 'i')}
]);
}
if(params.onlyEmpty == 'true') {
query.where('translations').size(0);
}
if(params.namespace) {
query.in('namespace', _.flatten([params.namespace]));
}
query.lean();
query.exec(function(err, masterTranslations) {
var result;
if(!err && masterTranslations) {
result = [];
_.each(masterTranslations, function(masterTranslation) {
result.push(masterTranslation);
});
res.type(express.mime.types.json);
res.end(JSON.stringify(result));
} else {
res.send(500, "Sorry. Couldn't retrieve masters. Here's the error: " + JSON.stringify(err));
}
});
};
async.mapSeries(seed, quickfetch, function(err, bestlist1) {
// async.mapSeries(_.flatten(bestlist1), quickfetch, function(err, bestlist2) {
// async.mapSeries(_.flatten(bestlist2), quickfetch, function(err, bestlist3) {
// fetched = fetched.concat(bestlist1).concat(bestlist2).concat(bestlist3)
// fetched = fetched.concat(bestlist1).concat(bestlist2)
// fetched = fetched.concat(bestlist1)
// console.log(fetched)
// process.exit(0)
callback(null, _.unique(_.flatten(fetched)))
// })
// })
})
function generateGraphData(dependencies) {
// Read data
var components = _.uniq(_.flatten(_.map(dependencies, function(values, item) {
var data = [];
data.push(item);
data = data.concat(values);
return data;
})));
// Mapped nodes
var nodes = _.map(components, function(component) {
return {
id: component
}
});
// Figure out links
var links = [];
_.each(dependencies, function(outerDependency, component, index) {
_.each(outerDependency, function(dependency) {
var sourceIndex = _.indexOf(components, dependency);
var targetIndex = _.indexOf(components, component);
var link = {
source: _.indexOf(components, component),
target: _.indexOf(components, dependency),
};
if(sourceIndex > -1 && targetIndex > -1) {
links.push(link);
}
})
});
return {
"directed" : true,
"multigraph" : false,
"graph" : [],
"nodes" : nodes,
"links" : links
}
};
function (callback_while) {
console.log("INDEX "+index)
var mytrain = data['train'].slice(0, index)
if (index<10)
{
index += 1
}
else if (index<20)
{
index += 2
}
else index += 10
var mytrainset = JSON.parse(JSON.stringify((bars.isDialogue(mytrain) ? _.flatten(mytrain) : mytrain)))
// filter train to contain only single label utterances
// var mytrainset = _.filter(mytrainset, function(num){ return num.output.length == 1 })
trainAndTest.trainAndTest_async(classifiers[_.values(classifierList)[0]], bars.copyobj(mytrainset), bars.copyobj(testset), function(err, stats1){
extractGlobal(_.values(classifierList)[0], mytrain, fold, stats1['stats'], glob_stats, classifierList)
console.log(_.values(classifierList)[0])
console.log(JSON.stringify(stats1['stats'], null, 4))
bars.generateoppositeversion2(JSON.parse(JSON.stringify(mytrainset)), function(err, sim_train1){
console.log("DEBUGGEN: size of the strandard train" + mytrainset.length + " and generated "+ sim_train1.length)
trainAndTest.trainAndTest_async(classifiers[_.values(classifierList)[1]], bars.copyobj(sim_train1), bars.copyobj(testset), function(err, stats2){
extractGlobal(_.values(classifierList)[1], mytrain, fold, stats2['stats'], glob_stats, classifierList)
console.log(_.values(classifierList)[1])
console.log(JSON.stringify(stats2['stats'], null, 4))
_.each(glob_stats, function(data, param, list){
master.plotlc(fold, param, glob_stats)
console.log("DEBUGLC: param "+param+" fold "+fold+" build")
master.plotlc('average', param, glob_stats)
})
callback_while();
})
})
})
},
// var gnuplot = __dirname + '/gnuplot'
/* @params classifiers - classifier for learning curves
@params dataset - dataset for evaluation, 20% is takes for evaluation
@params parameters - parameters we are interested in
@params step - step to increase a train set
The example of module.exports = {
learning_curves: learning_curves,
extractGlobal: extractGlobal,
getAverage: getAverage
}the input is following.
/*
{
"F1": {
"2": {module.exports = {
learning_curves: learning_curves,
extractGlobal: extractGlobal,
getAverage: getAverage
}
"PPDB": [ 0.6666666666666666 ],
"Original": [ 0.16666666666666666 ]
}
},
"Precision": {
"2": {
"PPDB": [ 0.7777777777777778 ],
"Original": [ 1 ]
}
},
"Recall": {
"2": {
"PPDB": [ 0.5833333333333334 ],
"Original": [ 0.09090909090909091 ]
}
},
"FN": {
"2": {
"PPDB": [ 5 ],
"Original": [ 10 ]
}
}
}
param
F1
trainsize
2
[ 0.6666666666666666, 0.16666666666666666 ]
classifiers
[ [ 'PPDB', [] ], [ 'Original', [] ] ]
*/
// function extractGlobal(parameters, classifiers, trainset, report, stat)
function extractGlobal(classifier, mytrain, fold, stats, glob_stats, classifierList)
{
var stats_prep = {}
_.each(stats, function(value, key, list){
if ((key.indexOf("Precision") != -1) || (key.indexOf("Recall") != -1 ) || (key.indexOf("F1") != -1) || (key.indexOf("Accuracy") != -1))
stats_prep[key] = value
}, this)
var results = {
'classifiers': classifierList,
'classifier': classifier,
'fold': fold,
'trainsize': mytrain.length,
'trainsizeuttr': _.flatten(mytrain).length,
'stats': stats_prep
}
master.extractGlobal(results, glob_stats)
}
/**
* @param a bag of all possible labels for a given sample after classification and observation is a tree of label hierarchy
* @return a set of consistent labels from the observation that could be built by greedy constructing for the bag of labels.
*/
function greedyLabelJoin(values, observation)
{
values = _.flatten(values)
possib = []
for (intent in observation)
{
for (attr in observation[intent])
{
if (Object.keys(observation[intent][attr]).length==0)
if ((values.indexOf(intent)!=-1) && (values.indexOf(attr)!=-1))
possib.push(joinJson([intent,attr]))
for (value in observation[intent][attr])
{
if ((values.indexOf(intent)!=-1) && (values.indexOf(attr)!=-1) && (values.indexOf(value)!=-1))
{
possib.push(joinJson([intent,attr,value]))
}
}
}
}
return [possib]
}
piper.wrapper = function(self, task /*, result...*/){
var readCallbacks = self.callbackQueue,
result = _.rest(arguments, 2),
callback, args;
// live access callbacks cache in case nested cb's
// add to the array.
while (args = readCallbacks[task].shift()) {
callback = args.pop(); // last arg MUST be callback
// console.log('>>>> pper wrapper', self.fastIndex.name, task, result.toString())
callback.apply(null, [].concat(_.flatten(args, /*shallow*/true), result));
}
// now done - delete cb cache
delete readCallbacks[task];
if (--self.refcount === 0) {
//console.log(' ... closing', self.filePath);
fs.close(self.fd);
self.fd = null;
}
};
}, function (err, termResults) {
terms = _.flatten(termResults);
var outKey = self._fmt('search', terms.join('_'));
var args = [outKey, terms.length];
for (var i = 0; i < terms.length; i++) {
args.push(self._fmt(WEIGHT, terms[i]));
}
args.push('AGGREGATE');
args.push('SUM');
var selectResults = function (primaryClient) {
primaryClient.zcard(outKey, function (err, cardResult) {
if (err) {
callback(err);
} else {
primaryClient.zrevrange(outKey, 0, options.searchLimit, 'WITHSCORES', function (err, results) {
if (err) {
callback(err);
return;
}
var scores = {};
var ids = [];
while (results.length) {
var docId = results.shift();
var docTermWeight = parseFloat(results.shift(), 10);
if (!scores.hasOwnProperty(docId)) {
scores[docId] = 0;
ids.push(docId);
}
scores[docId] += docTermWeight;
}
ids = ids.sort(function (a, b) { return scores[b] - scores[a] });
callback(null, [ids,scores,cardResult]);
});
}
});
};
var primaryClient = self._client(args[2]);
primaryClient.exists(outKey, function (err, existsResult) {
if (err) {
callback(err);
} else if (existsResult == 1) {
selectResults(primaryClient);
} else {
// handle clustered zunions through the use of copy and migration
if (self._redisCluster) {
var clients = [];
for (var i = 2; i < args.length - 2; i++) {
var c = self._client(args[i]);
clients.push([c, args[i], i]);
}
// determine the best migration path
async.map(clients, function (item, cb) {
item[0].zcard(item[1], function (err, reply) {
if (reply == 0 || err) {
cb(err, null);
} else {
cb(err, [item[1], reply]);
}
});
}, function (err, results) {
if (err) {
callback(err);
} else {
results = results.filter(function (a) {
if (a == null) {
return false;
} else {
return true;
}
});
if (results.length == 0) {
callback(false);
return;
}
var delKeys = [];
async.every(results, function (item, cb) {
var arg = item[0];
var c = self._client(arg);
if (c != primaryClient) {
// perform the dump and restore
c.dump(arg, function (err, reply) {
if (err) {
cb(false);
}
else if (reply) {
primaryClient.restore(arg, 0, reply, function (err, result) {
if (err) {
cb(false);
} else {
delKeys.push(arg);
cb(true);
}
});
} else {
cb(true);
}
});
} else {
cb(true);
}
}, function (success) {
if (success) {
primaryClient.zunionstore(args, function (err, results) {
if (err) {
callback(err);
} else {
for (var i = 0; i < delKeys.length; i++) {
primaryClient.del(delKeys[i]);
}
selectResults(primaryClient);
}
});
} else {
callback(success);
}
});
}
});
}
else {
var client = self._client(args[0]);
client.zunionstore(args, function (err, result) {
if (err) {
callback(err);
} else {
selectResults(client);
}
});
}
}
});
});
get: function (req, res, next) {
var city = req.params.city;
var cityObj = _.findWhere(_.flatten(_.pluck(Province.data, "cityList"), true), {name: city}) || {areaList: null};
res.json(cityObj.areaList);
}
function trainlen(train, index)
{
return _.flatten(JSON.parse(JSON.stringify(train)).slice(0, index))
}
trainAndTest.trainAndTest_async(classifiers[_.values(classifierList)[1]], bars.copyobj(sim_train2), bars.copyobj(testset), function(err, stats2){
console.log("DEBUGSIM:"+results2["simulated"].length+" size of the simulated train")
console.log("DEBUGSIMDIST: Simualte dist for the second run:"+JSON.stringify(bars.getDist(sim_train2)))
console.log("DEBUGSIM: Results for the second run "+ JSON.stringify(stats2['stats'], null, 4))
extractGlobal(_.values(classifierList)[1], mytrain, fold, stats2['stats'], glob_stats, classifierList)
console.log("DEBUGSIM: before results3")
var results3 = bars.simulateds(buffer_train3, mytrainset.length - sim_train3.length, gold, 0.03125)
console.log("DEBUGSIM: after results3")
buffer_train3 = results3["dataset"]
sim_train3 = sim_train3.concat(results3["simulated"])
console.log("DEBUGSIM: size of aggregated simulated after plus "+ sim_train3.length + " in utterances "+_.flatten(sim_train3).length)
trainAndTest.trainAndTest_async(classifiers[_.values(classifierList)[2]], bars.copyobj(sim_train3), bars.copyobj(testset), function(err, stats3){
console.log("DEBUGSIMDIST: Simualte dist for the third run:"+JSON.stringify(bars.getDist(sim_train3)))
extractGlobal(_.values(classifierList)[2], mytrain, fold, stats3['stats'], glob_stats, classifierList)
var results4 = bars.simulateds(buffer_train4, mytrainset.length - sim_train4.length, gold, 0)
buffer_train4 = results4["dataset"]
sim_train4 = sim_train4.concat(results4["simulated"])
trainAndTest.trainAndTest_async(classifiers[_.values(classifierList)[3]], bars.copyobj(sim_train4), bars.copyobj(testset), function(err, stats4){
console.log("DEBUGSIMDIST: Simualte dist 1st:"+JSON.stringify(bars.getDist(sim_train1)))
console.log("DEBUGSIMDIST: Simualte dist 2st:"+JSON.stringify(bars.getDist(sim_train2)))
console.log("DEBUGSIMDIST: Simualte dist 3st:"+JSON.stringify(bars.getDist(sim_train3)))
console.log("DEBUGSIMDIST: Simualte dist 4st:"+JSON.stringify(bars.getDist(sim_train4)))
extractGlobal(_.values(classifierList)[3], mytrain, fold, stats4['stats'], glob_stats, classifierList)
_.each(glob_stats, function(data, param, list){
master.plotlc(fold, param, glob_stats)
console.log("DEBUGLC: param "+param+" fold "+fold+" build")
master.plotlc('average', param, glob_stats)
})
callback_while();
})
})
})
}
loadDeps(s.dependencies);
}
}
}
// load root app
var tmp = {};
tmp[settings.name] = null;
loadDeps(tmp);
// load dependencies of root app
loadDeps(settings.dependencies);
exports._rewrites = _.flatten(exports._rewrites);
exports.handleUrl = function(ev, href, rel) {
exports._in_page = /#[A-Za-z_\-:\.]+/.test(href);
if (exports._in_page) { // in-page anchor
return;
}
if (href && exports.isAppURL(href) && rel !== 'external') {
var url = exports.appPath(href);
ev.preventDefault();
var match = exports.matchURL('GET', url);
if (/^_show\//.test(match.to) ||
/^_list\//.test(match.to) ||
/^_update\//.test(match.to)) {
exports.setURL('GET', url, {});
async.timesSeries(numOfFolds, function(fold, callback_fold){
console.log("DEBUGSIM: FOLD "+fold)
var index = 2
var data = partitions.partitions_consistent_by_fold(dataset, numOfFolds, fold)
var testset = (bars.isDialogue(data['test']) ? _.flatten(data['test']) : data['test'])
console.log("DEBUGSIM: size of the train before resizing "+data['train'].length)
// base the dst on the first 5 dialogues
var gold = data['train'].splice(0,5)
console.log("DEBUGSIM: size of the gold "+gold.length)
console.log("DEBUGSIM: size of the train after resizing "+data['train'].length)
var sim_train1 = []
var sim_train2 = []
var sim_train3 = []
var sim_train4 = []
// I think it's not important to do so
var buffer_train1 = _.flatten(JSON.parse(JSON.stringify(data['train'])))
var buffer_train2 = _.flatten(JSON.parse(JSON.stringify(data['train'])))
var buffer_train3 = _.flatten(JSON.parse(JSON.stringify(data['train'])))
var buffer_train4 = _.flatten(JSON.parse(JSON.stringify(data['train'])))
async.whilst(
function () { return (index <= data['train'].length && index <= 15 ) },
function (callback_while) {
console.log("DEBUGSIM: INDEX "+index)
var mytrain = data['train'].slice(0, index)
if (index<10)
{
index += 2
}
else if (index<20)
{
index += 2
}
else index += 10
var mytrainset = JSON.parse(JSON.stringify((bars.isDialogue(mytrain) ? _.flatten(mytrain) : mytrain)))
console.log("DEBUGSIM: size of the strandard train" + mytrain.length + " in utterances "+ mytrainset.length)
console.log("DEBUGSIM: before results1")
var results1 = bars.simulateds(buffer_train1, mytrainset.length - sim_train1.length, gold, 1)
console.log("DEBUGSIM: after results1")
buffer_train1 = results1["dataset"]
sim_train1 = sim_train1.concat(results1["simulated"])
trainAndTest.trainAndTest_async(classifiers[_.values(classifierList)[0]], bars.copyobj(sim_train1), bars.copyobj(testset), function(err, stats1){
console.log("DEBUGSIM:"+results1["simulated"].length+" size of the simulated train")
console.log("DEBUGSIMDIST: Simualte dist for the first run:"+JSON.stringify(bars.getDist(sim_train1)))
console.log("DEBUGSIM: Results for the first run "+ JSON.stringify(stats1['stats'], null, 4))
extractGlobal(_.values(classifierList)[0], mytrain, fold, stats1['stats'], glob_stats, classifierList)
console.log("DEBUGSIM: before results2")
var results2 = bars.simulateds(buffer_train2, mytrainset.length - sim_train2.length, gold, 0.5)
console.log("DEBUGSIM: after results2")
buffer_train2 = results2["dataset"]
sim_train2 = sim_train2.concat(results2["simulated"])
console.log("DEBUGSIM: size of aggregated simulated after plus "+ sim_train2.length + " in utterances "+_.flatten(sim_train2).length)
trainAndTest.trainAndTest_async(classifiers[_.values(classifierList)[1]], bars.copyobj(sim_train2), bars.copyobj(testset), function(err, stats2){
console.log("DEBUGSIM:"+results2["simulated"].length+" size of the simulated train")
console.log("DEBUGSIMDIST: Simualte dist for the second run:"+JSON.stringify(bars.getDist(sim_train2)))
console.log("DEBUGSIM: Results for the second run "+ JSON.stringify(stats2['stats'], null, 4))
extractGlobal(_.values(classifierList)[1], mytrain, fold, stats2['stats'], glob_stats, classifierList)
console.log("DEBUGSIM: before results3")
var results3 = bars.simulateds(buffer_train3, mytrainset.length - sim_train3.length, gold, 0.03125)
console.log("DEBUGSIM: after results3")
buffer_train3 = results3["dataset"]
sim_train3 = sim_train3.concat(results3["simulated"])
console.log("DEBUGSIM: size of aggregated simulated after plus "+ sim_train3.length + " in utterances "+_.flatten(sim_train3).length)
trainAndTest.trainAndTest_async(classifiers[_.values(classifierList)[2]], bars.copyobj(sim_train3), bars.copyobj(testset), function(err, stats3){
console.log("DEBUGSIMDIST: Simualte dist for the third run:"+JSON.stringify(bars.getDist(sim_train3)))
extractGlobal(_.values(classifierList)[2], mytrain, fold, stats3['stats'], glob_stats, classifierList)
var results4 = bars.simulateds(buffer_train4, mytrainset.length - sim_train4.length, gold, 0)
buffer_train4 = results4["dataset"]
sim_train4 = sim_train4.concat(results4["simulated"])
trainAndTest.trainAndTest_async(classifiers[_.values(classifierList)[3]], bars.copyobj(sim_train4), bars.copyobj(testset), function(err, stats4){
console.log("DEBUGSIMDIST: Simualte dist 1st:"+JSON.stringify(bars.getDist(sim_train1)))
console.log("DEBUGSIMDIST: Simualte dist 2st:"+JSON.stringify(bars.getDist(sim_train2)))
console.log("DEBUGSIMDIST: Simualte dist 3st:"+JSON.stringify(bars.getDist(sim_train3)))
console.log("DEBUGSIMDIST: Simualte dist 4st:"+JSON.stringify(bars.getDist(sim_train4)))
extractGlobal(_.values(classifierList)[3], mytrain, fold, stats4['stats'], glob_stats, classifierList)
_.each(glob_stats, function(data, param, list){
master.plotlc(fold, param, glob_stats)
console.log("DEBUGLC: param "+param+" fold "+fold+" build")
master.plotlc('average', param, glob_stats)
})
callback_while();
})
})
})
})
},
function (err, n) {
callback_fold()
});
}, function() {
template: function(err, m, stylesheets, callback) {
// frames is a container for variables and other less.js
// constructs.
//
// effects is a container for side-effects, which currently
// are limited to FontSets.
//
// deferred_externals is a list of externals, like
// pattern images, etc (by string URL)
// referred to by url() constructs in styles
var that = this,
env = _.extend({
effects: [],
deferred_externals: []
}, that.env),
output = [];
if (err) {
callback(err);
return;
}
var rulesets = _.flatten(stylesheets.map(function(rulesets) {
return rulesets[1].toList(env);
}));
// Iterate through layers and create styles custom-built
// for each of them, and apply those styles to the layers.
m.Layer.forEach(function(l) {
l.styles = [];
// Classes are given as space-separated alphanumeric
// strings.
var classes = (l['class'] || '').split(/\s+/g);
var matching = rulesets.filter(function(definition) {
return definition.appliesTo(l.name, classes);
});
var definitions = that.inheritRules(matching);
for (var i = 0; i < definitions.length; i++) {
var style = new tree.Style(l.name, definitions[i].attachment, definitions[i]);
if (style) {
l.styles.push(style.name);
// env.effects can be modified by this call
output.push(style.toXML(env));
}
}
var nl = new carto.tree.Layer(l);
output.push(nl.toXML());
});
output.unshift(env.effects.map(function(e) {
return e.toXML(env);
}).join('\n'));
Step(
function() {
var group = this.group();
try {
var map_properties = that.getMapProperties(rulesets, env);
} catch(err) {
env.error(err);
callback(env.errors);
return;
}
if (!map_properties.srs) {
map_properties.srs = 'srs="+proj=merc +a=6378137 ' +
'+b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 ' +
'+k=1.0 +units=m +nadgrids=@null +no_defs"';
}
// The only_validate flag can be set to prevent
// any externals from being downloaded - for instance,
// when only validation is needed.
if (env.only_validate !== true &&
env.deferred_externals.length) {
env.deferred_externals.forEach(function(def) {
var next = group();
new External(that.env, def)
.on('complete', function(external) {
next(null, map_properties);
});
});
} else {
var next = group();
next(null, map_properties);
}
},
function(err, map_properties) {
var properties = [];
for (var key in map_properties[0]) {
properties.push(' ' + map_properties[0][key]);
}
output.unshift(
'<?xml version="1.0" '
+ 'encoding="utf-8"?>\n'
+ '<!DOCTYPE Map[]>\n'
+ '<Map' + properties.join('') + '>\n');
output.push('</Map>');
if (env.errors.length) {
callback(env.errors);
} else {
callback(null, output.join('\n'))
}
}
);
},
async.mapSeries(toretrieve, quickfetch, function(err, resultArr) {
callback(err,_.compact(_.unique(_.flatten(resultArr))))
})
var smushed = _.map(articles, function(article){
return _.flatten(article.tags);
});
).then(function(expandedFields) {
self._config.fields = _.flatten(expandedFields);
});
function (callback_while) {
console.log("DEBUGSIM: INDEX "+index)
var mytrain = data['train'].slice(0, index)
if (index<10)
{
index += 2
}
else if (index<20)
{
index += 2
}
else index += 10
var mytrainset = JSON.parse(JSON.stringify((bars.isDialogue(mytrain) ? _.flatten(mytrain) : mytrain)))
console.log("DEBUGSIM: size of the strandard train" + mytrain.length + " in utterances "+ mytrainset.length)
console.log("DEBUGSIM: before results1")
var results1 = bars.simulateds(buffer_train1, mytrainset.length - sim_train1.length, gold, 1)
console.log("DEBUGSIM: after results1")
buffer_train1 = results1["dataset"]
sim_train1 = sim_train1.concat(results1["simulated"])
trainAndTest.trainAndTest_async(classifiers[_.values(classifierList)[0]], bars.copyobj(sim_train1), bars.copyobj(testset), function(err, stats1){
console.log("DEBUGSIM:"+results1["simulated"].length+" size of the simulated train")
console.log("DEBUGSIMDIST: Simualte dist for the first run:"+JSON.stringify(bars.getDist(sim_train1)))
console.log("DEBUGSIM: Results for the first run "+ JSON.stringify(stats1['stats'], null, 4))
extractGlobal(_.values(classifierList)[0], mytrain, fold, stats1['stats'], glob_stats, classifierList)
console.log("DEBUGSIM: before results2")
var results2 = bars.simulateds(buffer_train2, mytrainset.length - sim_train2.length, gold, 0.5)
console.log("DEBUGSIM: after results2")
buffer_train2 = results2["dataset"]
sim_train2 = sim_train2.concat(results2["simulated"])
console.log("DEBUGSIM: size of aggregated simulated after plus "+ sim_train2.length + " in utterances "+_.flatten(sim_train2).length)
trainAndTest.trainAndTest_async(classifiers[_.values(classifierList)[1]], bars.copyobj(sim_train2), bars.copyobj(testset), function(err, stats2){
console.log("DEBUGSIM:"+results2["simulated"].length+" size of the simulated train")
console.log("DEBUGSIMDIST: Simualte dist for the second run:"+JSON.stringify(bars.getDist(sim_train2)))
console.log("DEBUGSIM: Results for the second run "+ JSON.stringify(stats2['stats'], null, 4))
extractGlobal(_.values(classifierList)[1], mytrain, fold, stats2['stats'], glob_stats, classifierList)
console.log("DEBUGSIM: before results3")
var results3 = bars.simulateds(buffer_train3, mytrainset.length - sim_train3.length, gold, 0.03125)
console.log("DEBUGSIM: after results3")
buffer_train3 = results3["dataset"]
sim_train3 = sim_train3.concat(results3["simulated"])
console.log("DEBUGSIM: size of aggregated simulated after plus "+ sim_train3.length + " in utterances "+_.flatten(sim_train3).length)
trainAndTest.trainAndTest_async(classifiers[_.values(classifierList)[2]], bars.copyobj(sim_train3), bars.copyobj(testset), function(err, stats3){
console.log("DEBUGSIMDIST: Simualte dist for the third run:"+JSON.stringify(bars.getDist(sim_train3)))
extractGlobal(_.values(classifierList)[2], mytrain, fold, stats3['stats'], glob_stats, classifierList)
var results4 = bars.simulateds(buffer_train4, mytrainset.length - sim_train4.length, gold, 0)
buffer_train4 = results4["dataset"]
sim_train4 = sim_train4.concat(results4["simulated"])
trainAndTest.trainAndTest_async(classifiers[_.values(classifierList)[3]], bars.copyobj(sim_train4), bars.copyobj(testset), function(err, stats4){
console.log("DEBUGSIMDIST: Simualte dist 1st:"+JSON.stringify(bars.getDist(sim_train1)))
console.log("DEBUGSIMDIST: Simualte dist 2st:"+JSON.stringify(bars.getDist(sim_train2)))
console.log("DEBUGSIMDIST: Simualte dist 3st:"+JSON.stringify(bars.getDist(sim_train3)))
console.log("DEBUGSIMDIST: Simualte dist 4st:"+JSON.stringify(bars.getDist(sim_train4)))
extractGlobal(_.values(classifierList)[3], mytrain, fold, stats4['stats'], glob_stats, classifierList)
_.each(glob_stats, function(data, param, list){
master.plotlc(fold, param, glob_stats)
console.log("DEBUGLC: param "+param+" fold "+fold+" build")
master.plotlc('average', param, glob_stats)
})
callback_while();
})
})
})
})
},
.exec( function( e, results ){
console.log('Waitinglist: ' + String(results[0].length))
fn( _.flatten( results ) );
})
async.mapSeries(seeds, wordnetquickfetch, function(err, result) {
callback(null, _.unique(_.flatten(result)))
})
module.exports = function(files) {
let opts = {}, callback = function(){}
if (arguments.length === 2) {
callback = arguments[1]
} else if (arguments.length >= 3) {
opts = arguments[1]
callback = arguments[2]
}
if (!files || !files.length) {
return callback(new Error('No input files specified.'))
} else {
files = _.flatten(files.map(file => glob.sync(file)));
}
opts = _.extend({}, defaults, opts)
// make sure output directory exists
const outputDir = path.dirname(opts.output)
if (!fs.existsSync(outputDir)) {
require('mkdirp').sync(outputDir)
}
let offsetCursor = 0
const wavArgs = ['-ar', opts.samplerate, '-ac', opts.channels, '-f', 's16le']
const tempFile = mktemp('audiosprite')
opts.logger.debug('Created temporary file', { file: tempFile })
const json = {
resources: []
, spritemap: {}
}
spawn('ffmpeg', ['-version']).on('exit', code => {
if (code) {
callback(new Error('ffmpeg was not found on your path'))
}
if (opts.silence) {
json.spritemap.silence = {
start: 0
, end: opts.silence
, loop: true
}
if (!opts.autoplay) {
json.autoplay = 'silence'
}
appendSilence(opts.silence + opts.gap, tempFile, processFiles)
} else {
processFiles()
}
})
function mktemp(prefix) {
var tmpdir = require('os').tmpdir() || '.';
return path.join(tmpdir, prefix + '.' + Math.random().toString().substr(2));
}
function spawn(name, opt) {
opts.logger.debug('Spawn', { cmd: [name].concat(opt).join(' ') });
return require('child_process').spawn(name, opt);
}
function pad(num, size) {
var str = num.toString();
while (str.length < size) {
str = '0' + str;
}
return str;
}
function makeRawAudioFile(src, cb) {
var dest = mktemp('audiosprite')
opts.logger.debug('Start processing', { file: src })
fs.exists(src, function(exists) {
if (exists) {
let ffmpeg = spawn('ffmpeg', ['-i', path.resolve(src)]
.concat(wavArgs).concat('pipe:'))
ffmpeg.stdout.pipe(fs.createWriteStream(dest, {flags: 'w'}))
ffmpeg.on('close', function(code, signal) {
if (code) {
return cb({
msg: 'File could not be added',
file: src,
retcode: code,
signal: signal
})
}
cb(null, dest)
})
}
else {
cb({ msg: 'File does not exist', file: src })
}
})
}
function appendFile(name, src, dest, cb) {
var size = 0
var reader = fs.createReadStream(src)
var writer = fs.createWriteStream(dest, {
flags: 'a'
})
reader.on('data', function(data) {
size += data.length
})
reader.on('close', function() {
var originalDuration = size / opts.samplerate / opts.channels / 2
opts.logger.info('File added OK', { file: src, duration: originalDuration })
var extraDuration = Math.max(0, opts.minlength - originalDuration)
var duration = originalDuration + extraDuration
json.spritemap[name] = {
start: offsetCursor
, end: offsetCursor + duration
, loop: name === opts.autoplay || opts.loop.indexOf(name) !== -1
}
offsetCursor += originalDuration
var delta = Math.ceil(duration) - duration;
if (opts.ignorerounding)
{
opts.logger.info('Ignoring nearest second silence gap rounding');
extraDuration = 0;
delta = 0;
}
appendSilence(extraDuration + delta + opts.gap, dest, cb)
})
reader.pipe(writer)
}
function appendSilence(duration, dest, cb) {
var buffer = new Buffer(Math.round(opts.samplerate * 2 * opts.channels * duration))
buffer.fill(0)
var writeStream = fs.createWriteStream(dest, { flags: 'a' })
writeStream.end(buffer)
writeStream.on('close', function() {
opts.logger.info('Silence gap added', { duration: duration })
offsetCursor += duration
cb()
})
}
function exportFile(src, dest, ext, opt, store, cb) {
var outfile = dest + '.' + ext;
spawn('ffmpeg',['-y', '-ar', opts.samplerate, '-ac', opts.channels, '-f', 's16le', '-i', src]
.concat(opt).concat(outfile))
.on('exit', function(code, signal) {
if (code) {
return cb({
msg: 'Error exporting file',
format: ext,
retcode: code,
signal: signal
})
}
if (ext === 'aiff') {
exportFileCaf(outfile, dest + '.caf', function(err) {
if (!err && store) {
json.resources.push(dest + '.caf')
}
fs.unlinkSync(outfile)
cb()
})
} else {
opts.logger.info('Exported ' + ext + ' OK', { file: outfile })
if (store) {
json.resources.push(outfile)
}
cb()
}
})
}
function exportFileCaf(src, dest, cb) {
if (process.platform !== 'darwin') {
return cb(true)
}
spawn('afconvert', ['-f', 'caff', '-d', 'ima4', src, dest])
.on('exit', function(code, signal) {
if (code) {
return cb({
msg: 'Error exporting file',
format: 'caf',
retcode: code,
signal: signal
})
}
opts.logger.info('Exported caf OK', { file: dest })
return cb()
})
}
function processFiles() {
var formats = {
aiff: []
, wav: []
, ac3: ['-acodec', 'ac3', '-ab', opts.bitrate + 'k']
, mp3: ['-ar', opts.samplerate, '-f', 'mp3']
, mp4: ['-ab', opts.bitrate + 'k']
, m4a: ['-ab', opts.bitrate + 'k', '-strict', '-2']
, ogg: ['-acodec', 'libvorbis', '-f', 'ogg', '-ab', opts.bitrate + 'k']
, opus: ['-acodec', 'libopus', '-ab', opts.bitrate + 'k']
, webm: ['-acodec', 'libvorbis', '-f', 'webm', '-dash', '1']
};
if (opts.vbr >= 0 && opts.vbr <= 9) {
formats.mp3 = formats.mp3.concat(['-aq', opts.vbr])
}
else {
formats.mp3 = formats.mp3.concat(['-ab', opts.bitrate + 'k'])
}
// change quality of webm output - https://trac.ffmpeg.org/wiki/TheoraVorbisEncodingGuide
if (opts['vbr:vorbis'] >= 0 && opts['vbr:vorbis'] <= 10) {
formats.webm = formats.webm.concat(['-qscale:a', opts['vbr:vorbis']])
}
else {
formats.webm = formats.webm.concat(['-ab', opts.bitrate + 'k'])
}
if (opts.export.length) {
formats = opts.export.split(',').reduce(function(memo, val) {
if (formats[val]) {
memo[val] = formats[val]
}
return memo
}, {})
}
var rawparts = opts.rawparts.length ? opts.rawparts.split(',') : null
var i = 0
opts.logger.info(files);
async.forEachSeries(files, function(file, cb) {
i++
makeRawAudioFile(file, function(err, tmp) {
if (err) {
opts.logger.debug(err);
return cb(err)
}
function tempProcessed() {
fs.unlinkSync(tmp)
cb()
}
var name = path.basename(file).replace(/\.[a-zA-Z0-9]+$/, '')
appendFile(name, tmp, tempFile, function(err) {
if (rawparts != null ? rawparts.length : void 0) {
async.forEachSeries(rawparts, function(ext, cb) {
opts.logger.debug('Start export slice', { name: name, format: ext, i: i })
exportFile(tmp, opts.output + '_' + pad(i, 3), ext, formats[ext]
, false, cb)
}, tempProcessed)
} else {
tempProcessed()
}
})
})
}, function(err) {
if (err) {
return callback(new Error('Error adding file ' + err.message))
}
async.forEachSeries(Object.keys(formats), function(ext, cb) {
opts.logger.debug('Start export', { format: ext })
exportFile(tempFile, opts.output, ext, formats[ext], true, cb)
}, function(err) {
if (err) {
return callback(new Error('Error exporting file'))
}
if (opts.autoplay) {
json.autoplay = opts.autoplay
}
json.resources = json.resources.map(function(e) {
return opts.path ? path.join(opts.path, path.basename(e)) : e
})
var finalJson = {}
switch (opts.format) {
case 'howler':
case 'howler2':
finalJson[opts.format === 'howler' ? 'urls' : 'src'] = [].concat(json.resources)
finalJson.sprite = {}
for (var sn in json.spritemap) {
var spriteInfo = json.spritemap[sn]
finalJson.sprite[sn] = [spriteInfo.start * 1000, (spriteInfo.end - spriteInfo.start) * 1000]
if (spriteInfo.loop) {
finalJson.sprite[sn].push(true)
}
}
break
case 'createjs':
finalJson.src = json.resources[0]
finalJson.data = {audioSprite: []}
for (var sn in json.spritemap) {
var spriteInfo = json.spritemap[sn]
finalJson.data.audioSprite.push({
id: sn,
startTime: spriteInfo.start * 1000,
duration: (spriteInfo.end - spriteInfo.start) * 1000
})
}
break
case 'default':
default:
finalJson = json
break
}
fs.unlinkSync(tempFile)
callback(null, finalJson)
})
})
}
}