function groupbylabel(dataset, minsize, sizetrain)
{
var sends = _.groupBy(dataset , function(num){ return num['input']['CORENLP']['sentences'].length })
console.log("sentence distribution")
_.each(sends, function(value, key, list){
console.log(key)
console.log(value.length)
console.log("-------------")
}, this)
var gro = _.groupBy(dataset, function(num){ return num["output"][0] })
console.log("label distribution")
_.each(gro, function(value, key, list){
console.log(key)
console.log(value.length)
console.log("-------------")
}, this)
_.each(gro, function(value, key, list){
value = _.filter(value, function(num){ return num['input']['CORENLP']['sentences'].length >= minsize })
value = _.map(value, function(num){ num["input"]["CORENLP"]["sentences"].splice(10)
return num });
if (value.length < sizetrain)
delete gro[key]
else
gro[key] = _.sample(value, sizetrain)
}, this)
return gro
}
getShipData: function(ship) {
var data = this.staticShipData[ship.id],
myUnits,
enemies;
//PATHFINDING GRID
function makeUnitsUnwalkable(ship, grid) {
var units = ship.units;
_.each(units, function(u) {
if (u.x >= 0 && u.x < grid.width &&
u.y >= 0 && u.y < grid.height) {
grid.setWalkableAt(u.x, u.y, false);
}
});
return grid;
}
data.grid = new sh.PF.Grid(ship.width, ship.height,
ship.getPfMatrix());
data.gridWithUnits = makeUnitsUnwalkable(ship, data.grid.clone());
//UNITS BY TYPE
myUnits = ship.getPlayerUnits(this.id);
data.allies = _.groupBy(myUnits, 'type');
data.allies.all = myUnits;
enemies = _.filter(ship.units, function(u) {
return u.ownerID !== this.id;
}, this);
data.enemies = _.groupBy(enemies, 'type');
data.enemies.all = enemies;
data.ship = ship;
return data;
},
function(callback){
outstats = _.sortBy(outstats, function(num){ return num['score'] })
outstats.reverse()
var crossdist = _.map(outstats, function(num){
num['score floored'] = Math.floor(num['score']*10)*10
return num });
crossdist = _.groupBy(crossdist, function(num) {return num['score floored']})
console.log(JSON.stringify(crossdist, null, 4))
for (var col=100; col>=0; col -= 10) {
var ind = 0
var inf = 0
_.each(outstats, function(value, key, list){
if (value['score']*100 >= col)
{
ind = ind + 1
inf = inf + value['keyphrase influence']
}
}, this)
console.log(col/100 + " -- " +(ind/outstats.length*100).toFixed()+"%"+ " -- "+(inf*100).toFixed()+"%")
}
utils.closeredis()
}
],function(err,res){
var _ = require("underscore")._;
if(err){
fn(err,null);
} else {
var result = {
'热门':res[0]
};
_.extend(result,_.groupBy(res[1],function(o){return o.province.name;}));
fn(null,result);
}
})
.on ("end", function (){
// add EOF offset
offsets[lastKey].push(eofKey);
offsets[eofKey] = [endOffset, null];
var size = _.size(buckets),
sum = _.reduce(buckets, function(memo, num){ return memo + num; }, 0),
sorted = _.sortBy(buckets, function(val){ return val }),
median = sorted[Math.floor(size/2)],
max = sorted[sorted.length-1], // _.max(buckets),
maxkey = _.reduce(buckets, function(memo, val, key){ return memo + (val == max ? key : '') }, ''),
avg = (sum/size).toFixed(2),
info = util.format('buckets %d, max %d at %s, sum %d, avg %d, median %d', size, max, maxkey, sum, avg, median);
console.log(basename, info);
if (stats) {
// distribution in groups of 10
var grouped = _.groupBy(buckets, function(num){ return 1 + 10*(Math.floor((num-1)/10) ) });
_(grouped).each(function(arr, key, list){
list[key] = arr.length;
});
str = '';
_.each(grouped, function(value, key){ str += key+"\t"+value+"\n" });
fs.writeFileSync(countFile, '#'+info+'\n'
+ '#bucket_size (1-10, 11-20, etc.) \t #buckets\n'
+ str, 'utf8');
}
// offset data
var data = {
firstKey: firstKey,
keyLength: KEY_LENGTH,
version: WNdb.version,
name: basename,
stats: {
buckets: size,
words: sum,
biggest: max,
avg: avg,
median: median
},
offsets: offsets
};
fs.writeFileSync(jsonFile, JSON.stringify(data), 'utf8');
console.log(' wrote %s\n', jsonFile);
})
.then(function(loadedData) {
data = loadedData;
notify("Length prefilter: "+data.length);
data = processor.startupFilter(data);
data = processor.dataSetup(data);
notify("Length postfilter: "+data.length);
notify("Identifying categories");
var categoriesObj = _.groupBy(data, function(d) {return d[categoryAttribute];});
var categories = _.keys(categoriesObj).sort();
notify("Found "+categories.length+" categories");
processor.recSave(categories,categoriesObj);
saveCarreras(categories,categoriesObj);
notify("End");
});
// only two classifiers in string format
function plot(fold, parameter, stat, baseline, sota)
{
stat = stat[parameter]
var output = []
if (fold != 'average')
{
_.each(stat, function(rowvalue, row, list){
_.each(rowvalue, function(data, column, list){
var result = data[sota][fold] - data[baseline][fold]
if (bars.isNumber(result))
output.push([row, column, result])
}, this)
}, this)
}
else
{
_.each(stat, function(rowvalue, row, list){
_.each(rowvalue, function(data, column, list){
var result = distance.vec_minus(data[baseline], data[sota])
var average = distance.average(result)
output.push([row, column, average])
}, this)
}, this)
}
var results = _.groupBy(output, function(num){ return num[0] })
var string = ""
_.each(results, function(value, key, list){
_.each(value, function(value1, key1, list1){
string += value1.join(" ")+"\n"
}, this)
string += "\n"
}, this)
var mapfile = __dirname+"/learning_curves/map_"+fold+"_"+parameter+"_"+sota+"-"+baseline
fs.writeFileSync(mapfile, string)
var command = gnuplot +" -e \"set title '"+corpus+" : "+sota+" - "+baseline+"'; set output 'utils/learning_curves/"+fold+"_"+parameter+"_"+sota+"-"+baseline+".png'\" "+__dirname+"/com " + "-e \"plot \'"+mapfile+"\' using 2:1:3 with image\""
console.log(command)
execSync.run(command)
}
find.file(/\.js$/, src, function (fileSet) {
// Excludes all files that match any of the exclusion rules,
// and maps each of result to an object containing the
// name of the file, folder and full path
files = fileSet.filter(function (file) {
return excludes.every(function (rule) {
return !(new RegExp(rule).test(file));
});
}).map(function (file) {
var data = PATH_REGEXP.exec(file);
return {
fullPath: data[0],
path: data[1],
name: data[2],
docPath: data[1].replace(src, "")
};
});
// Sorts files alphabetically by fullPath
files = _.sortBy(files, function (file) {
return file.fullPath.toLowerCase();
});
// Groups files based on their path and maps each
// result to an object containing the path and package
// of each final folder and the group of files it contains
groups = _.map(_.groupBy(files, function (file) {
return file.path;
}), function (files, key) {
return {
path: key.replace(src, ""),
ns: key.replace(src, "").replace(/\//g, ".").slice(0, -1),
ns_class: key.replace(src, "").replace(/\//g, "_").slice(0, -1),
files: files
};
});
// Resolve the promise returning a 'fileset' object
def.resolve({
files: files,
groups: groups,
length: files.length
});
});
shuffle: function (count) {
var deck, groups, parts = [];
deck = randomize(_.clone(initialDeck));
groups = _.groupBy(deck, function (value, index) {
return index % count;
});
_.each(_.values(groups), function (group) {
var split = _.map(group, function (value) {
return value.id;
});
parts.push(split);
});
return parts;
},
function setstat(data)
{
var bytrainsize = _.groupBy(data, function(num){ return num["output"][0] })
var bytrainsizecount = {}
_.each(bytrainsize, function(value, key, list){
bytrainsizecount[key] = value.length
}, this)
var bynumsen = {}
_.each(bytrainsize, function(value, key, list){
var sen = 0
_.each(value, function(value1, key1, list){
sen += value1["input"]["CORENLP"]["sentences"].length
}, this)
bynumsen[key] = sen
}, this)
var bychar = {}
_.each(bytrainsize, function(value, key, list){
var tokens = 0
_.each(value, function(value1, key1, list){
_.each(value1["input"]["CORENLP"]["sentences"], function(sen, key, list){
tokens += sen['tokens'].length
}, this)
}, this)
bychar[key] = tokens
}, this)
var output = "bytrainsize\n" +JSON.stringify(bytrainsizecount, null, 4) + "\n" +
"bynumsen\n" + JSON.stringify(bynumsen, null, 4) + "\n" +
"bychar\n" + JSON.stringify(bychar, null, 4)
return output
}
success: function(response) {
var data = _.groupBy(response.rows, function(row) {
return row.key;
})
callback(data);
}
// invoke: 在list里的每个元素上调用名为methodName的函数,任何附加的函数传入，invoke将会转给调用的函数
var v = _.invoke([[2,1,3], [4,5,2]], 'sort')
console.log("v", v);
// pluck: 提取一个集合里指定的属性值
var users = [
{name:'moe', age: 40},
{name:'larry', age: 50}
];
console.log(_.pluck(users, 'name'));
// max, min, sortBy：取list中的最大，最小元素，自定义比较器
console.log(_.max(users, function(stooge){
return stooge.age
}));
console.log(_.min(users, function(stooge){
return stooge.age //{ name: 'moe', age: 40 }
}));
console.log(_.sortBy([3,4,1,2,6], function(num){
return Math.max(num);//[ 1, 2, 3, 4, 6 ]
}));
// groupBy 一个集合分成多个集合
console.log(_.groupBy(['one', 'two', 'three'], 'length'));//{ '3': [ 'one', 'two' ], '5': [ 'three' ] }
// size 得到list中元素个数
console.log(_.size({one:1, two:2, three: 3})); //3
// shuffle：随机打乱一个数据
console.log(_.shuffle([1,2,3,4,5,6,7]));
// countBy 把一个数据分组后计数
console.log(_.countBy([1,2,3,4,5], function(num){
return num % 2 == 0 ? 'even':'odd'; //{ odd: 3, even: 2 }
}));
var f = Fiber(function() {
var fiber = Fiber.current;
// outer loop for VALUE as keyphrase
_.each(keyphrases, function(value, key, list){
utils.recursionredis([value], 2, function(err, resp){
fiber.run(resp)
})
// paraphases = value + paraphrases(value)
var paraphases = Fiber.yield()
// inner loop for SALIENT as keyphrase
_.each(keyphrases, function(salient, key, list){
// if they are not the same
if (salient != value)
{
var comparisonlist = _.map(paraphases, function(phrase){ return [phrase,salient] });
if (comparisonlist.length != 0)
{
async.mapSeries(comparisonlist, utils.compare, function(err,resp){
fiber.run(resp)
})
var scoredlist = Fiber.yield();
}
else
{
var scoredlist = []
}
var sorted = _.sortBy(scoredlist, function(num){ return num[4] });
sorted = sorted.reverse()
if (sorted.length != 0)
{
output.push({ 'X': value,
'Y': salient,
'X best fitted from PPDB to Y': sorted[0][0],
'Y control poin' : sorted[0][1],
'X part to compare' : sorted[0][2],
'Y part to compare' : sorted[0][3],
'score' : sorted[0][4]
})
}
}
}, this)
}, this)
output = _.sortBy(output, function(num){ return num['score'] })
output.reverse()
var crossdist = _.map(output, function(num){
num['score floored'] = Math.floor(num['score']*10)*10
return num });
crossdist = _.groupBy(crossdist, function(num) {return num['score floored']})
console.log(JSON.stringify(crossdist, null, 4))
for (var col=100; col>=0; col -= 10) {
var ind = 0
_.each(output, function(value, key, list){
if (value['score']*100 >= col)
ind = ind + 1
}, this)
console.log(col/100 + " -- " +(ind/output.length*100).toFixed()+"%")
}
utils.closeredis()
})
exports.topbar = function(head, req) {
var settingsDoc = {};
var selectedThemeDoc = null;
var row;
var apps = [];
start({'headers' : {'Content-Type' : 'text/html'}});
while ((row = getRow())) {
if (row.key[0] === 0) settingsDoc = row.value;
if (row.key[0] === 1) {
if (row.key[3] == 'install') {
apps.push({
title : row.key[2],
db :row.value.db,
doc : row.doc
})
}
if (row.key[3] == 'link') {
apps.push({
title : row.key[2],
link :row.value.url,
doc : row.doc,
external : true
})
}
}
if (row.key[0] === 2) selectedThemeDoc = row.doc;
}
var settings = _.defaults(settingsDoc, dashboard_settings.defaultSettings);
var username, avatar, avatar_large = null;
var isAdmin = false;
var isUser = utils.isUser(req);
var isAdminParty = utils.isAdminParty(req);
if (utils.isAdmin(req) || isAdminParty) {
isAdmin = true;
}
if (isUser) {
username = req.userCtx.name;
avatar = gravatar.avatarURL({
email : req.userCtx.name,
size : 20,
default_image : 'mm'
});
avatar_large = gravatar.avatarURL({
email : req.userCtx.name,
size : 96,
default_image : 'mm'
});
}
var dashboard_url = dashboard_links.dashboardURL(settings, 'dashboard', 'dashboard', req);
var home_url = dashboard_url;
if (settings.frontpage.use_link) {
home_url = settings.frontpage.link_url;
}
var settings_url = dashboard_url + "settings";
var login_url = dashboard_url + 'login';
var profile_url = null;
if (settings.sessions.type == 'other') {
login_url = settings.sessions.other.login_url;
_.templateSettings = {
interpolate: /\{(.+?)\}/g
}
var template = _.template(settings.sessions.other.profile_url);
profile_url = template({username : username});
}
var hide_apps = (!isUser && settings.host_options.redirect_frontpage_on_anon);
apps = _.map(apps, function(app) {
if (app.db) {
app.link = dashboard_links.appUrl(settingsDoc, app.doc, req);
}
return app;
});
var grouped_apps = _.groupBy(apps, function(app) {
if (app.doc.onDropdownMenu) return "more_apps"
else return "apps";
});
var usrCtxString = encodeURI(JSON.stringify(req.userCtx));
var templateContext = {
dashboard_url : dashboard_url,
home_url : home_url,
settings_url : settings_url,
login_url : login_url,
profile_url : profile_url,
apps : grouped_apps,
username : username,
avatar : avatar,
avatar_large : avatar_large,
isAdmin: isAdmin,
isAdminParty : isAdminParty,
hide_apps : hide_apps,
settings : settings,
usrCtxString : usrCtxString,
sessionType : settings.sessions.type,
extraResources : []
};
if (selectedThemeDoc) {
if (selectedThemeDoc._attachments) {
_.each(selectedThemeDoc._attachments, function(value, name){
if (name === 'modules.js') return ;
if (value.content_type == "application/javascript") {
templateContext.extraResources.push({
doc : selectedThemeDoc._id,
attachment: name
});
}
});
}
log(templateContext.extraResources);
var template = handlebars.compile(selectedThemeDoc['template.html']);
send(template(templateContext));
} else {
send(templates['topbar.html'](templateContext));
}
}
groupScenarios: function(scenarios){
return _.groupBy(scenarios,function(s){ return s.feature; });
},