function collector(key, nextKey, index, startsWith, num, callback, buffer) {
var lines = buffer.toString().split('\n'),
matches = lines.map(function (line) {
return line.substring(0, line.indexOf(' '));
});
//console.log(' got lines for key ', key, lines.length);
// we got bunch of matches for key - now search within for startsWith
if (startsWith !== key) {
// binary search for startsWith within set of matches
var ind = _.sortedIndex(matches, startsWith);
if (ind >= lines.length || matches[ind].indexOf(startsWith) === -1) {
callback && callback([], startsWith);
resolve([]);
return;
}
var trie = new Trie();
trie.addStrings(matches);
//console.log('Trie > ', trie.matchesWithPrefix( startsWith ));
matches = trie.keysWithPrefix(startsWith);
}
var words = _.sample(matches, num);
callback && callback(words, startsWith);
resolve(words);
}
_.each(gro, function(value, key, list){
value = _.filter(value, function(num){ return num['input']['CORENLP']['sentences'].length >= minsize })
value = _.map(value, function(num){ num["input"]["CORENLP"]["sentences"].splice(10)
return num });
if (value.length < sizetrain)
delete gro[key]
else
gro[key] = _.sample(value, sizetrain)
}, this)
}, function(err, res, body) {
if (typeof body.results.dongers !== "undefined") {
var dongers = body.results.dongers;
var donger = _.sample(dongers).donger;
} else {
var donger = null;
}
callback(err, donger);
});
function partCallback(result) {
if (result) {
results = _.uniq(results.concat(result)); // make sure it's unique!
}
if (results.length < count && doParts.length) {
return tryPart();
}
// final random and trim excess
results = _.sample(results, count);
done();
}
function marktrans()
{
var dataset = JSON.parse(fs.readFileSync("./dataset.json"))
var dataset_limited = _.sample(dataset, 1000)
_.each(dataset_limited, function(value, key, list){
if (!("trans" in value["input"]) && value["output"][0]!="System" && value["input"]["text"].split(" ").length > 2)
value["input"]["trans"] = {}
}, this)
console.log(JSON.stringify(dataset, null, 4))
process.exit()
}
return new Promise(function(resolve, reject) {
// select at random a POS to look at
var doParts = _.sample(parts, parts.length);
tryPart();
function tryPart() {
var part = doParts.pop(),
rand = 'rand' + part,
factor = POS_factor[part],
weight = factor / POS_factor.Total;
// pick count according to relative weight
opts.count = Math.ceil(count * weight * 1.1); // guard against dupes
self[rand](opts, partCallback);
}
function partCallback(result) {
if (result) {
results = _.uniq(results.concat(result)); // make sure it's unique!
}
if (results.length < count && doParts.length) {
return tryPart();
}
// final random and trim excess
results = _.sample(results, count);
done();
}
function done() {
profile && (args.push(new Date() - start));
args[0] = results;
callback && callback.apply(null, args);
resolve(results);
}
}); // Promise
return new Promise(function(resolve, reject) {
//console.log('-- ', startsWith, num, self.fastIndex.indexKeys.length);
if (startsWith) {
key = startsWith.slice(0, KEY_LENGTH);
/**
* if key is 'a' or 'ab' (<3 chars), search for ALL keys starting with that.
*/
if (key.length < KEY_LENGTH) {
// calc trie if haven't done so yet
if (!trie) {
trie = new Trie();
trie.addStrings(self.fastIndex.indexKeys);
self.fastIndex.trie = trie;
//console.log(' +++ Trie calc ');
}
try {
// trie throws if not found!!!!!
keys = trie.keysWithPrefix(startsWith);
} catch (e) {
keys = [];
}
// read all keys then select random word.
// May be large disk read!
key = keys[0];
nextKey = _.last(keys);
}
if (!key || !(key in self.fastIndex.offsets)) {
callback && callback([], startsWith);
resolve([]);
}
} else {
// no startWith given - random select among keys
keys = _.sample(self.fastIndex.indexKeys, num);
// if num > 1, run each key independently and collect results
if (num > 1) {
var results = [], ii = 0;
_(keys).each(function (startsWith) {
self.rand(startsWith, 1, function (result) {
results.push(result[0]);
if (++ii == num) {
callback && callback(results, '');
resolve(results);
}
});
});
return;
}
key = keys;
}
// prepare the piper
var args = [key, nextKey, self],
task = 'rand:' + key + nextKey,
context = [startsWith, num, callback]; // last arg MUST be callback
// pay the piper
self.piper(task, IndexFile.readIndexBetweenKeys, args, context, collector);
function collector(key, nextKey, index, startsWith, num, callback, buffer) {
var lines = buffer.toString().split('\n'),
matches = lines.map(function (line) {
return line.substring(0, line.indexOf(' '));
});
//console.log(' got lines for key ', key, lines.length);
// we got bunch of matches for key - now search within for startsWith
if (startsWith !== key) {
// binary search for startsWith within set of matches
var ind = _.sortedIndex(matches, startsWith);
if (ind >= lines.length || matches[ind].indexOf(startsWith) === -1) {
callback && callback([], startsWith);
resolve([]);
return;
}
var trie = new Trie();
trie.addStrings(matches);
//console.log('Trie > ', trie.matchesWithPrefix( startsWith ));
matches = trie.keysWithPrefix(startsWith);
}
var words = _.sample(matches, num);
callback && callback(words, startsWith);
resolve(words);
}
}); // Promise
function purge(s, action) {
/*
The action will determine how we deal with the room/user removal.
These are the following scenarios:
if the user is the owner and (s)he:
1) disconnects (i.e. leaves the whole server)
- advise users
- delete user from people object
- delete room from rooms object
- delete chat history
- remove all users from room that is owned by disconnecting user
2) removes the room
- same as above except except not removing user from the people object
3) leaves the room
- same as above
if the user is not an owner and (s)he's in a room:
1) disconnects
- delete user from people object
- remove user from room.people object
2) removes the room
- produce error message (only owners can remove rooms)
3) leaves the room
- same as point 1 except not removing user from the people object
if the user is not an owner and not in a room:
1) disconnects
- same as above except not removing user from room.people object
2) removes the room
- produce error message (only owners can remove rooms)
3) leaves the room
- n/a
*/
//console.log(rooms);
if (people[s.id].inroom) { //user is in a room
var room = rooms[people[s.id].inroom]; //check which room user is in.
if (s.id === room.owner) { //user in room and owns room
if (action === "disconnect") {
/*
io.sockets.in(s.room).emit("update", "The owner (" +people[s.id].name + ") has left the server. The room is removed and you have been disconnected from it as well.");
var socketids = [];
for (var i=0; i<sockets.length; i++) {
socketids.push(sockets[i].id);
if(_.contains((socketids)), room.people) {
sockets[i].leave(room.name);
}
}
if(_.contains((room.people)), s.id) {
for (var i=0; i<room.people.length; i++) {
people[room.people[i]].inroom = null;
}
}
room.people = _.without(room.people, s.id); //remove people from the room:people{}collection
delete rooms[people[s.id].owns]; //delete the room
delete people[s.id]; //delete user from people collection
delete chatHistory[room.name]; //delete the chat history
*/
var roomId = people[s.id].owns;
io.sockets.in(s.room).emit("update", "The owner (" +people[s.id].name + ") has left the server. Searching new owner.");
room.people = _.without(room.people, s.id);
//находим тех кто в комнате
var socketids = [];
for (var i=0; i<sockets.length; i++) {
if(_.contains((sockets[i].id)), room.people) {
socketids.push(sockets[i].id);
}
}
//console.log('LEN'+room.people.length);
//console.log('SocketsIds'+socketids);
if(room.people.length == 0){
room.people = _.without(room.people, s.id); //remove people from the room:people{}collection
delete rooms[people[s.id].owns]; //delete the room
delete people[s.id]; //delete user from people collection
delete chatHistory[room.name]; //delete the chat history
return;
}
//console.log('LEN:'+socketids.length);
if(socketids.length > 1){
//console.log('search new');
//выбираем рандомного
var newOwner = _.sample(room.people);
//назначаем главным
//console.log('newOwner:'+newOwner);
for (var i=0; i<sockets.length; i++) {
if(sockets[i].id == newOwner){
//назначаем нового главного румы
rooms[people[s.id].owns].owner = sockets[i].id;
people[sockets[i].id].owns = people[s.id].owns;
io.sockets.in(s.room).emit("update", "The new owner room is "+people[sockets[i].id].name+" ");
}else{
//console.log('cant find owner?');
}
}
delete people[s.id]; //удаляем из народа
}else{
//console.log('delete room');
//delete the room
delete rooms[roomId];
room.people = _.without(room.people, s.id); //remove people from the room:people{}collection
delete people[s.id]; //delete user from people collection
delete chatHistory[room.name]; //delete the chat history
return;
}
sizePeople = _.size(people);
sizeRooms = _.size(rooms);
io.sockets.emit("update-people", {people: people, count: sizePeople});
io.sockets.emit("roomList", {rooms: rooms, count: sizeRooms});
var o = _.findWhere(sockets, {'id': s.id});
sockets = _.without(sockets, o);
} else if (action === "removeRoom") { //room owner removes room
io.sockets.in(s.room).emit("update", "The owner (" +people[s.id].name + ") has removed the room. The room is removed and you have been disconnected from it as well.");
var socketids = [];
for (var i=0; i<sockets.length; i++) {
socketids.push(sockets[i].id);
if(_.contains((socketids)), room.people) {
sockets[i].leave(room.name);
}
}
if(_.contains((room.people)), s.id) {
for (var i=0; i<room.people.length; i++) {
people[room.people[i]].inroom = null;
}
}
delete rooms[people[s.id].owns];
people[s.id].owns = null;
room.people = _.without(room.people, s.id); //remove people from the room:people{}collection
delete chatHistory[room.name]; //delete the chat history
sizeRooms = _.size(rooms);
io.sockets.emit("roomList", {rooms: rooms, count: sizeRooms});
} else if (action === "leaveRoom") { //room owner leaves room
io.sockets.in(s.room).emit("update", "The owner (" +people[s.id].name + ") has left the room. The room is removed and you have been disconnected from it as well.");
var socketids = [];
for (var i=0; i<sockets.length; i++) {
socketids.push(sockets[i].id);
if(_.contains((socketids)), room.people) {
sockets[i].leave(room.name);
}
}
if(_.contains((room.people)), s.id) {
for (var i=0; i<room.people.length; i++) {
people[room.people[i]].inroom = null;
}
}
delete rooms[people[s.id].owns];
people[s.id].owns = null;
room.people = _.without(room.people, s.id); //remove people from the room:people{}collection
delete chatHistory[room.name]; //delete the chat history
sizeRooms = _.size(rooms);
io.sockets.emit("roomList", {rooms: rooms, count: sizeRooms});
}
} else {//user in room but does not own room
//console.log('i not owner');
if (action === "disconnect") {
io.sockets.emit("update", people[s.id].name + " has disconnected from the server.");
if (_.contains((room.people), s.id)) {
var personIndex = room.people.indexOf(s.id);
room.people.splice(personIndex, 1);
s.leave(room.name);
}
delete people[s.id];
sizePeople = _.size(people);
io.sockets.emit("update-people", {people: people, count: sizePeople});
var o = _.findWhere(sockets, {'id': s.id});
sockets = _.without(sockets, o);
} else if (action === "removeRoom") {
s.emit("update", "Only the owner can remove a room.");
} else if (action === "leaveRoom") {
if (_.contains((room.people), s.id)) {
var personIndex = room.people.indexOf(s.id);
room.people.splice(personIndex, 1);
people[s.id].inroom = null;
io.sockets.emit("update", people[s.id].name + " has left the room.");
s.leave(room.name);
}
}
}
} else {
//The user isn't in a room, but maybe he just disconnected, handle the scenario:
if (action === "disconnect") {
io.sockets.emit("update", people[s.id].name + " has disconnected from the server.");
delete people[s.id];
sizePeople = _.size(people);
io.sockets.emit("update-people", {people: people, count: sizePeople});
var o = _.findWhere(sockets, {'id': s.id});
sockets = _.without(sockets, o);
}
}
}
var datasets = [
'turkers_keyphrases_only_rule.json',
'students_keyphrases_only_rule.json'
]
var data = []
_.each(datasets, function(value, key, list){
data = data.concat(JSON.parse(fs.readFileSync("../../datasets/Employer/Dialogue/"+value)))
}, this)
var keyphrases = []
keyphrases = utils.extractkeyphrases(data)
keyphrases = _.compact(_.unique(keyphrases))
keyphrases = _.sample(keyphrases, 40)
console.log("Keyphrases for Intent Offer")
console.log("size "+keyphrases.length)
console.log(keyphrases)
var output = []
var f = Fiber(function() {
var fiber = Fiber.current;
// outer loop for VALUE as keyphrase
_.each(keyphrases, function(value, key, list){
utils.recursionredis([value], 2, function(err, resp){
fiber.run(resp)
})
getRandomKey: function(length) {
srcArr = _.shuffle(srcArr);
var srcSample = _.sample(srcArr, length);
return srcSample.join('');
}
}
if ((str!="") && (act in labels))
dataset.push(elem)
// if (!(act in labels))
// console.log(elem)
}
}, this)
///console.log(JSON.stringify(dataset, null, 4))
///console.log(JSON.stringify(dataset.length, null, 4))
var sam = _.sample(dataset,1500)
var dist = _.countBy(sam, function(num) { return num.dsc })
dist = _.pairs(dist)
dist = _.sortBy(dist, function(num){ return num[1] });
console.log(JSON.stringify(dist, null, 4))
var omit = []
_.each(dist, function(value, key, list){ if (value[1]<10) omit.push(value[0]) }, this)
var sam = _.filter(sam, function(num){ return omit.indexOf(num.dsc) == -1 });
console.log(JSON.stringify(sam.length, null, 4))