onlycontent(Y, function (err1,replies1){
var compY = replies1
if (compY.length == 0)
compY = Y.split(" ")
if (compX.length == 0)
compX = X.split(" ")
compX = lemmatize(compX)
compY = lemmatize(compY)
var compXX = compX
var compYY = compY
_.each(elim, function(value, key, list){
compXX = _.without(compXX,value)
compYY = _.without(compYY,value)
}, this)
if (compYY.length == 0)
compYY = Y.split(" ")
if (compXX.length == 0)
compXX = X.split(" ")
compXX = _.compact(compXX)
compYY = _.compact(compYY)
callback(err1+err,[X, Y, compXX, compYY, distance(compXX, compYY)])
})
onlycontent(str, function (err,strcontent){
if (_.compact(strcontent).length == 0)
strcontent = str.split(" ")
strcontentlemma = lemmatize(strcontent)
strcontentlemma = elimination(strcontentlemma)
if (strcontentlemma.length == 0)
strcontentlemma = strcontent
strcontentlemma = _.compact(strcontentlemma)
callback(err, strcontentlemma)
})
var returnPathInfo = function(reqUrl) {
var querystring = require('querystring')
, url = require('url')
, path = url.parse(reqUrl).pathname
, query = url.parse(reqUrl).query || ''
, pathArray = _.compact(path.split('/'))
, pathInfo = {}
, lastElement = pathArray[pathArray.length-1] || []
pathInfo.path = path
if (lastElement.indexOf('.') !== -1) {
pathInfo.file = {}
pathInfo.file.name = lastElement.split('.')[0]
pathInfo.file.ext = _.last(lastElement.split('.'))
pathInfo.file.contentType = app.contentTypes[pathInfo.file.ext] || ''
if (pathInfo.file.contentType === '') console.log(('Missing content type for ' + pathInfo.file.ext).warn)
pathInfo.file.cache = app.contentTypes[pathInfo.file.ext[1]] || 'nocache'
}
else {
pathInfo.rest = {}
pathInfo.rest.page = pathArray[0] || 'home'
pathInfo.rest.control = '/' + (pathArray[0] || '')
pathInfo.rest.type = pathArray[1] || ''
pathInfo.rest.item = pathArray[2] || null
pathInfo.rest.title = pathArray[3] || ''
}
;(query !== '')
? pathInfo.query = querystring.parse(query)
: pathInfo.query = {}
return pathInfo
}
_createWritePromises: function(branches, trackedBranches) {
var writePromises = trackedBranches.map(function(branch) {
console.log("looking at " + branch.name + ", " + branch.status);
var info = branches[branch.name];
if (info) {
return this._db.updateTrackedBranch(branch.name, {
name : branch.name,
status : info.green ? 'success' : 'failed',
previousStatus : branch.status || 'new',
lastBuild : {
time : info.last_built,
url : info.url,
sha : info.sha
}
});
}
else {
console.log("Warning: did not find branch '" + branch.name + "' in server status JSON");
return null;
}
}, this);
return _.compact(writePromises);
}
_.each(json.map(Compensate), function(value, key, list){
if (value.length > n)
{
if (_.compact(value[n].toString()).length != 0)
buf = buf.concat(value[n])
}
})
exports.buildTrees(deps, cfg, opt, function (err, local, updated) {
if (err) {
return callback(err);
}
var all_names = _.uniq(_.keys(local).concat(_.keys(updated)));
var changed = all_names.map(function (name) {
var lversion = local[name] ? local[name].current_version: null;
var uversion = updated[name] ? updated[name].current_version: null;
if (lversion) {
var lpkg = local[name].versions[lversion];
if (lpkg.source === 'repository') {
// cannot even satisfy requirements with current package,
// this needs re-installing from repositories
return {
name: name,
version: uversion,
old: 'not satisfiable'
};
}
}
if (!local[name] && updated[name] || lversion !== uversion) {
return {name: name, version: uversion, old: lversion};
}
});
callback(null, _.compact(changed), local, updated);
});
socket.on('chatMsg', function(msg){
if(!socket.handshake||!socket.handshake.user|| socket.handshake.user.user.name==='Guest'){
return false;
}
var chatMessage = {};
chatMessage.msg = msg.msg || '';
var replies = msg.repliesTo.split(';');
if(replies.length>0){
replies = _.map(_.compact(replies),function(reply){
return reply.split(':');
});
}
models.user.find({'_id':{'$in':_.map(replies,function(reply){
return reply[1];
})}},{'profile_url':1,'name':1},function(err,results){
chatMessage.replies = [];
_.map(results,function(replied){
var replacer = new RegExp('@'+replied.name,'ig');
chatMessage.replies.push({'to':replied.name});
chatMessage.msg = chatMessage.msg.replace(replacer,'<a href="'+replied.profile_url+'">@'+replied.name+'</a>');
});
chatMessage.sender = {
'profile_url' :socket.handshake.user.user.profile_url,
'name' :socket.handshake.user.user.name,
'profile_pic_url' :socket.handshake.user.user.profile_pic_url
};
socket.get('chatId', function(err, chat){
if(socket.handshake.user.auth&&socket.handshake.user.auth.twitter&&msg.postToTwitter===true){
var twitterKeys = socket.handshake.user.auth.twitter;
streams.twitter.postMessage(twitterKeys.accessToken,twitterKeys.accessTokenSecret,chatMessage.msg+' #'+chat.replace('/',''),function(err,result){
if(err){
console.log(err);
return false;
}
if(result){
console.log(chatMessage.msg+ ' is posted!');
}
});
}
(new models.chatMsg({
'user' :chatMessage.sender,
'chatId' :chat,
'message' :chatMessage.msg,
'type' :'user',
'replies': chatMessage.replies
})).save(function(err, msg){
chatMessage.id = msg._id;
chatMessage.created_at = msg.created_at;
io.sockets['in'](chat).emit('chatMsg', chatMessage);
});
});
});
});
_.each(responses, function(response, key, list){
if (response[4] == 1)
{
if (!(actual in output))
output[actual] = []
output[actual].push(input)
output[actual] = _.compact(_.unique(output[actual]))
}
}, this)
function subst(str) {
var subst = []
var str = str.split(" ")
for (var start=0; start<=str.length; ++start) {
for (var end=start+1; end<=str.length; ++end) {
subst.push(str.slice(start,end).join(" "));
}
}
return _.compact(subst)
}
onlycontent(seed, function (err,replies){
toretrieve.push(replies.join(" "))
toretrieve = toretrieve.concat(replies)
toretrieve = _.compact(_.unique(toretrieve))
async.mapSeries(toretrieve, quickfetch, function(err, resultArr) {
callback(err,_.compact(_.unique(_.flatten(resultArr))))
})
})
function cleancompare(ar)
// function compare(X,Y)
{
// var elim = []
var elim = ['is', 'are', 'be', 'will', 'let', 'i', 'I', 'no', 'not', 'to', 'you', 'we', 'for',
'i\'ll', 'so', 'the', 'can\'t', 'let\'s', 'only', 'on']
var X = ar[0]
var Y = ar[1]
var compX = cleanposoutput(cleanpos(X))
var compY = cleanposoutput(cleanpos(Y))
if (compY.length == 0)
compY = Y.split(" ")
if (compX.length == 0)
compX = X.split(" ")
compX = lemmatize(compX)
compY = lemmatize(compY)
var compXX = compX
var compYY = compY
_.each(elim, function(value, key, list){
compXX = _.without(compXX,value)
compYY = _.without(compYY,value)
}, this)
if (compYY.length == 0)
compYY = Y.split(" ")
if (compXX.length == 0)
compXX = X.split(" ")
compXX = _.compact(compXX)
compYY = _.compact(compYY)
return [X, Y, compXX, compYY, distance(compXX, compYY)]
}
exports.hfetch = function (o, item) {
var splitChar = item.split('.').length > 0 ? '.' : '/'
, items = _.compact(item.split(splitChar))
, found = o;
if (items.length > 1) {
items.forEach(function(tag) {
found = exports.fetch(found, tag);
});
return found;
}
return exports.fetch(o, item);
};
client.get(word, function (err, response) {
if (response == null)
{
output[word] = []
callback(err, [])
}
else
{
var vec = _.compact(response.split(","))
vec = _.map(vec, function(value){ return parseFloat(value); });
output[word] = vec
callback(err, vec)
}
})
async.map(__peanut__.options.path, statPath, function(err, results) {
var fsStats = _.compact(results);
async.map(fsStats, processFileOrDir, function(err, results) {
var results = _(results).chain().flatten().compact().value();
callback(err, results);
});
function processFileOrDir(stats, cb) {
if (stats.isDirectory()) {
var files = findit.sync(stats.path);
async.map(files, parseFeature, function(err, results) {
cb(err, results);
});
};
if (stats.isFile()) {
parseFeature(stats.path, cb);
};
};
});
function generatengrams(sentence)
{
var tokenizer = new natural.RegexpTokenizer({pattern: /[^a-zA-Z0-9%'$+-]+/});
var words = tokenizer.tokenize(sentence);
var feature = []
_(3).times(function(n){
feature = feature.concat(bars.skipgrams(words, n, 3))
})
var features = []
_.each(feature, function(value, key, list){
if (!bars.isstopword(value))
features.push(value.join(" "))
}, this)
features = _.unique(features)
features = _.sortBy(features, function(num){ return num.length })
features = _.compact(features)
return features
}
}, (err, ret)=> err ? next(err) : res.json(_.compact(ret)));
], function(err, items) {
var prefix = options.appOptions.featherRoot,
featherFiles = items['feather-files'],
restProxyInfo = items['feather-restProxyInfo'];
var cssFiles = [];
// js -----------------------------------------------------------------
var jsFiles = [
{path: "/feather-client/lib/underscore-min.js", prefix: prefix},
options.appOptions["socket.io"].enabled ? {path: "/socket.io/socket.io.js", prefix: prefix, virtual: true} : null,
{path: "/feather-client/lib/json2.js", prefix: prefix, minifyAllowed: false},
{path: "/feather-client/lib/jquery-1.7.min.js", prefix: prefix},
// {path: "/feather-client/lib/jquery-1.9.migrate.min.js", prefix: prefix},
{path: "/feather-client/lib/jquery.tmpl.js", prefix: prefix},
{path: "/feather-client/lib/jquery.cookie.js", prefix: prefix},
{path: "/feather-client/lib/inherits.js", prefix: prefix},
{path: "/feather-client/feather.js", prefix: prefix},
{path: "/feather-client/base-class.js", prefix: prefix},
{path: "/feather-client/event-publisher.js", prefix: prefix},
{path: "/feather-client/dom-event-cache.js", prefix: prefix},
{path: "/feather-client/registry.js", prefix: prefix},
{path: "/feather-client/semaphore.js", prefix: prefix},
{path: "/feather-client/fsm.js", prefix: prefix},
{path: "/feather-client/util.js", prefix: prefix},
{path: "/feather-client/widget.js", prefix: prefix},
{path: "/feather-client/socket.js", prefix: prefix}
];
if (options.appOptions.rest && options.appOptions.rest.autoGenerateProxy && restProxyInfo) {
//add the restProxy files
jsFiles.push({path: "/feather-client/restProxy.js", prefix: prefix});
jsFiles.push({
path: restProxyInfo.path,
prefix: restProxyInfo.prefix
});
}
// Only add the auth scripts if this app has auth enabled.
if (options.appOptions.auth.enabled) {
//jsFiles.push({path: "feather-client/sha512.js", prefix: prefix});
jsFiles.push({path: "/feather-client/auth-client.js", prefix: prefix});
}
// Add datalinking if enabled
if (options.appOptions.data.datalinking.enabled) {
jsFiles.push({path: "/feather-client/lib/jquery.datalink.js", prefix: prefix});
}
// Add files for the ui provider if enabled
if (options.appOptions.ui.enabled) {
var uiJSPrefix = prefix,
uiCSSPrefix = prefix,
provider = options.appOptions.ui.provider,
providers = options.appOptions.ui.providers;
if (typeof provider === 'string') {
provider = _.find(providers, function(_provider) {
return _provider.name === provider;
});
}
//get the js files
var appFiles = _.keys(featherFiles.appFiles);
if (provider.jsRoot === "/") uiJSPrefix = options.appOptions.publicRoot;
_.each(provider.jsFiles, function(file) {
var _path = file,
_prefix = uiJSPrefix;
//detect automatic app overrides (via dropping in new files in the /public/_ui folder)
var fileName = path.basename(file);
var overrideFile = _.find(appFiles, function(_appFile) {
return _appFile.indexOf('/public/_ui/' + provider.name + '/js/' + fileName) > -1;
});
if (overrideFile) {
_prefix = options.appOptions.publicRoot;
_path = '/' + path.relative(options.appOptions.publicRoot, overrideFile);
}
jsFiles.push({path: _path, prefix: _prefix});
});
//get the css files
if (provider.cssRoot === "/") uiCSSPrefix = options.appOptions.publicRoot;
_.each(provider.cssFiles, function(file) {
var _path = file,
_prefix = uiCSSPrefix;
//detect automatic app overrides (via dropping in new files in the /public/_ui folder)
var fileName = path.basename(file);
var overrideFile = _.find(appFiles, function(_appFile) {
return _appFile.indexOf('/public/_ui/' + provider.name + '/css/' + fileName) > -1;
});
if (overrideFile) {
_prefix = options.appOptions.publicRoot;
_path = '/' + path.relative(options.appOptions.publicRoot, overrideFile);
}
cssFiles.push({path: _path, prefix: _prefix});
});
}
var jsOptions = _.extend(_.clone(options), {
template: '<clientscript type="text/javascript" src="${href}"></clientscript>',
cacheName: "feather-client-core.js",
hrefPrefix: "",
contentType: "text/javascript",
files: _.compact(jsFiles)
});
var frameworkPackaged = new Semaphore(callback);
frameworkPackaged.increment();
exports.packageResources(jsOptions, function(err) {
frameworkPackaged.execute(err);
});
if (cssFiles.length) {
var cssOptions = _.extend(_.clone(options), {
template: '<link rel="stylesheet" type="text/css" href="${href}" />',
cacheName: "feather-client-core.css",
contentType: "text/css",
files: cssFiles
});
frameworkPackaged.increment();
exports.packageResources(cssOptions, function(err) {
frameworkPackaged.execute(err);
});
}
});
async.mapSeries(toretrieve, quickfetch, function(err, resultArr) {
callback(err,_.compact(_.unique(_.flatten(resultArr))))
})
// collect only offer keyphrases
var keyph = []
_.each(turns, function(value, key, list){
if ('intent_keyphrases_rule' in value)
if ('Offer' in value['intent_keyphrases_rule'])
{
var phrase = value['intent_keyphrases_rule']['Offer']
phrase = phrase.replace("<ATTRIBUTE>","")
phrase = phrase.replace("<VALUE>","")
if (phrase != 'DEFAULT INTENT')
keyph.push(phrase)
}
}, this)
keyph = _.compact(_.unique(keyph))
var tocomp = utils.crosslist(keyph)
// filtering the gold standard keyphrases that are equal according to the comparison scheme
var f = Fiber(function() {
var fiber = Fiber.current;
async.mapSeries(tocomp, utils.compare, function(err,results){
fiber.run(results);
})
var results = Fiber.yield();
_.each(results, function(value, key, list){
if (value[4] == 1)
keyph = _.without(keyph, value[1])
, bodyClass: function(req, res) {
var parts = require('url').parse(req.url).pathname.split('/');
parts = _.compact(parts);
if (parts.length == 0) return 'home';
return parts.join('-');
}
return _.map(label, function(lab){ return _.uniq(_.compact(lab))})
var Fiber = require('fibers');
var datasets = [
'turkers_keyphrases_only_rule.json',
'students_keyphrases_only_rule.json'
]
var data = []
_.each(datasets, function(value, key, list){
data = data.concat(JSON.parse(fs.readFileSync("../../datasets/Employer/Dialogue/"+value)))
}, this)
var keyphrases = []
keyphrases = utils.extractkeyphrases(data)
keyphrases = _.compact(_.unique(keyphrases))
keyphrases = _.sample(keyphrases, 40)
console.log("Keyphrases for Intent Offer")
console.log("size "+keyphrases.length)
console.log(keyphrases)
var output = []
var f = Fiber(function() {
var fiber = Fiber.current;
// outer loop for VALUE as keyphrase
_.each(keyphrases, function(value, key, list){
utils.recursionredis([value], 2, function(err, resp){
fiber.run(resp)