}, function(err){
synonyms = _.map(_.unique(synonyms), function(num){ return [num, wordCount.indexOf(num)] });
antonyms = _.map(_.unique(antonyms), function(num){ return [num, wordCount.indexOf(num)] });
synonyms = _.sortBy(synonyms, function(num){ return num[1] }).reverse()
antonyms = _.sortBy(antonyms, function(num){ return num[1] }).reverse()
wordnet.close()
synonyms = _.filter(synonyms, function(num){ return (num[0].indexOf("_")==-1) })
antonyms = _.filter(antonyms, function(num){ return (num[0].indexOf("_")==-1) })
// eliminate the numbers
synonyms = _.unique(_.flatten(_.map(synonyms, function(num){ return num[0] })))
antonyms = _.unique(_.flatten(_.map(antonyms, function(num){ return num[0] })))
console.vlog("DEBUGWORDNET: synonyms: "+JSON.stringify(synonyms, null, 4))
console.vlog("DEBUGWORDNET: antonyms: "+JSON.stringify(antonyms, null, 4))
callbackg(null, {
'synonyms':_.filter(synonyms, function(num){ return (num[0].indexOf("_")==-1) }),
'antonyms':_.filter(antonyms, function(num){ return (num[0].indexOf("_")==-1) })
})
})
var f = Fiber(function() {
var fiber = Fiber.current;
var keyphrases = utils.extractkeyphrases(data)
var output = {}
utils.clusteration(_.unique(keyphrases), function(err, clusters){
// fiber.run(clusters)
// _.each(clusters, function(value, key, list){
// utils.normalizer(value[0], function (err, norm_X){
// setTimeout(function() {
// fiber.run(norm_X)
// }, 2000);
// })
// var keyphr = Fiber.yield()
// output[keyphr] = value
// }, this)
async.mapSeries(clusters, utils.normalizer, function(err, norm) {
_.each(norm, function(value, key, list){
output[value.join(" ")] = clusters[key]
}, this)
console.log(output)
})
// normalizer(X, function (err, norm_X){
// console.log(output)
})
// process.exit(0)/
})
var cleanposfromredis = function(data, withscores)
{
if (withscores == false)
{
var output = []
_.each(data, function(value, key, list){
if (key % 2 == 0)
output.push(value.split("^")[0])
}, this)
// if you don't need score and POS than unique on seeds
return _.unique(output)
}
else
{
var output = []
var temp = []
_.each(data, function(value, key, list){
if (key % 2 == 0)
{
temp = temp.concat(value.split("^"))
}
else
{
temp.push(parseFloat(value))
output.push(temp.slice(0))
temp = []
}
}, this)
output = _.sortBy(output, function(num){ return num[0] })
return output
}
}
wordnet.lookupSynonyms(seed, function(results) {
var output = []
_.each(results, function(value, key, list){
if (value['pos'] = "v")
output.push(value['lemma'].split("_").join(" "))
}, this)
callback(null, _.unique(output))
})
wordnet.lookupSynonyms(seed, function(results) {
console.log(results)
_.each(results, function(value, key, list){
var list = _.map(value['synonyms'], function(num){ return num.split("_").join(" ") })
output = output.concat(list)
}, this)
callback(null, _.unique(output))
})
_.each(responses, function(response, key, list){
if (response[4] == 1)
{
if (!(actual in output))
output[actual] = []
output[actual].push(input)
output[actual] = _.compact(_.unique(output[actual]))
}
}, this)
it("should shuffle without repeats in 40 turns", function (done) {
var splits = [];
for (var i = 0; i < 40; i++) {
splits.push(deck.shuffle(1).join(","));
}
splits.should.have.length(_.unique(splits).length);
done();
});
var onlyIntents = function(labels)
{
var output = []
_.each(labels, function(label, key, list){
var lablist = splitJson(label)
output = output.concat(lablist[0])
}, this)
return _.unique(output)
}
onlycontent(seed, function (err,replies){
toretrieve.push(replies.join(" "))
toretrieve = toretrieve.concat(replies)
toretrieve = _.compact(_.unique(toretrieve))
async.mapSeries(toretrieve, quickfetch, function(err, resultArr) {
callback(err,_.compact(_.unique(_.flatten(resultArr))))
})
})
async.mapSeries([seed], quickfetch, function(err, bestlist1) {
// async.mapSeries(_.flatten(bestlist1), quickfetch, function(err, bestlist2) {
// async.mapSeries(_.flatten(bestlist2), quickfetch, function(err, bestlist3) {
// fetched = fetched.concat(bestlist1).concat(bestlist2).concat(bestlist3)
// fetched = fetched.concat(bestlist1).concat(bestlist2)
fetched = fetched.concat(bestlist1)
callback(null, _.unique(_.flatten(fetched)))
// })
// })
})
/*
input : ['20,000 NIS']
output: ['20,000 NIS', 'Salary']
*/
function addcomplement(data)
{
_.each(RuleValues, function(values, attribute, list){
_.each(values, function(value, key2, list2){
if (_.isArray(value))
value = value[1]
if (data.indexOf(value) != -1)
data.push(attribute)
}, this)
}, this)
return _.unique(data)
}
async.mapSeries(seed, quickfetch, function(err, bestlist1) {
// async.mapSeries(_.flatten(bestlist1), quickfetch, function(err, bestlist2) {
// async.mapSeries(_.flatten(bestlist2), quickfetch, function(err, bestlist3) {
// fetched = fetched.concat(bestlist1).concat(bestlist2).concat(bestlist3)
// fetched = fetched.concat(bestlist1).concat(bestlist2)
// fetched = fetched.concat(bestlist1)
// console.log(fetched)
// process.exit(0)
callback(null, _.unique(_.flatten(fetched)))
// })
// })
})
function generatengrams(sentence)
{
var tokenizer = new natural.RegexpTokenizer({pattern: /[^a-zA-Z0-9%'$+-]+/});
var words = tokenizer.tokenize(sentence);
var feature = []
_(3).times(function(n){
feature = feature.concat(bars.skipgrams(words, n, 3))
})
var features = []
_.each(feature, function(value, key, list){
if (!bars.isstopword(value))
features.push(value.join(" "))
}, this)
features = _.unique(features)
features = _.sortBy(features, function(num){ return num.length })
features = _.compact(features)
return features
}
async.mapSeries(toretrieve, quickfetch, function(err, resultArr) {
callback(err,_.compact(_.unique(_.flatten(resultArr))))
})
function inSentence(sentence, keyphrase, goldLable)
{
// var goldLable = keyphrase
// console.log(sentence)
keyphrase = keyphrase.toLowerCase()
// var sentencelist = sentence.split(" ")
// var output = []
// var sentenceList = sentence.split(" ")
// _.each(sentenceList, function(element, key, list){
// if (element == keyphrase)
// output.push(key)
// }, this)
var listToFind = []
var output = []
listToFind.push(keyphrase)
listToFind = _.unique(listToFind.concat(keyphrase.split(" ")))
// remove "with" from labels
listToFind = _.without(listToFind,'with')
listToFind = _.without(listToFind,'without')
listToFind = _.without(listToFind,'working')
listToFind = _.without(listToFind,'no')
listToFind = _.without(listToFind,'usd')
// process.exit(0)
var alreadycared = false
_.each(listToFind, function(phrase, key, list){
// console.log(phrase)
var index = compeletePhrase(sentence, phrase)
// console.log(index)
if (index != -1)
{
if ((goldLable == 'Leased Car') && (alreadycared == false))
{
alreadycared = true
var negation = truth_utils.negation(sentence.replace('without','no'),['agreement'], truth_filename)
if (negation == true)
{
// output.push([-1,-1,'No agreement',''])
output.push(['No agreement','no agreement',getWords(sentence,"no agreement"), whatinSentence(sentence,['no agreement'])])
}
else
{
var negation1 = truth_utils.negation(sentence.replace('without','no'),['car','lease'], truth_filename)
if (negation1 == true)
output.push(['Without leased car','car',getWords(sentence,"car"), whatinSentence(sentence,['car','leased'])])
else
output.push(['With leased car','car',getWords(sentence,"car"), whatinSentence(sentence,['car','leased'])])
}
}
output.push([goldLable, sentence.slice(index, index + phrase.length), getWords(sentence, phrase), [index, index + phrase.length]])
}
// else
// res = false
}, this)
// if (sentence == "i agree to with leased @ 8 hours with slow promotion track")
// {
// console.log(output)
// console.log(goldLable)
// }
// process.exit(0)
return output
}
var Fiber = require('fibers');
var datasets = [
'turkers_keyphrases_only_rule.json',
'students_keyphrases_only_rule.json'
]
var data = []
_.each(datasets, function(value, key, list){
data = data.concat(JSON.parse(fs.readFileSync("../../datasets/Employer/Dialogue/"+value)))
}, this)
var keyphrases = []
keyphrases = utils.extractkeyphrases(data)
keyphrases = _.compact(_.unique(keyphrases))
keyphrases = _.sample(keyphrases, 40)
console.log("Keyphrases for Intent Offer")
console.log("size "+keyphrases.length)
console.log(keyphrases)
var output = []
var f = Fiber(function() {
var fiber = Fiber.current;
// outer loop for VALUE as keyphrase
_.each(keyphrases, function(value, key, list){
utils.recursionredis([value], 2, function(err, resp){
fiber.run(resp)
var callback=function(){
console.log("ex")
console.log(retrieved.length)
retrieved = _.uniq(retrieved)
console.log(retrieved.length)
console.log(retrieved)
var TPlist = []
var TPlistret = []
var flag = 0
var TP = 0
var FP = 0
var rightpairs = []
var wrongpairs = []
_.each(retrieved, function(value, key, list){
var flag = 0
var comparison = value.replace("you","")
comparison = comparison.replace("we","")
_.each(Object.keys(stats['Offer']), function(gold, key, list){
if (natural.JaroWinklerDistance(comparison, gold)>Distance_threshold)
{
// if (!(gold in TPlist))
// {
// TP = TP + 1
TPlist.push(gold)
TPlistret.push(value)
flag = 1
// rightpairs.push([value,gold])
// }
}
}, this)
if (flag == 0)
{
FP = FP + 1
wrongpairs.push(value)
}
}, this)
var FN = Object.keys(stats['Offer']).length - _.unique(TPlist).length
TPlistret = _.unique(TPlistret)
TP = TPlistret.length
var Precision = TP/(TP+FP)
var Recall = TP/(TP+FN)
var F1 = 2*Precision*Recall/(Recall+Precision)
console.log("TP"+TP)
console.log("FP"+FP)
console.log("FN"+FN)
console.log("TP list")
console.log(TPlistret)
console.log("FP list")
console.log(wrongpairs)
console.log('Precision '+Precision)
console.log('Recall '+Recall)
console.log('F1 '+F1)
process.exit(0)
}
_.each(test_turns, function(test, key, list){
console.log(test)
console.log("Test number "+ key)
var classes = []
var explanation = []
var test1 = {
'original': test['input_original'],
'filtered':test['input_normalized'],
'filtered':test['input_modified'],
}
_.each(modes, function(mode, key, list){
var train_keyphrase_buffer = []
if (classes.length == 0)
_.each(train_turns, function(train, key, list){
console.log("Train number "+ key)
_.each(train['intent_absolute'], function(keyphrase, intent, list){
if (train_keyphrase_buffer.indexOf(keyphrase) == -1)
{
train_keyphrase_buffer.push(keyphrase)
var train1 = {
'original': train['input_original'],
'filtered':train['input_normalized'],
'modified':train['input_modified'],
'keyphrase':[keyphrase],
'intent': intent
}
var results = mode(test1, train1)
console.log(JSON.stringify(results, null, 4))
if (results['classes'].length > 0)
if (modess.permit(results, test))
{
classes = classes.concat(results['classes'])
if (results['classes'].length > 0)
{
results['explanation']['reason'] = results['reason']
results['explanation']['turn'] = train
results['explanation']['classes'] = results['classes']
explanation.push(results['explanation'])
}
}
}
}, this)
}, this)
}, this)
// after the run of all modes
if (classes.length == 0)
if (modess.isOK(test1))
classes.push('Accept')
if (classes.length == 0)
if (modess.isNO(test1))
classes.push('Reject')
if (classes.length == 0)
if (modess.onlyOffer(test))
classes.push('Offer')
test_turns[key]['classified'] = _.unique(classes)
test_turns[key]['results'] = stats.addCasesHash(test['output'], _.unique(classes), true)
stats.addCasesLabels(test['output'], _.unique(classes), true)
test_turns[key]['explanation'] = explanation
}, this)
utils.recursionredis(seeds, [1], function(err,actual) {
console.log("number of PPDB paraphrases " + actual.length)
utils.clusteration(_.unique(actual), function(err, clusters){
fiber.run(clusters)
})
})
// collect only offer keyphrases
var keyph = []
_.each(turns, function(value, key, list){
if ('intent_keyphrases_rule' in value)
if ('Offer' in value['intent_keyphrases_rule'])
{
var phrase = value['intent_keyphrases_rule']['Offer']
phrase = phrase.replace("<ATTRIBUTE>","")
phrase = phrase.replace("<VALUE>","")
if (phrase != 'DEFAULT INTENT')
keyph.push(phrase)
}
}, this)
keyph = _.compact(_.unique(keyph))
var tocomp = utils.crosslist(keyph)
// filtering the gold standard keyphrases that are equal according to the comparison scheme
var f = Fiber(function() {
var fiber = Fiber.current;
async.mapSeries(tocomp, utils.compare, function(err,results){
fiber.run(results);
})
var results = Fiber.yield();
_.each(results, function(value, key, list){
if (value[4] == 1)
keyph = _.without(keyph, value[1])
_.each(sortedClasses, function (classe) {
if (current_cluster !== classe.cluster) {
if (current_cluster !== "") {
txt += "\n } \n";
}
current_cluster = classe.cluster;
txt += 'subgraph ' + classe.cluster + ' {\n style="filled"; color=black;fillcolor=lightgray;fontsize=30; label = "' + classe.name + '"; \n color=lightgrey; \n node[shape=record,style=filled,fillcolor=gray95] \n';
//shape="folder";
cluster++;
}
let methods = [];
_.each(classe.methods, function (method) {
methods.push(method + "\\l");
});
methods = _.unique(methods).sort();
let properties = [];
_.each(classe.properties, function (property) {
properties.push(property + "\\l");
});
properties = _.unique(properties).sort();
if (classe.type === "css") {
}
else if (classe.is_global === true) {
classe.type = "Global";
classe.name = "";
}
let title = "";
if (!_.isUndefined(classe.type)) {
title = " \«" + classe.type.toUpperCase() + "»\\n "
}
title += classe.name;
let propsep = "";
if (properties.length > 0) {
propsep = '|';
}
let methodsep = "";
if (methods.length > 0) {
methodsep = '|';
}
txt += classe.key + '[label = "{' + title + propsep + properties.join("") + methodsep + methods.join("") + '}", fillcolor=' + classe.fill + ']\n ';
});
async.mapSeries(seeds, wordnetquickfetch, function(err, result) {
callback(null, _.unique(_.flatten(result)))
})
utils.wordnetsynonyms(['offer'], function(err,actual){
console.log("number of Wordnet paraphases "+ actual.length)
utils.clusteration(_.unique(actual), function(err, clusters){
fiber.run(clusters)
})
})