this.archive({},function(err, articles){
var smushed = _.map(articles, function(article){
return _.flatten(article.tags);
});
var uniqued = _.uniq(smushed);
next(null,uniqued);
});
exports.buildTrees(deps, cfg, opt, function (err, local, updated) {
if (err) {
return callback(err);
}
var all_names = _.uniq(_.keys(local).concat(_.keys(updated)));
var changed = all_names.map(function (name) {
var lversion = local[name] ? local[name].current_version: null;
var uversion = updated[name] ? updated[name].current_version: null;
if (lversion) {
var lpkg = local[name].versions[lversion];
if (lpkg.source === 'repository') {
// cannot even satisfy requirements with current package,
// this needs re-installing from repositories
return {
name: name,
version: uversion,
old: 'not satisfiable'
};
}
}
if (!local[name] && updated[name] || lversion !== uversion) {
return {name: name, version: uversion, old: lversion};
}
});
callback(null, _.compact(changed), local, updated);
});
exports.readCookie = function () {
var value = cookies.readBrowserCookie('_dashboard_urls');
if (value) {
var durls = JSON.parse(unescape(value));
exports._dashboard_urls = _.uniq(exports._dashboard_urls.concat(durls));
}
};
module.exports = function (config, callback) {
if (!intf) {
return callback(false);
}
function addIp(ip, callback) {
con.info('Adding local IP ' + ip + ' for forwards; if asked for password, give your local (sudo) password.');
var ifconfig = pspawn('sudo', ['ifconfig', intf, 'add', ip]);
ifconfig.on('exit', callback);
}
function addMissingIps(exitCode) {
if (exitCode === 0 && missing.length > 0) {
addIp(missing.shift(), addMissingIps);
} else {
callback(exitCode === 0);
}
}
var allForwards = _.flatten(_.values(config.forwards).concat(_.values(config.localForwards)));
var ips = [];
allForwards.forEach(function (f) {
var m = f.from.match(/^([0-9.]+):/);
ips.push(m[1]);
});
ips = _.uniq(ips);
var currentIps = _.pluck(os.networkInterfaces()[intf], 'address');
var missing = _.difference(ips, currentIps);
// Add any missing IP:s and finally call the callback.
addMissingIps(0);
};
exports.discussion = function (head, req) {
start({code: 200, headers: {'Content-Type': 'text/html'}});
var row, rows = [];
while (row = getRow()) {
rows.push(row);
}
var comments = _.filter(rows, function (r) {
return r.doc.type === 'comment';
});
var doc_row = _.detect(rows, function (r) {
return r.doc.type === 'page';
});
var empty_doc = !(doc_row);
var doc = doc_row ? doc_row.doc: {};
comments = _.map(comments, function (c) {
c.doc.type = 'comment';
c.doc.pptime = utils.prettyTime(c.doc.time);
return c.doc;
});
// add edit events to comments
if (doc && doc.history) {
_.each(doc.history, function (h) {
h.type = 'edit';
h.is_edit = true;
h.pptime = utils.prettyTime(h.time);
comments.push(h);
});
}
comments = _.sortBy(comments, function (c) {
return c.time;
});
var participants = comments.map(function (c) {
return c.user;
});
participants = _.uniq(participants);
if (req.query.page && req.query.page[0] === '_') {
return shows.not_found(null, req);
}
events.once('afterResponse', discussion_ui.bindDiscussion);
return {
title: doc.title || utils.idToTitle(req.query.page),
content: templates.render('discussion.html', req, {
doc: doc,
page_title: doc.title || utils.idToTitle(req.query.page),
page_subtitle: doc.subtitle,
pageid: doc._id || req.query.page,
participants: participants,
comments: comments,
current_tab: "discussion_page",
logged_in: !!(req.userCtx.name),
empty_doc: empty_doc
})
};
};
it("Split word arrays into graphemes", function() {
var cumulativeWords = ['one', 'two', 'three'];
var focusWords = ['four', 'five', 'six'];
var graphemes = _.uniq(_.union(cumulativeWords, focusWords).join('').split(''));
console.log(graphemes);
expect(graphemes.length).toBe(13);
});
getLeaguesWithLiveGames: function() {
var leagueIds = _.map(this.liveGames, _.bind(function(game) {
return game.league_id;
}, this));
return _.uniq(leagueIds);
},
_.intersection = _.intersect = function(array) {
var rest = slice.call(arguments, 1);
return _.filter(_.uniq(array), function(item) {
return _.every(rest, function(other) {
return _.indexOf(other, item) >= 0;
});
});
};
function prepText(text) {
if (_.isArray(text)) return text;
var deduped = _.uniq(tokenizer.tokenize(text));
if (!this.options.stopwords) return deduped;
return _.reject(deduped, _.bind(isStopword, null,
_.isString(this.options.stopwords) ? this.options.stopwords : natural_stopwords
));
}
registerScriptReceived: function(playerID) {
'use strict';
this.receivedTheScript.push(playerID);
if (_.uniq(this.receivedTheScript).length >= this.numberOfPlayers) {
//all players have received the script, create next turn
if (this.battleModel.winner !== undefined) {
battles.finish(this);
} else {
this.nextTurn();
}
}
},
neodb.query(query, {}, function (err, results) {
if (err) request.reply(err);
else {
var summary = results.map(function (result) {
return {
movie: JSON.parse(result['movie']['_data']['data'].infos),
person: JSON.parse(result['person']['_data']['data'].infos),
department: result['relation']['_data']['data'].department
}
});
request.reply(_.uniq(summary));
}
});
function tokenizedataset(dataset)
{
vocabulary = []
for (var sample in dataset)
{
if (dataset[sample].length!=0)
{
var words = tokenizer.tokenize(normalizer(dataset[sample]['input']));
vocabulary = vocabulary.concat(words);
}
}
return _.uniq(vocabulary);
}
function partCallback(result) {
if (result) {
results = _.uniq(results.concat(result)); // make sure it's unique!
}
if (results.length < count && doParts.length) {
return tryPart();
}
// final random and trim excess
results = _.sample(results, count);
done();
}
exports.validate = function (fields, doc, values, raw, path, extra) {
values = values || {};
fields = fields || {};
raw = raw || {};
// Expecting sub-object, not a value
if (typeof values !== 'object') {
var e = new Error('Unexpected property - validation 1');
e.field = path;
e.has_field = false;
return [e];
}
// Ensure we walk through all paths of both fields and values by combining
// the keys of both. Otherwise, we might miss out checking for missing
// required fields, or may not detect the presence of extra fields.
var keys = _.uniq(_.keys(fields).concat(_.keys(values)));
var fields_module = require('./fields');
return _.reduce(keys, function (errs, k) {
var f = fields[k];
if (f === undefined) {
// Extra value with no associated field detected
if (!extra) {
// ignore system properties
if (!(path.length === 0 && k.charAt(0) === '_')) {
var e = new Error('Unexpected property - validation 2');
e.field = path.concat([k]);
e.has_field = false;
errs.push(e);
}
}
return errs;
}
var val = values[k];
var fn = exports.validate;
if (f instanceof fields_module.Field ||
f instanceof fields_module.Embedded ||
f instanceof fields_module.EmbeddedList) {
fn = exports.validateField;
}
if (f instanceof fields_module.AttachmentField) {
val = utils.attachmentsBelowPath(doc, path.concat([k]));
}
return errs.concat(
fn.call(this, f, doc, val, raw[k], path.concat([k]), extra)
);
}, []);
};
ChatRoom.prototype.user_connect = function (user) {
var index = _.sortedIndex(users, user, iterator('fn', 'ln')),
u_last_active = (this.last_active = (new Date).getTime()) / 1000;
console.log('index is ' + index);
if (sids[aid]) {
// if user is already currently connected, i.e. opens chat in a new tab
users[index]['la'] = u_last_active;
users[index]['si'].push(user['si']);
_.uniq(users[index]['si']);
}
user['la'] = u_last_active;
user['si'] = [user[si]];
users.splice(index, 0, user);
};
function generateGraphData(dependencies) {
// Read data
var components = _.uniq(_.flatten(_.map(dependencies, function(values, item) {
var data = [];
data.push(item);
data = data.concat(values);
return data;
})));
// Mapped nodes
var nodes = _.map(components, function(component) {
return {
id: component
}
});
// Figure out links
var links = [];
_.each(dependencies, function(outerDependency, component, index) {
_.each(outerDependency, function(dependency) {
var sourceIndex = _.indexOf(components, dependency);
var targetIndex = _.indexOf(components, component);
var link = {
source: _.indexOf(components, component),
target: _.indexOf(components, dependency),
};
if(sourceIndex > -1 && targetIndex > -1) {
links.push(link);
}
})
});
return {
"directed" : true,
"multigraph" : false,
"graph" : [],
"nodes" : nodes,
"links" : links
}
};
EmbeddedList.prototype.authorize = function (nDoc, oDoc, nVal, oVal, user) {
var type = this.type;
var perms = this.permissions;
nVal = nVal || [];
oVal = oVal || [];
// a unique list of embedded ids from both the old and new document
var ids = _.uniq(_.pluck(nVal, '_id').concat(_.pluck(oVal, '_id')));
return _.reduce(ids, function (errs, id, i) {
var curr_errs = [];
var nd = _.detect(nVal, function (v) {
return v && v._id === id;
});
nd = nd || {_deleted: true};
var od = _.detect(oVal, function (v) {
return v && v._id === id;
});
var args = [nDoc, oDoc, nd, od, user];
if (_.isFunction(perms)) {
curr_errs = utils.getErrors(perms, args);
}
var fn;
// on add
if (nd && !od) {
fn = perms.add;
}
// on remove
else if (nd._deleted) {
fn = perms.remove;
}
// on update
else if (JSON.stringify(nd) !== JSON.stringify(od)) {
fn = perms.update;
}
if (fn) {
curr_errs = curr_errs.concat(utils.getErrors(fn, args));
}
curr_errs = _.map(curr_errs, function (e) {
e.field = [i].concat(e.field || []);
return e;
});
return errs.concat(curr_errs);
}, []);
};
getVerifiedLinksObject: function () {
var self = this,
rtnObj = {verified:[],unverified:[]};
// sort the links
self.links.filter(function (link) {
if (self.grapher.verifiedLink(link)) {
rtnObj.verified.push(link);
} else {
rtnObj.unverified.push(link);
}
});
// remove duplicates
rtnObj.verified = _.uniq(rtnObj.verified);
rtnObj.unverified = _.uniq(rtnObj.unverified);
if (self.grapher.options.stripDeeperLinks) {
rtnObj.verified = self.removeDeeperLinks(rtnObj.verified);
rtnObj.unverified = self.removeDeeperLinks(rtnObj.unverified);
}
return rtnObj;
}
exports.buildDocLevelSchema = function(data) {
var tsets = {}; //: root
var tsetName = data[0].document;
tsets['TS_' + tsetName] = { name: tsetName, fullName: 'TS_' + tsetName, collection: {} };
var tables = _.uniq(_.pluck(data, 'doc_table'));
_.each(tables, function(value) {
var tableName = 'Table_' + value;
var curTable = { name: tableName, fullName: tableName, collection: {} };
var curTableSegments = _.filter(data, function(segment) {
return segment.doc_table == value;
}, this);
this.buildTableLevelSchema(curTable, curTableSegments);
tsets['TS_' + tsetName].collection[tableName] = curTable;
}, this);
return tsets;
};
lists.intersection = function (head, req) {
var extraKeys = [];
if (req.query.key) {
extraKeys.push(req.query.key);
}
if (req.query.extra_keys) {
var split = req.query.extra_keys.split(' ');
extraKeys = extraKeys.concat(split);
}
extraKeys = _.uniq(_.map(extraKeys, function(key) {return key.toLowerCase()}));
var realJson = true;
if (req.query.streamJson) {
realJson = false;
}
start({'headers' : {'Content-Type' : 'application/json'}});
if (realJson) send('[\n');
var count = 0;
var row;
while ((row = getRow())) {
var doc_intersection = _.intersection(row.doc.tag, extraKeys);
if (doc_intersection.length == extraKeys.length) {
var res = {
id : row.id,
name : row.doc.name,
size : row.doc.size,
s : row.doc.s,
l : row.doc.l,
m : row.doc.m
}
var pre = '';
if (count++ > 0 && realJson) pre = ',';
send(pre + JSON.stringify(res) + '\n');
}
}
if (realJson) send(']');
}
mailProvider.findAll(query, function(err, mails) {
if (err) throw err;
var subtags = _.uniq(mails, false, function(mail) {
return mail.subtag;
});
mails = _.sortBy(mails, function(mail) {
return mail.subtag[0];
});
res.render('index', {
filterId: filterId,
title: 'メール',
mails: mails,
counts: mails.length,
util: util,
subtags: subtags,
filters: filters
});
});
_(3).times(function(n){
var buf = []
_.each(json.map(Compensate), function(value, key, list){
if (value.length > n)
{
if (_.compact(value[n].toString()).length != 0)
buf = buf.concat(value[n])
}
})
// buf = _.uniq(_.compact(buf))
// buf = _.uniq(_.compact(buf))
// if (buf.length != 0) label[n] = buf
buf = _.uniq(buf)
if ((buf.length > 0) && (typeof(buf[0])!="undefined"))
label[n] = buf
if ((typeof(buf[0])=="undefined"))
label[n] = []
})
LibSynphony.prototype.buildIndexDataForWord = function(wordList, i) {
//creates a unique array of all gpcs in a word
var temp = _.clone(wordList[i].GPCForm);
var syll = wordList[i]["Syllables"];
var u = _.uniq(temp);
var gpc;
wordList[i].GPCS = u;
wordList[i].GPCcount = u.length;
//creates a reverse form of the word's gpcs
wordList[i].Reverse = temp.reverse().join("");
if (wordList[i].GPCS !== undefined) {
//creates arrays grouped by gpc and syllable length
for (var j = 0, jlen = u.length; j < jlen; j++) {
gpc = u[j].toLowerCase();
if (!wordList[gpc + "__" + syll]) {
wordList[gpc + "__" + syll] = [];
}
wordList[gpc + "__" + syll].push(wordList[i]);
}
}
};
cleanWords: function cleanWords(wordList) {
var letters = Ruzzle.letters;
var limit = Ruzzle.WORDS_NUMBER_LIMIT;
var lettersInBoard = [];
var word = '';
var words = [];
for (var y = 0; y < letters.length; y++) {
for (var x = 0; x < letters[y].length; x++) {
lettersInBoard.push(letters[y][x]);
}
}
lettersInBoard = _.uniq(lettersInBoard);
var regexpLetters = new RegExp('[' + lettersInBoard.join('') + ']');
for (var i = 0, count = wordList.length; i < count && i < limit; i++) {
word = wordList[i];
if (regexpLetters.test(word)) {
words.push(word);
}
}
return words;
},
LibSynphony.prototype.getUniqueWordsFromHtmlString = function(textHTML) {
return _.uniq(this.getWordsFromHtmlString(textHTML));
};
exports.add = function (url) {
exports._dashboard_urls.unshift(url);
exports._dashboard_urls = _.uniq(exports._dashboard_urls);
exports.updateCookie();
};
exports.getURLs = function () {
return _.uniq(exports._dashboard_urls);
};
LibSynphony.prototype.checkStory = function(
aFocusWordList,
aWordCumulativeList,
aGPCsKnown,
storyHTML,
sightWords
) {
var letters;
var story_vocab;
if (aGPCsKnown.length > 0) {
letters = this.fullGPC2Regular(aGPCsKnown).join("|");
// break the text into words
story_vocab = this.getWordsFromHtmlString(storyHTML, letters);
} else {
letters = "";
// break the text into words
story_vocab = this.getWordsFromHtmlString(storyHTML);
}
// Just in case users want ? for glottal or similar nonsense with other special
// RE characters. See https://issues.bloomlibrary.org/youtrack/issue/BL-7075.
if (letters.includes("\\")) letters = letters.replace("\\", "\\\\");
if (letters.includes("?")) letters = letters.replace("?", "\\?");
if (letters.includes("+")) letters = letters.replace("+", "\\+");
if (letters.includes("*")) letters = letters.replace("*", "\\*");
// get unique word list
var story_vocab_compacted = _.uniq(story_vocab);
// count total words in the story
var total_words = _.filter(story_vocab, function(word) {
return isNaN(word) === true;
}).length;
// if aGPCsKnown is empty, return now
// BL-2359: Need to allow stages based on word lists rather than known graphemes
//if (aGPCsKnown.length === 0)
// return new StoryCheckResults([], [], [], [], [], 0, total_words);
// first we do diffs on aFocusWordList and aWordCumulativeList with story_vocab words
var story_focus_words = _.intersection(
aFocusWordList,
story_vocab_compacted
);
var story_cumulative_words = _.intersection(
_.pluck(aWordCumulativeList, "Name"),
story_vocab
);
this.array_sort_length(story_focus_words);
/* TODO: has to handle utf8 */
// FIRST PASS: we handle words which are currently in focus
var focus_words = _.intersection(story_focus_words, story_vocab_compacted);
var remaining_words = _.difference(story_vocab_compacted, focus_words);
this.array_sort_length(focus_words);
// SECOND PASS: we handle words which are part of the cumulative word bank
// aWordCumulativeList is an object that contains the following fields:
// GPCForm,GPCS,GPCcount,Name,Reverse,SyllShape,Syllables
var cumulative_words = _.intersection(
story_cumulative_words,
remaining_words
);
remaining_words = _.difference(remaining_words, cumulative_words);
this.array_sort_length(cumulative_words);
// THIRD PASS: we handle words which have not been matched yet to check if they are
// decodable at this point. This can match words which are longer than the syllable
// selectors specify but contain all the gpcs. We do this using a regular expression
// with the array of knownGPCs. This is not the most accurate method; we should
// first segment the word with all gpcs, then test with known gpcs. This also checks
// for the possibility that the word is not yet in our database.
// This only works for simple gpc notation, not complex.
// Why not for full gpc? Once you have covered the regular spelling patterns (in English)
// you will match all the other words, so everything gets tagged as 'possible'. Not useful!!
var possible_words = [];
if (
typeof theOneLanguageDataInstance === "undefined" ||
theOneLanguageDataInstance["UseFullGPCNotation"] === false
) {
// allow punctuation characters in the words
// BL-1216 Use negative look-ahead to keep letters from being counted as punctuation
// even if Unicode says something is a punctuation character when the user
// has specified it as a letter (like single quote).
var re = new XRegExp(
"^((" +
letters +
")+((?![" +
letters +
"])[\\p{P}]*(" +
letters +
")*)*)$",
"gi"
);
possible_words = _.filter(remaining_words, function(word) {
return word.match(re);
});
// BL-1217: exclude words with unknown graphemes, specifically 'aa' when only 'a' is known
if (typeof theOneLanguageDataInstance !== "undefined") {
// get the unknown letters
var unknownGPCs = _.difference(
_.pluck(theOneLanguageDataInstance["GPCS"], "GPC"),
letters.split("|")
); // .join('|');
if (Array.isArray(unknownGPCs) && unknownGPCs.length > 0) {
// remove from the list of unknownGPCs characters used to build multi-graphs in the list aGPCsKnown
unknownGPCs = _.filter(unknownGPCs, function(gpc) {
return letters.indexOf(gpc) === -1;
});
if (unknownGPCs.length > 0) {
re = new XRegExp("(" + unknownGPCs.join("|") + ")+", "gi");
possible_words = _.filter(possible_words, function(word) {
return !word.match(re);
});
}
}
}
remaining_words = _.difference(remaining_words, possible_words);
this.array_sort_length(possible_words);
}
// FOURTH PASS: we handle sight words
// NOTE: Handle sight words after checking for decodability, because a sight word may become decodable.
var sight_words = [];
if (sightWords.length > 0) {
sight_words = _.intersection(sightWords.split(" "), remaining_words);
remaining_words = _.difference(remaining_words, sight_words);
this.array_sort_length(sight_words);
}
// FIFTH PASS: we handle everything else that's left over
var readable =
focus_words.length + cumulative_words.length + possible_words.length;
return new StoryCheckResults(
focus_words,
cumulative_words,
possible_words,
sight_words,
remaining_words,
readable,
total_words
);
};
LibSynphony.prototype.selectGPCWordsWithArrayCompare = function(
aDesiredGPCs,
aKnownGPCs,
restrictToKnownGPCs,
allowUpperCase,
aSyllableLengths,
aSelectedGroups,
aPartsOfSpeech
) {
var word_already_exists,
aSelectedWordObjects,
aWordObjects,
aVocabKey,
aCriteria;
var groups = this.chooseVocabGroups(aSelectedGroups);
aWordObjects = [];
aSelectedWordObjects = [];
aVocabKey = this.constructSourceArrayNames(aDesiredGPCs, aSyllableLengths); //result: "a__1" or "a_a__1" or "wicc_a_a__1"
aCriteria = aKnownGPCs;
//let's concatenate all vocabulary into 1 array
for (var g = 0, len = groups.length; g < len; g++) {
//eg: group1, group2...
for (var i = 0, ilen = aVocabKey.length; i < ilen; i++) {
//eg: ["a_a__1"]
if (groups[g][aVocabKey[i]]) {
//make sure it exists
aWordObjects = aWordObjects.concat(groups[g][aVocabKey[i]]);
}
}
}
//this is the place to branch into checking for taught graphemes vs
//selecting all words that have the current grapheme.
if (!restrictToKnownGPCs) {
//select all words that have the current_gpc
aSelectedWordObjects = _.uniq(aWordObjects);
} else {
//we start restricting the word list using criteria that include
//known graphemes, part of speech, etc.
//add uppercase gpcs to aCriteria***
if (allowUpperCase) {
//if true then we add uppercase
for (var k = 0, klen = aKnownGPCs.length; k < klen; k++) {
var temp = [];
for (
var j = 0, jlen = theOneLanguageDataInstance.GPCS.length;
j < jlen;
j++
) {
if (
theOneLanguageDataInstance.GPCS[j]["GPC"] ===
aKnownGPCs[k]
) {
if (
theOneLanguageDataInstance.GPCS[j]["GPCuc"] !== ""
) {
temp.push(
theOneLanguageDataInstance.GPCS[j]["GPCuc"]
);
}
}
}
aCriteria = aCriteria.concat(temp);
}
}
//lets add symbols that can be matched at any time
//these can come from the AlwaysMatch, SyllableBreak, StressSymbol, or MorphemeBreak fields
//anything else must exist in the aKnownGPCs in order to be accepted
if (alwaysMatch.length > 0) {
aCriteria = aCriteria.concat(alwaysMatch);
} else {
if (
typeof theOneLanguageDataInstance["AlwaysMatch"] !==
"undefined" &&
theOneLanguageDataInstance["AlwaysMatch"] !== ""
) {
alwaysMatch = alwaysMatch.concat(
theOneLanguageDataInstance["AlwaysMatch"]
);
}
if (
typeof theOneLanguageDataInstance["SyllableBreak"] !==
"undefined" &&
theOneLanguageDataInstance["SyllableBreak"] !== ""
) {
alwaysMatch.push(theOneLanguageDataInstance["SyllableBreak"]);
}
if (
typeof theOneLanguageDataInstance["StressSymbol"] !==
"undefined" &&
theOneLanguageDataInstance["StressSymbol"] !== ""
) {
alwaysMatch.push(theOneLanguageDataInstance["StressSymbol"]);
}
if (
typeof theOneLanguageDataInstance["MorphemeBreak"] !==
"undefined" &&
theOneLanguageDataInstance["MorphemeBreak"] !== ""
) {
alwaysMatch.push(theOneLanguageDataInstance["MorphemeBreak"]);
}
aCriteria = aCriteria.concat(alwaysMatch);
}
//start checking words
for (var w = 0, wlen = aWordObjects.length; w < wlen; w++) {
var keep = true;
//first we check for allowable gpcs
var gpcform = aWordObjects[w]["GPCForm"];
var test_word = _.difference(gpcform, aCriteria);
if (test_word.length > 0) {
keep = false;
}
//then we check for part of speech constraint
var ps_check = false;
if (aPartsOfSpeech.length > 0) {
if (aWordObjects[w]["PartOfSpeech"]) {
for (
var p = 0, plen = aPartsOfSpeech.length;
p < plen;
p++
) {
if (
aWordObjects[w]["PartOfSpeech"] ===
aPartsOfSpeech[p]
) {
ps_check = true;
}
}
}
if (ps_check === false) keep = false;
}
//if keep is still true, then this word object
//has passed all checks and is suitable for use
if (keep === true) {
word_already_exists = false;
for (
var m = 0, mlen = aSelectedWordObjects.length;
m < mlen;
m++
) {
//check to see that we don't add more than one instance of the word to our list
if (aSelectedWordObjects[m] === aWordObjects[w]) {
word_already_exists = true;
}
}
if (word_already_exists === false) {
aSelectedWordObjects.push(aWordObjects[w]);
}
}
} //end of wordObject loop
}
return aSelectedWordObjects;
};
var callback=function(){
console.log("ex")
console.log(retrieved.length)
retrieved = _.uniq(retrieved)
console.log(retrieved.length)
console.log(retrieved)
var TPlist = []
var TPlistret = []
var flag = 0
var TP = 0
var FP = 0
var rightpairs = []
var wrongpairs = []
_.each(retrieved, function(value, key, list){
var flag = 0
var comparison = value.replace("you","")
comparison = comparison.replace("we","")
_.each(Object.keys(stats['Offer']), function(gold, key, list){
if (natural.JaroWinklerDistance(comparison, gold)>Distance_threshold)
{
// if (!(gold in TPlist))
// {
// TP = TP + 1
TPlist.push(gold)
TPlistret.push(value)
flag = 1
// rightpairs.push([value,gold])
// }
}
}, this)
if (flag == 0)
{
FP = FP + 1
wrongpairs.push(value)
}
}, this)
var FN = Object.keys(stats['Offer']).length - _.unique(TPlist).length
TPlistret = _.unique(TPlistret)
TP = TPlistret.length
var Precision = TP/(TP+FP)
var Recall = TP/(TP+FN)
var F1 = 2*Precision*Recall/(Recall+Precision)
console.log("TP"+TP)
console.log("FP"+FP)
console.log("FN"+FN)
console.log("TP list")
console.log(TPlistret)
console.log("FP list")
console.log(wrongpairs)
console.log('Precision '+Precision)
console.log('Recall '+Recall)
console.log('F1 '+F1)
process.exit(0)
}