it("should populate nmf tags for the nb tags", () => {
item = instance.naiveBayesTag(item, {fields: ["text"]});
item = instance.conditionallyNmfTag(item, {});
assert.isTrue(("nb_tags" in item));
assert.deepEqual(item.nmf_tags, {
tag2: {
tag21: 0.8,
tag22: 0.7,
tag23: 0.6,
},
tag3: {
tag31: 0.7,
tag32: 0.6,
tag33: 0.5,
},
});
assert.deepEqual(item.nmf_tags_parent, {
tag21: "tag2",
tag22: "tag2",
tag23: "tag2",
tag31: "tag3",
tag32: "tag3",
tag33: "tag3",
});
});
it("should not populate nmf tags for things that were not nb tagged", () => {
item = instance.naiveBayesTag(item, {fields: ["text"]});
item = instance.conditionallyNmfTag(item, {});
assert.isTrue(("nmf_tags" in item));
assert.isTrue(!("tag4" in item.nmf_tags));
assert.isTrue(("nmf_tags_parent" in item));
assert.isTrue(!("tag4" in item.nmf_tags_parent));
});
it("should skip items with missing fields", () => {
assert.isTrue(instance.acceptItemByFieldValue(item,
{field: "no-left", op: "==", rhsValue: 1}) === null);
assert.isTrue(instance.acceptItemByFieldValue(item,
{field: "lhs", op: "==", rhsField: "no-right"}) === null);
assert.isTrue(instance.acceptItemByFieldValue(item,
{field: "lhs", op: "=="}) === null);
});
it("should implement >= ", () => {
assert.isTrue(instance.acceptItemByFieldValue(item,
{field: "lhs", op: ">=", rhsValue: 1}) !== null);
assert.isTrue(instance.acceptItemByFieldValue(item,
{field: "lhs", op: ">=", rhsValue: 2}) !== null);
assert.isTrue(instance.acceptItemByFieldValue(item,
{field: "lhs", op: ">=", rhsValue: 3}) === null);
});
it("should understand NaiveBayesTextTagger", () => {
item = instance.naiveBayesTag(item, {fields: ["text"]});
assert.isTrue("nb_tags" in item);
assert.isTrue(!("tag1" in item.nb_tags));
assert.equal(item.nb_tags.tag2, 0.86);
assert.equal(item.nb_tags.tag3, 0.90);
assert.equal(item.nb_tags.tag5, 0.90);
assert.isTrue("nb_tokens" in item);
assert.deepEqual(item.nb_tokens, ["this", "is", "a", "sentence"]);
assert.isTrue("nb_tags_extended" in item);
assert.isTrue(!("tag1" in item.nb_tags_extended));
assert.deepEqual(item.nb_tags_extended.tag2, {
label: "tag2",
logProb: Math.log(0.86),
confident: true,
});
assert.deepEqual(item.nb_tags_extended.tag3, {
label: "tag3",
logProb: Math.log(0.90),
confident: true,
});
assert.deepEqual(item.nb_tags_extended.tag5, {
label: "tag5",
logProb: Math.log(0.90),
confident: true,
});
assert.isTrue("nb_tokens" in item);
assert.deepEqual(item.nb_tokens, ["this", "is", "a", "sentence"]);
});
it("should handle long map to short map", () => {
let right = makeItem();
right.map = {a: 5, b: -1, c: 3};
item.map.d = 999;
let combined = instance.combinerMax(item, right, {field: "map"});
assert.deepEqual(combined.map, {a: 5, b: 2, c: 3, d: 999});
});
it("should handle long array to short array", () => {
let right = makeItem();
right.arr1 = [5, 1, 4];
item.arr1.push(7);
let combined = instance.combinerMax(item, right, {field: "arr1"});
assert.deepEqual(combined.arr1, [5, 3, 4, 7]);
});
it("should count when missing right", () => {
let right = makeItem();
item.combined_map = {"fake": 42};
let combined = instance.combinerCollectValues(item, right,
{left_field: "combined_map", right_key_field: "url_domain", right_value_field: "time", operation: "count"});
assert.deepEqual(combined.combined_map, {"fake": 42});
});
it("should handle erroring steps", () => {
let final = instance.executeRecipe({}, [
{function: "set_default", field: "foo", value: 1},
{function: "accept_item_by_field_value", field: "missing", op: "invalid", rhsField: "moot", rhsValue: "m00t"},
{function: "set_default", field: "bar", value: 10},
]);
assert.equal(final, null);
});
it("should handle unknown steps", () => {
let final = instance.executeRecipe({}, [
{function: "set_default", field: "foo", value: 1},
{function: "missing"},
{function: "set_default", field: "bar", value: 10},
]);
assert.equal(final, null);
});
it("should handle working steps", () => {
let final = instance.executeRecipe({}, [
{function: "set_default", field: "foo", value: 1},
{function: "set_default", field: "bar", value: 10},
]);
assert.equal(final.foo, 1);
assert.equal(final.bar, 10);
});
it("should count when missing left", () => {
let right = makeItem();
right.url_domain = "maseratiusa.com/maserati";
right.time = 41;
let combined = instance.combinerCollectValues(item, right,
{left_field: "combined_map", right_key_field: "url_domain", right_value_field: "time", operation: "count"});
assert.deepEqual(combined.combined_map, {"maseratiusa.com/maserati": 1});
});
it("should copy values", () => {
item = instance.copyValue(item, {src: "one", dest: "again"});
assert.isTrue("again" in item);
assert.equal(item.again, 1);
item.one = 100;
assert.equal(item.one, 100);
assert.equal(item.again, 1);
});
it("should error on bogus operation", () => {
let right = makeItem();
right.url_domain = "maseratiusa.com/maserati";
right.time = 41;
let combined = instance.combinerCollectValues(item, right,
{left_field: "combined_map", right_key_field: "url_domain", right_value_field: "time", operation: "missing"});
assert.equal(combined, null);
});
it("should handle maps corrects", () => {
item = instance.copyValue(item, {src: "map", dest: "again"});
assert.deepEqual(item.again, {a: 1, b: 2, c: 3});
item.map.c = 100;
assert.deepEqual(item.again, {a: 1, b: 2, c: 3});
item.map = 342;
assert.deepEqual(item.again, {a: 1, b: 2, c: 3});
});
it("should scalar multiply a nested map with logrithms", () => {
item = instance.scalarMultiplyTag(item, {field: "tags", k: 3, log_scale: true});
assert.isTrue(Math.abs(item.tags.a.aa - Math.log(0.1 + 0.000001) * 3) <= EPSILON);
assert.isTrue(Math.abs(item.tags.a.ab - Math.log(0.2 + 0.000001) * 3) <= EPSILON);
assert.isTrue(Math.abs(item.tags.a.ac - Math.log(0.3 + 0.000001) * 3) <= EPSILON);
assert.isTrue(Math.abs(item.tags.b.ba - Math.log(4.0 + 0.000001) * 3) <= EPSILON);
assert.isTrue(Math.abs(item.tags.b.bb - Math.log(5.0 + 0.000001) * 3) <= EPSILON);
assert.isTrue(Math.abs(item.tags.b.bc - Math.log(6.0 + 0.000001) * 3) <= EPSILON);
});
it("should scalar multiply a nested map", () => {
item = instance.scalarMultiplyTag(item, {field: "tags", k: 3, log_scale: false});
assert.isTrue(Math.abs(item.tags.a.aa - 0.3) <= EPSILON);
assert.isTrue(Math.abs(item.tags.a.ab - 0.6) <= EPSILON);
assert.isTrue(Math.abs(item.tags.a.ac - 0.9) <= EPSILON);
assert.isTrue(Math.abs(item.tags.b.ba - 12) <= EPSILON);
assert.isTrue(Math.abs(item.tags.b.bb - 15) <= EPSILON);
assert.isTrue(Math.abs(item.tags.b.bc - 18) <= EPSILON);
});
it("should still keep the 2 largest", () => {
item = instance.keepTopK(item, {field: "map", k: 2});
assert.equal(Object.keys(item.map).length, 2);
assert.isTrue(!("a" in item.map));
assert.isTrue("b" in item.map);
assert.equal(item.map.b, 2);
assert.isTrue("c" in item.map);
assert.equal(item.map.c, 3);
});
it("should keep the 2 smallest", () => {
item = instance.keepTopK(item, {field: "map", k: 2, descending: false});
assert.equal(Object.keys(item.map).length, 2);
assert.isTrue("a" in item.map);
assert.equal(item.map.a, 1);
assert.isTrue("b" in item.map);
assert.equal(item.map.b, 2);
assert.isTrue(!("c" in item.map));
});
it("should overwrite when both", () => {
let right = makeItem();
right.url_domain = "maseratiusa.com/maserati";
right.time = 41;
item.combined_map = {"fake": 42, "maseratiusa.com/maserati": 77};
let combined = instance.combinerCollectValues(item, right,
{left_field: "combined_map", right_key_field: "url_domain", right_value_field: "time", operation: "overwrite"});
assert.deepEqual(combined.combined_map, {"fake": 42, "maseratiusa.com/maserati": 41});
});
it("should normalize a map to sum to 1", () => {
item = instance.probNormalize(item, {field: "map"});
assert.equal(Object.keys(item.map).length, 3);
assert.isTrue("a" in item.map);
assert.isTrue(Math.abs(item.map.a - 0.16667) <= EPSILON);
assert.isTrue("b" in item.map);
assert.isTrue(Math.abs(item.map.b - 0.33333) <= EPSILON);
assert.isTrue("c" in item.map);
assert.isTrue(Math.abs(item.map.c - 0.5) <= EPSILON);
});
it("should handle working steps", () => {
let final = instance.executeCombinerRecipe(
{foo: 1, bar: 10},
{foo: 1, bar: 10},
[
{function: "combiner_add", field: "foo"},
{function: "combiner_add", field: "bar"},
]);
assert.equal(final.foo, 2);
assert.equal(final.bar, 20);
});
it("should handle unknown steps", () => {
let final = instance.executeCombinerRecipe(
{foo: 1, bar: 10},
{foo: 1, bar: 10},
[
{function: "combiner_add", field: "foo"},
{function: "missing"},
{function: "combiner_add", field: "bar"},
]);
assert.equal(final, null);
});
it("should handle erroring steps", () => {
let final = instance.executeCombinerRecipe(
{foo: 1, bar: 10, baz: 0},
{foo: 1, bar: 10, baz: "hundred"},
[
{function: "combiner_add", field: "foo"},
{function: "combiner_add", field: "baz"},
{function: "combiner_add", field: "bar"},
]);
assert.equal(final, null);
});
it("should calculate add vectors from maps", () => {
item = instance.vectorAdd(item, {left: "map", right: "map2"});
assert.equal(Object.keys(item.map).length, 4);
assert.isTrue("a" in item.map);
assert.equal(item.map.a, 1);
assert.isTrue("b" in item.map);
assert.equal(item.map.b, 4);
assert.isTrue("c" in item.map);
assert.equal(item.map.c, 6);
assert.isTrue("d" in item.map);
assert.equal(item.map.d, 4);
});
it("should apply softmax across the subtags", () => {
item = instance.applySoftmaxTags(item, {field: "tags"});
assert.isTrue("a" in item.tags);
assert.isTrue("aa" in item.tags.a);
assert.isTrue("ab" in item.tags.a);
assert.isTrue("ac" in item.tags.a);
assert.isTrue(Math.abs(item.tags.a.aa - 0.30061) <= EPSILON);
assert.isTrue(Math.abs(item.tags.a.ab - 0.33222) <= EPSILON);
assert.isTrue(Math.abs(item.tags.a.ac - 0.36717) <= EPSILON);
assert.isTrue("b" in item.tags);
assert.isTrue("ba" in item.tags.b);
assert.isTrue("bb" in item.tags.b);
assert.isTrue("bc" in item.tags.b);
assert.isTrue(Math.abs(item.tags.b.ba - 0.09003) <= EPSILON);
assert.isTrue(Math.abs(item.tags.b.bb - 0.24473) <= EPSILON);
assert.isTrue(Math.abs(item.tags.b.bc - 0.66524) <= EPSILON);
});
it("should error on mismatch types", () => {
let right = makeItem();
right.lhs = [1, 2, 3];
let combined = instance.combinerMax(item, right, {field: "lhs"});
assert.equal(combined, null);
});
it("should error on left string", () => {
let right = makeItem();
let combined = instance.combinerMax(item, right, {field: "foo"});
assert.equal(combined, null);
});
it("should error on missing left, and right is a string", () => {
let right = makeItem();
right.error = "error";
let combined = instance.combinerMax(item, right, {field: "error"});
assert.equal(combined, null);
});
it("should handle small number", () => {
let right = makeItem();
item.lhs = 99;
let combined = instance.combinerMax(item, right, {field: "lhs"});
assert.equal(combined.lhs, 99);
});