it('compare sizes lean vs not lean', async function() {
const schema = new mongoose.Schema({ name: String });
const MyModel = mongoose.model('Test', schema);
await MyModel.create({ name: 'test' });
// Module that estimates the size of an object in memory
const sizeof = require('object-sizeof');
const normalDoc = await MyModel.findOne();
// To enable the `lean` option for a query, use the `lean()` function.
const leanDoc = await MyModel.findOne().lean();
sizeof(normalDoc); // >= 1000
sizeof(leanDoc); // 86, 10x smaller!
// In case you were wondering, the JSON form of a Mongoose doc is the same
// as the POJO. This additional memory only affects how much memory your
// Node.js process uses, not how much data is sent over the network.
JSON.stringify(normalDoc).length === JSON.stringify(leanDoc.length); // true
// acquit:ignore:start
assert.ok(sizeof(normalDoc) >= 1000);
assert.equal(sizeof(leanDoc), 86);
assert.equal(JSON.stringify(normalDoc).length, JSON.stringify(leanDoc).length);
// acquit:ignore:end
});
client.on("message", function (msg, rinfo) {
dnsTrafficInBytes = dnsTrafficInBytes + msg.length + 28;
var freshDomainIp = parseIpFromDnsResponse(msg);
if(!isDnsResponseSentToClient && freshDomainIp != 0) {
var freshDomainTtl = "00000004";
var dnsPacketInHexWithoutTransactionId = "818000010001" + clientReq.substr(16) + "c00c00010001" + freshDomainTtl + "0004" +freshDomainIp;
var dnsResponseToSend = dnsResponseToSendTransactionId + dnsPacketInHexWithoutTransactionId;
if (sizeof(domainToIpArray) + 500 <= cacheLimitInBytes) {
domainToIpArray[domainNameInHex] = {'ttl' : defaultDnsResponseTtleInSeconds, 'dnsPacketInHexWithoutTransactionId' : dnsPacketInHexWithoutTransactionId, 'lastUsed': new Date().getTime()};
}
else {
removeLeastRecentlyUsedDomainFromCache();
domainToIpArray[domainNameInHex] = {'ttl' : defaultDnsResponseTtleInSeconds, 'dnsPacketInHexWithoutTransactionId' : dnsPacketInHexWithoutTransactionId, 'lastUsed': new Date().getTime()};
}
setTimeout(function() {
sendDnsResponse((dnsResponseToSendTransactionId + domainToIpArray[domainNameInHex]['dnsPacketInHexWithoutTransactionId']), clientIp, clientPort, "cold cache");
}, dnsResponseDelayInMilliseconds);
}
else if (!isDnsResponseSentToClient && freshDomainIp == 0) {
sendDnsResponse(msg.toString('hex', 0, msg.length), clientIp, clientPort, "original DNS server");
}
if (freshDomainIp != 0) {
processDnsResponse(msg);
}
});
p.login(function(err, player) {
if (err) {
res.json({
code: 3,
message: err
});
} else if (player) {
res.json({
code: 0,
message: {id: player.id, level: player.level}
});
} else {
if (sizeof(p) > maxSize)
res.json({
code: 2,
message: 'max input limit exceeded'
});
else
p.save(function(err, p) {
if (err)
res.json({
code: 3,
message: err
});
else
res.json({
code: 0,
message: {id: p.id, level: p.level}
});
});
}
});
Logger.prototype._bufferLog = function(message) {
if (!message || !message.line) return;
if (message.line.length > configs.MAX_LINE_LENGTH) {
message.line = message.line.substring(0, configs.MAX_LINE_LENGTH) + ' (cut off, too long...)';
debug('Line was longer than ' + configs.MAX_LINE_LENGTH + ' chars and was truncated.');
}
this._bufByteLength += sizeof(message);
// this._bufLength += getBytes(JSON.stringify(message));
this._buf.push(message);
if (this._bufByteLength >= this._flushLimit) {
this._flush();
return;
}
if (!this._flusher) {
this._flusher = setTimeout(_.bind(this._flush, this), configs.FLUSH_INTERVAL);
}
};
export default function () {
Meteor.methods({
async esBuckets({index, type, queries, aggs}) {
console.log("esBuckets", index, type, queries, aggs);
this.unblock();
try {
let search = {
"_source": false,
"size": 0,
"query": {
"bool": {
"must": [],
"filter": [{
"term": {
"summary.contribution._is_latest": "true"
}
}]
}
},
aggs : aggs
};
if (!_.find(_.map(queries, "terms"), "summary.contribution._private_key"))
search.query.bool.filter.push({
"term": { "summary.contribution._is_activated": "true" }
});
if (_.isArray(queries)) search.query.bool.must.push(...queries);
let resp = await esClient.search({
"index": index,
"type": type,
"body": search
});
return _.reverse(_.sortBy(resp.aggregations.buckets.buckets, ["doc_count"]));
} catch(error) {
console.error("esBuckets", index, type, queries, error.message);
throw new Meteor.Error("esBuckets", error.message);
}
},
async esCount({index, type, queries, filters, countField}) {
console.log("esCount", index, type, queries, filters, countField);
this.unblock();
try {
let search = {
"query": {
"bool": {
"must": [],
"filter": [{
"term": {
"summary.contribution._is_latest": "true"
}
}]
}
}
};
if (!_.find(_.map(queries, "terms"), "summary.contribution._private_key")) {
search.query.bool.filter.push({
"term": { "summary.contribution._is_activated": "true" }
});
}
if (_.isArray(queries)) search.query.bool.must.push(...queries);
if (_.isArray(filters)) search.query.bool.filter.push(...filters);
if (_.trim(countField) !== "") search.aggs = { count: { sum: { field: countField }}};
console.log("esCount search:", JSON.stringify(search));
let resp = _.trim(countField) !== "" ?
await esClient.search({
"index": index,
"type": type,
"body": _.extend({}, search, { "_source": false, "size": 0 }),
"timeout": "60s"
}) :
await esClient.count({
"index": index,
"type": type,
"body": search
});
let count = _.trim(countField) !== "" ? resp.aggregations.count.value : resp.count;
console.log("esCount hits:", count);
return count;
} catch(error) {
console.error("esCount", index, type, queries, filters, countField, error.message);
throw new Meteor.Error("esCount", error.message);
}
},
async esPage({index, type, queries, filters, source, sort}, pageSize, pageNumber) {
console.log("esPage", pageSize, pageNumber, index, type, queries, filters, sort);
this.unblock();
try {
let search = {
"from": pageSize * (pageNumber - 1),
"size": pageSize,
"_source": source,
"query": {
"bool": {
"must": [],
"filter": [{
"term": {
"summary.contribution._is_latest": "true"
}
}]
}
}
};
if (sort) search.sort = sort;
if (!_.find(_.map(queries, "terms"), "summary.contribution._private_key")) {
search.query.bool.filter.push({
"term": { "summary.contribution._is_activated": "true" }
});
}
if (_.isArray(queries)) search.query.bool.must.push(...queries);
if (_.isArray(filters)) search.query.bool.filter.push(...filters);
console.log("esPage search:", JSON.stringify(search));
let resp = await esClient.search({
"index": index,
"type": type,
"body": search,
"timeout": "60s"
});
console.log("esPage hits:", resp.hits.total);
return resp.hits.hits.map(hit => hit._source);
} catch(error) {
console.error("esPage", index, type, queries, filters, source, sort, pageSize, pageNumber, error.message);
throw new Meteor.Error("esPage", error.message);
}
},
async esScroll({index, type, queries, filters, source, sort}, pageSize) {
console.log("esScroll", index, type, queries, filters, sort);
this.unblock();
try {
let search = {
"size": pageSize,
"_source": source,
"query": {
"bool": {
"must": [],
"filter": [{
"term": {
"summary.contribution._is_latest": "true"
}
}]
}
}
};
if (sort) search.sort = sort;
if (!_.find(_.map(queries, "terms"), "summary.contribution._private_key"))
search.query.bool.filter.push({
"term": { "summary.contribution._is_activated": "true" }
});
if (_.isArray(queries)) search.query.bool.must.push(...queries);
if (_.isArray(filters)) search.query.bool.filter.push(...filters);
let resp = await esClient.search({
"index": index,
"scroll": '30s',
"type": type,
"body": search
});
return resp;
} catch(error) {
console.error("esScroll", index, type, queries, filters, source, sort, pageSize, error.message);
throw new Meteor.Error("esScroll", error.message);
}
},
async esScrollByID(scrollID) {
console.log("esScrollByID", scrollID);
this.unblock();
try {
let resp = await esClient.scroll({
"scrollId": scrollID,
"scroll": '30s'
});
return resp;
} catch(error) {
console.error("esScrollByID", scrollID);
throw new Meteor.Error("esScrollByID", error.message);
}
},
async esContributionIDs({index, queries, filters}) {
console.log("esContributionIDs", index, queries, filters);
this.unblock();
try {
let search = {
"_source": false,
"size": 0,
"query": {
"bool": {
"must": [],
"filter": [{
"term": {
"summary.contribution._is_latest": "true"
}
}]
}
},
aggs : {buckets: {terms: {field: "summary.contribution.id", size:1e6}}}
};
if (!_.find(_.map(queries, "terms"), "summary.contribution._private_key"))
search.query.bool.filter.push({
"term": { "summary.contribution._is_activated": "true" }
});
if (_.isArray(queries)) search.query.bool.must.push(...queries);
if (_.isArray(filters)) search.query.bool.filter.push(...filters);
let resp = await esClient.search({
"index": index,
"type": "contribution",
"body": search
});
return resp.aggregations.buckets.buckets.map((id) => id.key);
} catch(error) {
console.error("esContributionIDs", index, queries, filters, error.message);
throw new Meteor.Error("esContributionIDs", error.message);
}
},
async esCreatePrivateContribution({index, contributor, _contributor, name, contribution, summary}) {
console.log("esCreatePrivateContribution", index, contributor, _contributor, name);
this.unblock();
try {
// Get the next contribution ID
let next_id = await esClient.search({
"index": index,
"type": "contribution",
"body": {
"size": 1,
"_source": "summary.contribution.id",
"query": {
"exists": {
"field": "summary.contribution.id"
}
},
"sort": [{
"summary.contribution.id": "desc"
}]
}
});
if (isNaN(_.parseInt(next_id.hits.hits[0]._source.summary.contribution.id)))
throw new Error('Failed to retrieve new contribution ID.');
next_id = _.parseInt(next_id.hits.hits[0]._source.summary.contribution.id) + 1;
let timestamp = moment().utc().toISOString();
let contributionSummary = {
"id": next_id,
"version": null,
"contributor": contributor,
"timestamp": timestamp,
"data_model_version": _.last(versions),
"_contributor": _contributor,
"_name": name,
"_is_activated": "false",
"_is_latest": "true",
"_private_key": uuid.v4(),
"_history": [{
"id": next_id,
"version": null,
"contributor": _contributor,
"timestamp": timestamp,
"data_model_version": _.last(versions)
}]
};
summary = summary || {};
summary.contribution = summary.contribution || {};
summary.contribution = _.merge(summary.contribution, contributionSummary);
let contributionRow = {
"contributor": contributor,
"id": next_id,
"version": null,
"timestamp": timestamp,
"data_model_version": _.last(versions)
};
contribution = contribution || {};
contribution.contribution =
contribution.contribution &&
contribution.contribution.length &&
contribution.contribution.length === 1 &&
contribution.contribution || [{}];
contribution.contribution[0] = _.merge(contribution.contribution[0], contributionRow);
// Create the new contribution and return the new contribution ID
await esClient.index({
"index": index,
"type": "contribution",
"id": next_id + "_0",
"body": {
"summary": summary,
"contribution": contribution
},
"refresh": true
});
return next_id;
} catch(error) {
console.error("esCreatePrivateContribution", index, contributor, _contributor, name, error.message);
throw new Meteor.Error("esCreatePrivateContribution", error.message);
}
},
async esUpdatePrivateContribution({index, contributor, _contributor, id, contribution, summary}) {
console.log("esUpdatePrivateContribution", index, contributor, _contributor, id);
this.unblock();
try {
let contributionRow = {
"contributor": contributor,
"id": _.parseInt(id),
"data_model_version": _.last(versions)
};
contribution = contribution || {};
contribution.contribution =
contribution.contribution &&
contribution.contribution.length &&
contribution.contribution.length === 1 &&
contribution.contribution || [{}];
contribution.contribution[0] = _.merge(contribution.contribution[0], contributionRow);
summary = summary || {};
summary.contribution = summary.contribution || {};
summary.contribution = _.merge(summary.contribution, contributionRow);
summary.contribution._is_valid = "false",
console.log("esUpdatePrivateContribution updating es index", index, contributor, _contributor, id, sizeof(summary), sizeof(contribution));
await esClient.update({
"index": index,
"type": "contribution",
"id": id + "_0",
"refresh": true,
"body": {
"doc": {
"summary": summary,
"contribution": contribution
}
}
});
} catch(error) {
console.error("esUpdatePrivateContribution", index, contributor, _contributor, id, error.message);
throw new Meteor.Error("esUpdatePrivateContribution", error.message);
}
},
async esUpdatePrivatePreSummaries({index, contributor, id}) {
console.log("esUpdatePrivatePreSummaries", index, contributor, id);
this.unblock();
const summarizer = new SummarizeContribution({});
let contribution = {};
let summary = {};
try {
let resp = await esClient.search({
"index": index,
"type": "contribution",
"body": {
"_source": {
"includes": ["summary.contribution.*", "contribution.*", "criteria.*"]
},
"query": {
"bool": {
"filter": [{
"term": {
"summary.contribution.id": id
}
}, {
"term": {
"summary.contribution.contributor.raw": contributor
}
}]
}
}
}
});
if (resp.hits.total > 0) {
if (resp.hits.hits[0]._source.contribution && _.isPlainObject(resp.hits.hits[0]._source.contribution.contribution))
resp.hits.hits[0]._source.contribution.contribution = [resp.hits.hits[0]._source.contribution.contribution];
contribution = resp.hits.hits[0]._source.contribution;
summary.contribution = resp.hits.hits[0]._source.summary.contribution;
}
await summarizer.preSummarizePromise(contribution, {summary: summary});
console.log("esUpdatePrivatePreSummaries updating contribution doc", index, contributor, id + "_0");
await esClient.update({
"index": index,
"type": "contribution",
"id": id + "_0",
"refresh": true,
"body": {
"doc": { summary: summarizer.json.contribution.summary }
}
});
console.log("esUpdatePrivatePreSummaries updated contribution doc", index, contributor, id + "_0");
let bulkIndex = [], rowIdx = 1;
_.without(_.keys(summarizer.json), 'contribution').forEach((indexType) => {
_.keys(summarizer.json[indexType]).forEach((name) => {
_.keys(summarizer.json[indexType][name]).forEach((parent) => {
bulkIndex.push(
{ index: { _index: index, _type: indexType, _id: id + '_' + rowIdx } },
{ summarizer: { indexType, name, parent }}
);
rowIdx += 1;
});
});
});
await BPromise.map(_.chunk(bulkIndex, 100), (bulkIndexChunk, i, n) => {
return new Promise((resolve) => {
console.log('esUpdatePrivatePreSummaries starting chunk', i+1, 'of', n);
esClient.bulk({
body: bulkIndexChunk.map(row => {
if (row && row.summarizer)
row = summarizer.json[row.summarizer.indexType][row.summarizer.name][row.summarizer.parent];
row.summary = row.summary || {};
row.summary.contribution = summarizer.json.contribution.summary.contribution;
return row;
})
}, (err, resp) => {
if (!resp || resp.errors) {
console.error('esUpdatePrivatePreSummaries errors in chunk', i+1, 'of', n, JSON.stringify(resp));
resolve(false);
} else {
console.log('esUpdatePrivatePreSummaries finished chunk', i+1, 'of', n);
resolve(true);
}
});
});
}, { concurrency: 5 }).then((results) => {
if (!_.every(results, Boolean))
throw new Meteor.Error("esUpdatePrivatePreSummaries", "Failed to upload private contribution.");
});
console.log("esUpdatePrivatePreSummaries finished", index, contributor, id);
} catch(error) {
console.error("esUpdatePrivatePreSummaries", index, contributor, id, error.message);
throw new Meteor.Error("esUpdatePrivatePreSummaries", error.message);
}
},
async esUpdatePrivateSummaries({index, contributor, id}) {
console.log("esUpdatePrivateSummaries", index, contributor, id);
this.unblock();
const summarizer = new SummarizeContribution({});
let contribution = {};
let summary = {};
try {
let resp = await esClient.search({
"index": index,
"type": "contribution",
"body": {
"_source": {
"includes": ["summary.contribution.*", "contribution.*"]
},
"query": {
"bool": {
"filter": [{
"term": {
"summary.contribution.id": id
}
}, {
"term": {
"summary.contribution.contributor.raw": contributor
}
}]
}
}
}
});
if (resp.hits.total > 0) {
if (resp.hits.hits[0]._source.contribution && _.isPlainObject(resp.hits.hits[0]._source.contribution.contribution))
resp.hits.hits[0]._source.contribution.contribution = [resp.hits.hits[0]._source.contribution.contribution];
contribution = resp.hits.hits[0]._source.contribution;
summary.contribution = resp.hits.hits[0]._source.summary.contribution;
}
await summarizer.summarizePromise(contribution, {summary: summary});
console.log("esUpdatePrivateSummaries updating contribution doc", index, contributor, id + "_0");
await esClient.update({
"index": index,
"type": "contribution",
"id": id + "_0",
"refresh": true,
"body": {
"doc": { summary: summarizer.json.contribution.summary }
}
});
console.log("esUpdatePrivateSummaries updated contribution doc", index, contributor, id + "_0");
let bulkIndex = [], rowIdx = 1;
_.without(_.keys(summarizer.json), 'contribution').forEach((indexType) => {
_.keys(summarizer.json[indexType]).forEach((name) => {
_.keys(summarizer.json[indexType][name]).forEach((parent) => {
bulkIndex.push(
{ index: { _index: index, _type: indexType, _id: id + '_' + rowIdx } },
{ summarizer: { indexType, name, parent }}
);
rowIdx += 1;
});
});
});
await BPromise.map(_.chunk(bulkIndex, 100), (bulkIndexChunk, i, n) => {
return new Promise((resolve) => {
console.log('esUpdatePrivateSummaries starting chunk', i+1, 'of', n);
esClient.bulk({
body: bulkIndexChunk.map(row => {
if (row && row.summarizer)
row = summarizer.json[row.summarizer.indexType][row.summarizer.name][row.summarizer.parent];
row.summary = row.summary || {};
row.summary.contribution = summarizer.json.contribution.summary.contribution;
return row;
})
}, (err, resp) => {
if (!resp || resp.errors) {
console.error('esUpdatePrivateSummaries errors in chunk', i+1, 'of', n, JSON.stringify(resp));
resolve(false);
} else {
console.log('esUpdatePrivateSummaries finished chunk', i+1, 'of', n);
resolve(true);
}
});
});
}, { concurrency: 5 }).then((results) => {
if (!_.every(results, Boolean))
throw new Meteor.Error("esUpdatePrivateSummaries", "Failed to upload private contribution.");
});
console.log("esUpdatePrivateSummaries finished", index, contributor, id);
} catch(error) {
console.error("esUpdatePrivateSummaries", index, contributor, id, error.message);
throw new Meteor.Error("esUpdatePrivateSummaries", error.message);
}
},
// TODO: pass login token to authenticate changes
async esGetPrivateContributionSummaries({index, contributor, includeActivated}) {
console.log("esGetPrivateContributionSummaries", index, contributor);
this.unblock();
try {
let resp = await esClient.search({
"index": index,
"type": "contribution",
"body": {
"size": 100,
"_source": {
"excludes": ["*.vals", "*._geo_shape"],
"includes": ["summary.contribution.*", "summary._all.*", "summary._incomplete_summary"]
},
"query": {
"bool": {
"filter": (includeActivated ?
[{
"term": {
"summary.contribution.contributor.raw": contributor
}
}] : [{
"term": {
"summary.contribution.contributor.raw": contributor
}
},{
"term": {
"summary.contribution._is_activated": "false"
}
}]
)
}
},
"sort": [{
"summary.contribution._is_activated": "asc"
}, {
"summary.contribution.id": "desc"
}]
}
});
return resp.hits.hits.map(hit => hit._source);
} catch(error) {
console.error("esGetPrivateContributionSummaries", index, contributor, error.message);
throw new Meteor.Error("esGetPrivateContributionSummaries", error.message);
}
},
async esGetPrivateContributionSummary({index, id, contributor}) {
console.log("esGetPrivateContributionSummary", index, id, contributor);
this.unblock();
try {
let resp = await esClient.search({
"index": index,
"type": "contribution",
"body": {
"_source": {
"excludes": ["*.vals", "*._geo_shape"],
"includes": ["summary.contribution.*", "summary._all.*"]
},
"query": {
"bool": {
"filter": [{
"term": {
"summary.contribution.id": id
}
}, {
"term": {
"summary.contribution.contributor.raw": contributor
}
}, {
"term": {
"summary.contribution._is_activated": "false"
}
}]
}
}
}
});
return resp.hits.total > 0 && resp.hits.hits[0]._source;
} catch(error) {
console.error("esGetPrivateContributionSummary", index, id, contributor, error.message);
throw new Meteor.Error("esGetPrivateContributionSummary", error.message);
}
},
async esGetContribution({index, id}) {
console.log("esGetContribution", index, id);
this.unblock();
try {
let resp = await esClient.search({
"index": index,
"type": "contribution",
"body": {
"_source": {
"includes": ["contribution.*"]
},
"query": {
"bool": {
"filter": [{
"term": {
"summary.contribution.id": id
}
}]
}
}
}
});
if (resp.hits.total > 0 && resp.hits.hits[0]._source.contribution && _.isPlainObject(resp.hits.hits[0]._source.contribution.contribution))
resp.hits.hits[0]._source.contribution.contribution = [resp.hits.hits[0]._source.contribution.contribution];
return resp.hits.total > 0 && resp.hits.hits[0]._source.contribution;
} catch(error) {
console.error("esGetContribution", index, id, error.message);
throw new Meteor.Error("esGetContribution", error.message);
}
},
async esUpdateContributionName({index, id, name}) {
console.log("esUpdateContributionName", index, id, name);
this.unblock();
try {
await esClient.update({
"index": index,
"type": "contribution",
"id": id + "_0",
"refresh": true,
"body": {
"doc": {
"summary": {
"contribution": {
"_name": name
}
}
}
}
});
} catch(error) {
console.error("esUpdateContributionName", index, id, name, error.message);
throw new Meteor.Error("esUpdateContributionName", error.message);
}
},
async esUpdateContributionDescription({index, id, description}) {
console.log("esUpdateContributionDescription", index, id, description);
this.unblock();
try {
let resp = await esClient.search({
"index": index,
"type": "contribution",
"body": {
"_source": {
"includes": ["summary.contribution._history"]
},
"query": {
"bool": {
"filter": [{
"term": {
"summary.contribution.id": id
}
}]
}
}
}
});
if (resp.hits.total > 0) {
let _history = resp.hits.hits[0]._source.summary.contribution._history;
_history[0].description = description;
resp = await esClient.updateByQuery({
"index": index,
"refresh": true,
"body": {
"script": {
"source": "ctx._source.summary.contribution._history = params._history",
"params": {_history}
},
"query": {
"term": {
"summary.contribution.id": id
}
}
}
});
}
} catch(error) {
console.error("esUpdateContributionDescription", index, id, description, error.message);
}
try {
await esClient.update({
"index": index,
"type": "contribution",
"id": id + "_0",
"refresh": true,
"body": {
"script": {
"source": `
ctx._source.summary.contribution.description = params.description;
ctx._source.contribution.contribution[0].description = params.description;
`,
"params": {description}
}
}
});
} catch(error) {
console.error("esUpdateContributionDescription", index, id, description, error.message);
throw new Meteor.Error("esUpdateContributionDescription", error.message);
}
},
// TODO: pass login token to authenticate changes
async esUpdateContributionReference({index, id, contributor, _contributor, timestamp, reference, description}) {
console.log("esUpdateContributionReference", index, id, contributor, _contributor, timestamp, reference, description);
this.unblock();
timestamp = timestamp || moment().utc().toISOString();
let doi = _.toUpper(_.trim(reference));
let _reference = {};
try {
_reference = Meteor.call("getReferenceMetadata", doi);
} catch(error) {
console.error("esUpdateContributionReference", index, id, contributor, _contributor, reference, description, error.message);
}
let _history = [{
"id": _.parseInt(id),
"version": null,
"contributor": _contributor,
"timestamp": timestamp || moment().utc().toISOString(),
"data_model_version": _.last(versions),
"description": description
}];
try {
let resp = await esClient.search({
"index": index,
"type": "contribution",
"body": {
"_source": {
"includes": ["summary.contribution._history"]
},
"query": {
"bool": {
"filter": [{
"term": {
"summary.contribution.reference.raw": doi
}
}, {
"term": {
"summary.contribution._is_activated": "true"
}
}, {
"term": {
"summary.contribution._is_latest": "true"
}
}]
}
}
}
});
if (resp.hits.total > 0) {
_history[0].version = parseInt(resp.hits.hits[0]._source.summary.contribution._history[0].version) + 1;
_history.push(...resp.hits.hits[0]._source.summary.contribution._history);
} else {
_history[0].version = 1;
}
} catch(error) {
_history[0].version = 1;
console.error("esUpdateContributionReference", index, id, contributor, _contributor, reference, description, error.message);
}
let version = _history[0].version;
try {
await esClient.update({
"index": index,
"type": "contribution",
"id": id + "_0",
"refresh": true,
"body": {
"script": {
"source": `
ctx._source.summary.contribution.version = null;
ctx._source.summary.contribution._reference = null;
ctx._source.summary.contribution._history = null;
`
}
}
});
// TODO: change this to an update by script and only update the reference info:
await esClient.update({
"index": index,
"type": "contribution",
"id": id + "_0",
"refresh": true,
"body": {
"doc": {
"summary": {
"contribution": {
"version": version,
"contributor": contributor,
"timestamp": timestamp,
"data_model_version": _.last(versions),
"description": description,
"reference": doi,
"_reference": _reference,
"_history": _history
}
},
"contribution": {
"contribution": [{
"id": _.parseInt(id),
"version": version,
"contributor": contributor,
"timestamp": timestamp,
"data_model_version": _.last(versions),
"description": description,
"reference": doi
}]
}
}
}
});
} catch(error) {
console.error("esUpdateContributionReference", index, id, contributor, _contributor, reference, description, error.message);
throw new Meteor.Error("esUpdateContributionReference", error.message);
}
},
async esValidatePrivateContribution({index, id, contributor}) {
console.log("esValidatePrivateContribution", index, id, contributor);
this.unblock();
try {
const validator = new ValidateContribution({});
let resp = await esClient.search({
"index": index,
"type": "contribution",
"body": {
"_source": {
"includes": ["contribution.*"]
},
"query": {
"bool": {
"filter": [{
"term": {
"summary.contribution.id": id
}
}]
}
}
}
});
if (resp.hits.total > 0 && resp.hits.hits[0]._source.contribution && _.isPlainObject(resp.hits.hits[0]._source.contribution.contribution))
resp.hits.hits[0]._source.contribution.contribution = [resp.hits.hits[0]._source.contribution.contribution];
await validator.validatePromise(resp.hits.hits[0]._source.contribution);
await esClient.update({
"index": index,
"type": "contribution",
"id": id + "_0",
"refresh": true,
"body": {
"doc": {
"summary": {
"contribution": {
"_is_valid": _.keys(validator.validation.errors).length ? "false" : "true"
}
}
}
}
});
return validator.validation;
} catch(error) {
console.error("esValidatePrivateContribution", index, id, contributor, error.message);
throw new Meteor.Error("esValidatePrivateContribution", error.message);
}
},
async esDeletePrivateContribution({index, id, contributor}) {
console.log("esDeletePrivateContribution", index, id, contributor);
this.unblock();
try {
let resp = await esClient.deleteByQuery({
"index": index,
"refresh": true,
"body": {
"query": {
"term": {
"summary.contribution.id": id
}
}
}
});
return true;
} catch(error) {
console.error("esDeletePrivateContribution", index, id, contributor, error.message);
throw new Meteor.Error("esDeletePrivateContribution", error.message);
}
},
async esActivateContribution({index, id}) {
console.log("esActivateContribution", index, id);
this.unblock();
let prev_id;
let contributionSummary;
try {
let resp = await esClient.search({
"index": index,
"type": "contribution",
"body": {
"_source": {
"includes": ["summary.contribution"]
},
"query": {
"bool": {
"filter": [{
"term": {
"summary.contribution.id": id
}
}]
}
}
}
});
if (resp.hits.total > 0 && resp.hits.hits[0]._source.summary)
contributionSummary = resp.hits.hits[0]._source.summary.contribution;
if (resp.hits.total > 0 && resp.hits.hits[0]._source.summary.contribution._history.length > 1) {
prev_id = resp.hits.hits[0]._source.summary.contribution._history[1].id;
await esClient.updateByQuery({
"index": index,
"refresh": true,
"body": {
"script": {
"source": "ctx._source.summary.contribution._is_latest = \"false\""
},
"query": {
"term": {
"summary.contribution.id": prev_id
}
}
}
});
}
} catch(error) {
console.error("esActivateContribution", index, id, error.message);
}
try {
contributionSummary._is_activated = "true";
contributionSummary.timestamp = moment().utc().toISOString();
resp = await esClient.updateByQuery({
"index": index,
"refresh": true,
"body": {
"script": {
"source": "ctx._source.summary.contribution = params.contributionSummary",
"params": {contributionSummary}
},
"query": {
"term": {
"summary.contribution.id": id
}
}
}
});
console.log("esActivateContribution activated ", resp.updated, "of", resp.total);
} catch(error) {
console.error("esActivateContribution", index, id, error.message);
throw new Meteor.Error("esActivateContribution", error.message);
}
Meteor.call("esUploadActivatedContributionToS3", {index, id});
},
async esUploadActivatedContributionToS3({index, id}) {
console.log("esUploadActivatedContributionToS3", index, id);
this.unblock();
try {
let resp = await esClient.search({
"index": index,
"type": "contribution",
"body": {
"_source": {
"includes": ["contribution", "summary.contribution._is_latest", "summary.contribution._history"]
},
"query": {
"bool": {
"filter": [{
"term": {
"summary.contribution.id": id
}
}]
}
}
}
});
const contribution = resp.hits.hits[0]._source.contribution;
const history = resp.hits.hits[0]._source.summary.contribution._history;
const isLatest = resp.hits.hits[0]._source.summary.contribution._is_latest;
const exporter = new ExportContribution({});
const contributionText = exporter.toText(contribution);
//console.log("esUploadActivatedContributionToS3", id, isLatest, contribution, contributionText);
if (isLatest === 'true') {
history.slice(1).forEach(v => {
try {
s3DeleteKeys({
bucket: 'magic-contributions',
keys: [`${v.id}/magic_contribution_${v.id}.txt`]
}).then(() => {
console.log("esUploadActivatedContributionToS3", `Deleted ${v.id}/magic_contribution_${v.id}.txt`);
s3DeleteKeys({
bucket: 'magic-contributions',
keys: [`${v.id}/`]
}).then(() => {
console.log("esUploadActivatedContributionToS3", `Deleted ${v.id}/`);
});
});
} catch (e) {
console.log("esUploadActivatedContributionToS3", `Error deleting ${v.id}`, e);
}
});
try {
s3UploadObject({
bucket: 'magic-contributions',
key: `${id}/magic_contribution_${id}.txt`,
body: contributionText
}).then(() => {
console.log("esUploadActivatedContributionToS3", `Uploaded ${id}/magic_contribution_${id}.txt to magic-contributions`);
});
} catch (e) {
console.log("esUploadActivatedContributionToS3", `Error uploading latest ${v.id}`, e);
}
}
try {
s3UploadObject({
bucket: 'magic-activated-contributions',
key: `${id}/magic_contribution_${id}.txt`,
body: contributionText
}).then(() => {
console.log("esUploadActivatedContributionToS3", `Uploaded ${id}/magic_contribution_${id}.txt to magic-activated-contributions`);
});
} catch (e) {
console.log("esUploadActivatedContributionToS3", `Error uploading ${v.id}`, e);
}
} catch(error) {
console.error("esUploadActivatedContributionToS3", index, id, error);
throw new Meteor.Error("esUploadActivatedContributionToS3", error.message);
}
console.error("esUploadActivatedContributionToS3 finished", id);
},
async esDeactivateContribution({index, id}) {
console.log("esDeactivateContribution", index, id);
this.unblock();
let prev_id;
try {
let resp = await esClient.search({
"index": index,
"type": "contribution",
"body": {
"_source": {
"includes": ["summary.contribution._history"]
},
"query": {
"bool": {
"filter": [{
"term": {
"summary.contribution.id": id
}
}]
}
}
}
});
if (resp.hits.total > 0 && resp.hits.hits[0]._source.summary.contribution._history.length > 1) {
prev_id = resp.hits.hits[0]._source.summary.contribution._history[1].id;
await esClient.updateByQuery({
"index": index,
"refresh": true,
"body": {
"script": {
"source": "ctx._source.summary.contribution._is_latest = \"true\""
},
"query": {
"term": {
"summary.contribution.id": prev_id
}
}
}
});
}
} catch(error) {
console.error("esDeactivateContribution", index, id, error.message);
}
try {
await esClient.updateByQuery({
"index": index,
"refresh": true,
"body": {
"script": {
"source": "ctx._source.summary.contribution._is_activated = \"false\""
},
"query": {
"term": {
"summary.contribution.id": id
}
}
}
});
} catch(error) {
console.error("esDeactivateContribution", index, id, error.message);
throw new Meteor.Error("esDeactivateContribution", error.message);
}
return true;
}
});
};
it('lossy decimal should be smaller than lossless compression', function () {
assert.equal(
sizeof(compressedLossyDecimal) < sizeof(compressed),
true
);
});
it('should be at least 40% smaller than the original with given example', function () {
assert.equal(
1 - sizeof(compressed) / sizeof(object) > 0.4,
true
);
});
server.on("message", function (msg, rinfo) {
console.log("Total Traffic: " + ((dnsTrafficInBytes + tcpSynTrafficInBytes)/1000) + " KB------Current Cache Size: " + (sizeof(domainToIpArray)/1000.0) + " KB------Number of Domains Cached: " + Object.size(domainToIpArray));
var clientReq = msg.toString('hex', 0, msg.length);
if(clientReq.substr(clientReq.length - 8, 4) == "0001") {
var min = Number.MAX_VALUE, minIp = "";
var domainNameInHex = clientReq.substr(24, clientReq.length - 32);
var clientIp = rinfo.address;
var clientPort = rinfo.port;
var isDnsResponseSentToClient = false;
var dnsResponseToSendTransactionId = clientReq.substr(0, 4);
var ipArray = [];
if (typeof domainToIpArray[domainNameInHex] === "undefined") {
var message = new Buffer(msg);
var client = dgram.createSocket("udp4");
client.on('error', function(err) {});
client.bind();
dnsIpList.forEach(function(dnsIp) {
client.send(message, 0, message.length, 53, dnsIp, function(err, bytes) {
if (err) {}
else if(bytes) {
dnsTrafficInBytes = dnsTrafficInBytes + message.length + 28;
}
});
});
setTimeout(function() {
client.close();
}, 2000);
client.on("message", function (msg, rinfo) {
dnsTrafficInBytes = dnsTrafficInBytes + msg.length + 28;
var freshDomainIp = parseIpFromDnsResponse(msg);
if(!isDnsResponseSentToClient && freshDomainIp != 0) {
var freshDomainTtl = "00000004";
var dnsPacketInHexWithoutTransactionId = "818000010001" + clientReq.substr(16) + "c00c00010001" + freshDomainTtl + "0004" +freshDomainIp;
var dnsResponseToSend = dnsResponseToSendTransactionId + dnsPacketInHexWithoutTransactionId;
if (sizeof(domainToIpArray) + 500 <= cacheLimitInBytes) {
domainToIpArray[domainNameInHex] = {'ttl' : defaultDnsResponseTtleInSeconds, 'dnsPacketInHexWithoutTransactionId' : dnsPacketInHexWithoutTransactionId, 'lastUsed': new Date().getTime()};
}
else {
removeLeastRecentlyUsedDomainFromCache();
domainToIpArray[domainNameInHex] = {'ttl' : defaultDnsResponseTtleInSeconds, 'dnsPacketInHexWithoutTransactionId' : dnsPacketInHexWithoutTransactionId, 'lastUsed': new Date().getTime()};
}
setTimeout(function() {
sendDnsResponse((dnsResponseToSendTransactionId + domainToIpArray[domainNameInHex]['dnsPacketInHexWithoutTransactionId']), clientIp, clientPort, "cold cache");
}, dnsResponseDelayInMilliseconds);
}
else if (!isDnsResponseSentToClient && freshDomainIp == 0) {
sendDnsResponse(msg.toString('hex', 0, msg.length), clientIp, clientPort, "original DNS server");
}
if (freshDomainIp != 0) {
processDnsResponse(msg);
}
});
}
else {
var dnsResponseFromCache = dnsResponseToSendTransactionId + domainToIpArray[domainNameInHex]['dnsPacketInHexWithoutTransactionId'];
sendDnsResponse(dnsResponseFromCache, clientIp, clientPort, "warm cache");
domainToIpArray[domainNameInHex]['lastUsed'] = new Date().getTime();
}
function removeLeastRecentlyUsedDomainFromCache () {
var domainNameList = Object.getOwnPropertyNames(domainToIpArray);
var leastUsedDomainTimestamp = parseInt(domainToIpArray[domainNameList[0]]['lastUsed']);
var leastUsedDomainIndex = 0;
for ( var i = 0; i < domainNameList.length; i++) {
if (parseInt(domainToIpArray[domainNameList[i]]['lastUsed']) < leastUsedDomainTimestamp) {
leastUsedDomainTimestamp = parseInt(domainToIpArray[domainNameList[i]]['lastUsed']);
leastUsedDomainIndex = i;
}
}
delete domainToIpArray[domainNameList[leastUsedDomainIndex]];
}
function parseIpFromDnsResponse(msg) {
var resInString = msg.toString('hex', 0 , msg.length);
var isAuthority = parseInt(resInString.substr(16, 4), 16);
var newResponse = resInString + "";
if (!isAuthority) {
newResponse = "";
var noOfAnswer = parseInt(resInString.substr(12, 4), 16);
var posAfterQuery = resInString.indexOf("00010001", 24);
newResponse = resInString.substr(0, posAfterQuery + 8);
var resAfterQuery = resInString.substr(posAfterQuery + 8);
var loop = 0;
var initialStart = 0;
var currentAnswerCount = noOfAnswer;
while (loop < noOfAnswer) {
var dataLength = parseInt(resAfterQuery.substr(20, 4), 16);
var answerLength = 24 + (dataLength * 2);
var answer = resAfterQuery.substr(initialStart, answerLength);
if (dataLength != 4 && currentAnswerCount > 1) {
resAfterQuery = resAfterQuery.replace(answer, "");
currentAnswerCount = currentAnswerCount - 1;
}
else if(dataLength == 4) {
return answer.substr(answer.length - 8);
}
loop++;
}
}
else {
return 0;
}
}
function sendDnsResponse(msg, ip, port, responseType) {
try {
var message = new Buffer(msg, 'hex');
server.send(message, 0, message.length, port, ip, function(err, bytes) {
if (err) {}
else if (bytes) {
if (responseType === "cold cache" && isDnsResponseSentToClient === false) {
setTimeout(function() {
delete domainToIpArray[domainNameInHex];
}, parseInt(domainToIpArray[domainNameInHex]['ttl']) * 1000 * 8);
}
isDnsResponseSentToClient = true;
}
});
}
catch (exception) {}
}
function processDnsResponse(msg) {
var perDnsResponseIpArray = [];
var resInString = msg.toString('hex', 0 , msg.length);
var isAuthority = parseInt(resInString.substr(16, 4), 16);
var newResponse = resInString + "";
if (!isAuthority) {
newResponse = "";
var noOfAnswer = parseInt(resInString.substr(12, 4), 16);
var posAfterQuery = resInString.indexOf("00010001", 24);
newResponse = newResponse + resInString.substr(0, posAfterQuery + 8);
var resAfterQuery = resInString.substr(posAfterQuery + 8);
var loop = 0;
var initialStart = 0;
while (loop < noOfAnswer) {
var dataLength = parseInt(resAfterQuery.substr(20,4), 16);
var answerLength = 24 + (dataLength * 2);
var answer = resAfterQuery.substr(initialStart, answerLength);
if(dataLength == 4) {
var ip = answer.substr(answer.length - 8);
var ipTtl = answer.substr(12, 8);
evaluateIpAddress((parseInt(ip.substr(0, 2), 16) + "." + parseInt(ip.substr(2, 2), 16) + "." + parseInt(ip.substr(4, 2), 16) + "." + parseInt(ip.substr(6, 2), 16)), (parseInt(ipTtl, 16)).toString());
}
loop++;
resAfterQuery = resAfterQuery.substr(answerLength);
}
}
}
function evaluateIpAddress(currentIp, ttl) {
var rtt = 0;
var timeToSynAck, initialTimestamp;
var options = { protocol: raw.Protocol.TCP };
var socket = raw.createSocket (options);
socket.setOption (raw.SocketLevel.IPPROTO_IP, raw.SocketOption.IP_HDRINCL, 1);
socket.on ("close", function () {
});
socket.on ("error", function (error) {
socket.close();
});
socket.on ("message", function (buffer, source) {
if(source === currentIp) {
timeToSynAck = (new Date().getTime() - initialTimestamp);
rtt = timeToSynAck;
if(rtt < min){
min = rtt;
minIp = currentIp;
var ipSegments = minIp.split('.');
var newDomainToIpArrayString = "818000010001" + clientReq.substr(16) + "c00c00010001" + "00000004" + "0004";
for (var loop = 0; loop < ipSegments.length; loop++) {
var tempIpSegmentInHex = parseInt(ipSegments[loop]).toString(16);
newDomainToIpArrayString = newDomainToIpArrayString + ("00" + tempIpSegmentInHex).substr(tempIpSegmentInHex.length);
}
domainToIpArray[domainNameInHex]['dnsPacketInHexWithoutTransactionId'] = newDomainToIpArrayString;
domainToIpArray[domainNameInHex]['lastUsed'] = new Date().getTime();
domainToIpArray[domainNameInHex]['ttl'] = ttl;
}
tcpSynTrafficInBytes = tcpSynTrafficInBytes + buffer.length;
socket.close();
}
});
var commonIpHeader = '4500003c5d4840004006';
var ipHeader = commonIpHeader + calculateChecksum(commonIpHeader + '0000' + convertIpToHex(selfIp) + convertIpToHex(currentIp)) + convertIpToHex(selfIp) + convertIpToHex(currentIp);
for (var i = 0; i < portArrayInHex.length; i++) {
var commonTcpHeader = 'dd0f' + portArrayInHex[i] + 'f49432e60000000080022000';
var tcpHeader = commonTcpHeader + calculateChecksum(convertIpToHex(selfIp) + convertIpToHex(currentIp) + '00060020' + commonTcpHeader + '0000' + '0000020405b40103030801010402') + '0000020405b40103030801010402';
var finalRawPacket = ipHeader + tcpHeader;
finalRawPacket = ("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + finalRawPacket).substr(finalRawPacket.length);
var buffer = new Buffer(finalRawPacket, 'hex');
socket.send (buffer, 0, buffer.length, currentIp, function (error, bytes) {
if (error) {}
else {
initialTimestamp = new Date().getTime();
tcpSynTrafficInBytes = tcpSynTrafficInBytes + buffer.length;
}
});
}
}
function calculateChecksum(header) {
var hexArray = [];
for (var i = 0; i < header.length; i = i + 4) {
hexArray.push(header.substr(i, 4));
}
var hexSum = "0000";
for (var i = 0; i < hexArray.length; i = i + 2) {
var sum = (parseInt(hexArray[i], 16) + parseInt(hexArray[i + 1], 16)).toString(16);
hexSum = (parseInt(hexSum, 16) + parseInt(sum, 16)).toString(16);
}
var sumInBin = Hex2Bin(hexSum);
sumInBin = ("00000000000000000000" + sumInBin).substr(sumInBin.length);
var reducedSumInBinary = Dec2Bin(parseInt(sumInBin.substr(0, 4), 2) + parseInt(sumInBin.substr(4), 2));
reducedSumInBinary = ("0000000000000000" + reducedSumInBinary).substr(reducedSumInBinary.length);
var flippedSumInBinary = parseInt("1111111111111111", 2) ^ parseInt(reducedSumInBinary, 2);
return( (Dec2Hex(flippedSumInBinary)).toString());
}
function Bin2Dec(n){if(!checkBin(n))return 0;return parseInt(n,2).toString(10)}
function Hex2Bin(n){if(!checkHex(n))return 0;return parseInt(n,16).toString(2)}
function Dec2Hex(n){if(!checkDec(n)||n<0)return 0;return n.toString(16)}
function checkHex(n){return/^[0-9A-Fa-f]{1,64}$/.test(n)}
function checkBin(n){return/^[01]{1,64}$/.test(n)}
function checkDec(n){return/^[0-9]{1,64}$/.test(n)}
function Dec2Bin(n){if(!checkDec(n)||n<0)return 0;return n.toString(2)}
function convertIpToHex(ip) {
var ipInHex = '';
var ipSegments = ip.split('.');
for (var loop = 0; loop < ipSegments.length; loop++) {
var tempIpSegmentInHex = parseInt(ipSegments[loop]).toString(16);
ipInHex = ipInHex + ("00" + tempIpSegmentInHex).substr(tempIpSegmentInHex.length);
}
return ipInHex;
}
}
});
const cacheSize = () => {
return sizeof(__data);
};
import sizeof from 'object-sizeof';
import Config from './Config';
const __data = {};
const Cache = {
store (keys, data) {
const dataSize = sizeof(data);
const maxSize = Config.get('maxCacheSize');
const k = key(keys);
if(__data[k]) delete __data[k];
while( ((cacheSize() + dataSize) > maxSize)) {
if(!numItems()) {
return;
}
purgeOldest();
}
__data[k] = {
lastAccessed: now(),
data
};
},
retrieve (keys) {
const result = __data[key(keys)];
['d', 'D=ARG', 'number of features per observation (default: 16)'],
['i', 'I=ARG', 'number of iterations (default: 4)'],
['p', 'P=ARG', 'number of partitons (default: number of workers)']
]).bindHelp().parseSystem();
var file = opt.options.F;
var N = Number(opt.options.N) || 1000000;
var K = Number(opt.options.K) || 10;
var D = Number(opt.options.D) || 16;
var P = opt.options.P;
var nIterations = Number(opt.options.I) || 4;
var seed = 1;
var sample = [1, []];
for (var i = 0; i < D; i++) sample[1].push(Math.random());
var approx_data_size = N * sizeOf(sample);
console.log('Input data: ' + (file || 'random'));
console.log('Number of observations: ' + N);
console.log('Number of clusters: ' + K);
console.log('Features per observation: ' + D);
console.log('Iterations: ' + nIterations);
console.log('Partitions: ' + (P || 'number of workers'));
console.log('Approximate dataset size: ' + Math.ceil(approx_data_size / (1024 * 1024)) + ' Mb\n');
var sc = skale.context();
var points = file ? sc.textFile(file).map(function (e) {
var tmp = e.split(' ').map(parseFloat);
return [tmp.shift(), tmp];
}).persist() : ml.randomSVMData(sc, N, D, seed, P).persist();