console.log=function(d){
log_file.write(util.format(d)+'\n');
log_stdout.write(util.format(d)+'\n');
};
fs.writeFile(filePath, doc, function(e){//会先清空原先的内容
if(e) throw e;
console.log(util.format('Snapshot file [%s] is created!', filePath));
});
analyzer.on('declaration', function(rule, property, value) {
if (value.indexOf('!important') > -1) {
analyzer.incrMetric('importants');
analyzer.addOffender('importants', format('%s {%s: %s}', rule.selectors.join(', '), property, value));
}
});
function create_filename(file) {
var ext = path.extname(file.name);
var basename = path.basename(file.name, ext);
var filename = util.format('%s_%d%s', basename, Date.now(), ext);
return filename;
}
return util.format(xray.templates.map, opts.vector_layers.map(function(layer){
var rgb = xray.color(layer.id).join(',');
return util.format(xray.templates.layer, layer.id, rgb, rgb, rgb, rgb, rgb, layer.id, layer.id)
}).join('\n'));
downloader.prototype.downloadItAct = function(urlinfo){
var spiderCore = this.spiderCore;
var self = this;
var timeOuter = false;
var pageLink = urlinfo['url'];
if(urlinfo['redirect'])pageLink = urlinfo['redirect'];
var useProxy = false;
if(urlinfo['urllib']&&spiderCore.settings['use_proxy']===true){
if(spiderCore.spider.getDrillerRule(urlinfo['urllib'],'use_proxy')===true)useProxy=true;
}
if(useProxy){
var proxyRouter = spiderCore.settings['proxy_router'].split(':');
var __host = proxyRouter[0];
var __port = proxyRouter[1];
var __path = pageLink;
}else{
var urlobj = urlUtil.parse(pageLink);
var __host = urlobj['hostname'];
var __port = urlobj['port'];
var __path = urlobj['path'];
// var __path = pageLink;
}
var startTime = new Date();
var options = {
'host': __host,
'port': __port,
'path': __path,
'method': 'GET',
'headers': {
"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like NeoCrawler) Chrome/31.0.1650.57 Safari/537.36",
"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Encoding":"gzip",
"Accept-Language":"zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4",
"Referer":urlinfo['referer']||'',
"void-proxy":urlinfo['void_proxy']?urlinfo['void_proxy']:"",
"Cookie":this.transCookieKvPair(urlinfo['cookie'])
}
};
logger.debug(util.format('Request start, %s',pageLink));
var req = http.request(options, function(res) {
logger.debug(util.format('Response, %s',pageLink));
var result = {
"remote_proxy":res.headers['remoteproxy'],
"drill_count":0,
"cookie":res.headers['Cookie'],
"url":urlinfo['url'],
//"url":res.req.path,
//"statusCode":res.statusCode,
"origin":urlinfo
};
if(result['url'].startsWith('/'))result['url'] = urlUtil.resolve(pageLink,result['url']);
result['statusCode'] = res.statusCode;
if(parseInt(res.statusCode)==301||parseInt(res.statusCode)==302){
if(res.headers['location']){
result['origin']['redirect'] = urlUtil.resolve(pageLink,res.headers['location']);
logger.debug(pageLink+' 301 Moved Permanently to '+res.headers['location']);
}
}
var compressed = /gzip|deflate/.test(res.headers['content-encoding']);
var bufferHelper = new BufferHelper();
// res.setEncoding('utf8');
res.on('data', function (chunk) {
bufferHelper.concat(chunk);
});
res.on('end', function (chunk) {
self.timeout_count--;
if(timeOuter){
clearTimeout(timeOuter);
timeOuter = false;
}
result["cost"] = (new Date()) - startTime;
logger.debug('download '+pageLink+', cost:'+result["cost"]+'ms');
var page_encoding = urlinfo['encoding'];
if(page_encoding==='auto'){
page_encoding = self.get_page_encoding(res.headers);
}
page_encoding = page_encoding.toLowerCase().replace('\-','')
if(!compressed || typeof unzip == 'undefined'){
if(urlinfo['format']=='binary'){
result["content"] = bufferHelper.toBuffer();
}else{
result["content"] = iconv.decode(bufferHelper.toBuffer(),page_encoding);//page_encoding
}
spiderCore.emit('crawled',result);
}else{
unzip(bufferHelper.toBuffer(), function(err, buff) {
if (!err && buff) {
if(urlinfo['format']=='binary'){
result["content"] = buff;
}else{
result["content"] = iconv.decode(buff,page_encoding);
}
spiderCore.emit('crawled',result);
}else{
spiderCore.emit('crawling_failure',urlinfo,'unzip failure');
}
});
}
});
});
timeOuter = setTimeout(function(){
if(req){
logger.error('Cost '+((new Date())-startTime)+'ms download timeout, '+pageLink);
req.abort();
req=null;
spiderCore.emit('crawling_failure',urlinfo,'download timeout');
if(self.timeout_count++>spiderCore.settings['spider_concurrency']){logger.fatal('too much timeout, exit.');process.exit(1);}
}
},spiderCore.settings['download_timeout']*1000);
req.on('error', function(e) {
logger.error('problem with request: ' + e.message+', url:'+pageLink);
if(timeOuter){
clearTimeout(timeOuter);
timeOuter = false;
}
if(req){
req.abort();
req = null;
spiderCore.emit('crawling_failure',urlinfo,e.message);
}
});
req.end();
}
sails.after('lifted', function () {
console.log();
sails.log.warn(util.format('Ignoring invalid blueprint prefix configured for controller `%s`.', globalId));
sails.log.warn('`prefix` should be a string, e.g. "/api/v1".');
STRINGFILE.logMoreInfoLink(STRINGFILE.get('links.docs.config.blueprints'), sails.log.warn);
});
exports['FWAPI-237: Ignore rules that don\'t change'] = function (t) {
var vm1 = helpers.generateVM({ uuid: helpers.uuidNum(1) });
var vm2 = helpers.generateVM({ uuid: helpers.uuidNum(2) });
var vm3 = helpers.generateVM({ uuid: helpers.uuidNum(3) });
var vm4 = helpers.generateVM({
owner_uuid: mod_uuid.v4(),
uuid: helpers.uuidNum(4)
});
var allVMs = [ vm1, vm2, vm3, vm4 ];
var updateUUID = mod_uuid.v4();
var payload = {
rules: [
{
rule: 'FROM any TO all vms ALLOW icmp TYPE 8 CODE 0',
enabled: true,
global: true
},
{
uuid: updateUUID,
owner_uuid: vm1.owner_uuid,
rule: util.format('FROM any TO vm %s ALLOW udp PORT 514',
vm1.uuid),
enabled: true
}
],
vms: allVMs
};
var expRules = [clone(payload.rules[0]), clone(payload.rules[1])];
var updatePayload = {
rules: [
{
uuid: updateUUID,
owner_uuid: vm1.owner_uuid,
rule: util.format('FROM any TO vm %s ALLOW udp PORT 515',
vm1.uuid),
enabled: true
}
],
vms: allVMs
};
async.series([
function (cb) {
fw.add(payload, function (err, res) {
t.ifError(err);
if (err) {
return cb();
}
for (var i in res.rules) {
t.ok(res.rules[i].uuid, 'rule has a uuid');
expRules[i].uuid = res.rules[i].uuid;
t.ok(res.rules[i].version, 'rule has a version');
expRules[i].version = res.rules[i].version;
}
t.deepEqual(helpers.sortRes(res), helpers.sortRes({
rules: clone(expRules),
vms: [ vm1.uuid, vm2.uuid, vm3.uuid, vm4.uuid ].sort()
}), 'rules returned');
cb();
});
}, function (cb) {
helpers.fwGetEquals(t, expRules[0], cb);
}, function (cb) {
helpers.fwListEquals(t, expRules, cb);
}, function (cb) {
var globalPayload = {
rules: [ expRules[0] ],
vms: allVMs
};
fw.update(globalPayload, function (err, res) {
t.ifError(err);
if (err) {
return cb();
}
// Nothing's changed in the global rule, so nothing should
// be updated.
t.deepEqual(helpers.sortRes(res), { vms: [], rules: [] },
'rules returned');
cb();
});
}, function (cb) {
var globalPayload = {
rules: [ expRules[0] ],
vms: allVMs
};
globalPayload.rules[0].rule =
'FROM any TO all vms ALLOW icmp TYPE all';
fw.update(globalPayload, function (err, res) {
t.ifError(err);
if (err) {
return cb();
}
// This global rule affects all VMs, so they all should be updated.
t.deepEqual(helpers.sortRes(res), {
vms: [ vm1.uuid, vm2.uuid, vm3.uuid, vm4.uuid ],
rules: globalPayload.rules
}, 'rules returned');
cb();
});
}, function (cb) {
fw.update(updatePayload, function (err, res) {
t.ifError(err);
if (err) {
return cb();
}
// This rule should only affect the VM that it mentions.
t.deepEqual(helpers.sortRes(res), {
vms: [ vm1.uuid ],
rules: updatePayload.rules
}, 'rules returned');
cb();
});
}, function (cb) {
helpers.fwGetEquals(t, updatePayload.rules[0], cb);
}, function (cb) {
fw.update(updatePayload, function (err, res) {
t.ifError(err);
if (err) {
return cb();
}
// Repeating the same update again shouldn't do anything.
t.deepEqual(helpers.sortRes(res), { vms: [], rules: [] },
'rules returned');
cb();
});
}, function (cb) {
var vm2updated = clone(vm2);
vm2updated.nics[0].ip = '1.2.3.4';
var vmPayload = {
localVMs: [ vm2updated ],
vms: allVMs
};
fw.update(vmPayload, function (err, res) {
t.ifError(err);
if (err) {
return cb();
}
// None of the rules are affected by the VM's IP changing, so only
// the modified VM is updated.
t.deepEqual(helpers.sortRes(res), {
vms: [ vm2updated.uuid ],
rules: []
}, 'rules returned');
cb();
});
}, function (cb) {
var addPayload = {
rules: [ {
uuid: mod_uuid.v4(),
rule: 'FROM all vms TO all vms ALLOW tcp PORT all',
enabled: true,
owner_uuid: vm1.owner_uuid
} ],
vms: allVMs,
allowAdds: true
};
// Add a rule that affects all VMs owned by the owner of VMs 1, 2, 3
fw.update(addPayload, function (err, res) {
t.ifError(err);
if (err) {
return cb();
}
t.deepEqual(helpers.sortRes(res), {
vms: [ vm1.uuid, vm2.uuid, vm3.uuid ],
rules: addPayload.rules
}, 'rules returned');
cb();
});
}, function (cb) {
var vm2updated = clone(vm2);
vm2updated.nics[0].ip = '1.2.3.5';
var vmPayload = {
localVMs: [ vm2updated ],
vms: allVMs
};
fw.update(vmPayload, function (err, res) {
t.ifError(err);
if (err) {
return cb();
}
// Now that there's a rule that affects all VMs of one owner,
// updating the IP address affects those VMs, so they need their
// active firewall rules set updated.
t.deepEqual(helpers.sortRes(res), {
vms: [ vm1.uuid, vm2.uuid, vm3.uuid ],
rules: []
}, 'rules returned');
cb();
});
}, function (cb) {
var vm4updated = clone(vm4);
vm4updated.nics[0].ip = '1.2.3.20';
var vmPayload = {
localVMs: [ vm4updated ],
vms: allVMs
};
fw.update(vmPayload, function (err, res) {
t.ifError(err);
if (err) {
return cb();
}
// Updating the IP address of the VM that has a different owner
// shouldn't do anything to other VMs.
t.deepEqual(helpers.sortRes(res), {
vms: [ vm4updated.uuid ],
rules: []
}, 'rules returned');
cb();
});
}
], function () {
t.done();
});
};
exports['update non-existent rule'] = function (t) {
var vm = helpers.generateVM();
var payload = {
rules: [
{
owner_uuid: vm.owner_uuid,
rule: util.format('FROM vm %s TO any BLOCK tcp PORT 8080',
vm.uuid),
uuid: mod_uuid.v4(),
enabled: true
}
],
vms: [vm]
};
var expRules = [clone(payload.rules[0])];
var vmsEnabled = {};
var allRules;
async.series([
function (cb) {
fw.update(payload, function (err) {
t.ok(err, 'Error returned');
return cb();
});
}, function (cb) {
payload.allowAdds = true;
fw.update(payload, function (err, res) {
t.ifError(err);
if (err) {
return cb();
}
t.ok(res.rules[0].uuid, 'rule has a uuid');
expRules[0].uuid = res.rules[0].uuid;
t.ok(res.rules[0].version, 'rule has a version');
expRules[0].version = res.rules[0].version;
t.deepEqual(res, {
rules: expRules,
vms: [ vm.uuid ]
}, 'rules returned');
allRules = helpers.defaultZoneRules(vm.uuid);
allRules[vm.uuid].out.tcp = [
helpers.blockPortOutTCP('any', 8080)
];
t.deepEqual(helpers.zoneIPFconfigs(4), allRules,
'IPv4 firewall rules correct');
t.deepEqual(helpers.zoneIPFconfigs(6), allRules,
'IPv6 firewall rules correct');
vmsEnabled[vm.uuid] = true;
t.deepEqual(helpers.getIPFenabled(), vmsEnabled,
'ipf enabled in VMs');
cb();
});
}, function (cb) {
helpers.fwGetEquals(t, expRules[0], cb);
}
], function () {
t.done();
});
};
var crawlerSubscribe = function(entry) {
// 所有栏目列表 http://r.inews.qq.com/getCatList?qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&appver=16_android_3.2.1&version=0
// 获取某一栏目文章ids http://r.inews.qq.com/getSubNewsIndex?qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&format=json&apptype=android&chlid=1250&appver=16_android_3.2.1
// 获取指定ids文章 http://r.inews.qq.com/getSubNewsListItems?uid=22c4d53a-7d52-4f50-9185-90a77c7b80b0&qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&ids=20131107A000D700%2C20131106A000DE00%2C20131106A000DD00%2C20131105A000DS00%2C20131105A000DR00%2C20131104A000CW00%2C20131101A000DZ00%2C20131031A000BZ00%2C20131030A000BV00%2C20131029A000C600%2C20131028A0017800%2C20131025A0019S00%2C20131024A000CG00%2C20131023A000CM00%2C20131022A000I400%2C20131022A000I300%2C20131022A0007700%2C20131018A000EB00%2C20131017A0015800%2C20131016A000AX00&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&appver=16_android_3.2.1
// 获取某篇文章内容 http://r.inews.qq.com/getSubNewsContent?id=20131129A000H600&qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&appver=16_android_3.2.1
var subscribeNewsIdsLink = 'http://r.inews.qq.com/getSubNewsIndex?qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&format=json&apptype=android&chlid=%s&appver=16_android_3.2.1';
var subscribeNewsListLink = 'http://r.inews.qq.com/getSubNewsListItems?uid=22c4d53a-7d52-4f50-9185-90a77c7b80b0&qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&ids=%s&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&appver=16_android_3.2.1'
var url = '';
if(entry.ids) {
var ids = '';
var i = 0;
for(i=0; (i<20) && (i<entry.ids.length); i++) {
if(i > 0) {
ids = ids + ','
}
ids = ids + entry.ids[i].id;
}
url = util.format(subscribeNewsListLink, ids);
entry.ids = entry.ids.slice(i);
}else {
url = util.format(subscribeNewsIdsLink, entry.id);
}
var req = {uri: url, method: "GET", headers: headers};
if(proxyEnable) {
req.proxy = proxyUrl;
}
request(req, function (err, res, body) {
var json = data2Json(err, res, body);
if(!json || (json.ret != 0)) {
console.log("hzfdbg file[" + __filename + "]" + " crawlerSubscribe():JSON.parse() error");
return;
}
if(entry.ids) {
setTimeout(function() {
crawlerSubscribe(entry);
}, 100);
}else if(json.ids) {
entry.ids = json.ids;
setTimeout(function() {
crawlerSubscribe(entry);
}, 100);
return;
}
var newsList = json.newslist;
if((!newsList) || (!newsList.length) || (newsList.length <= 0)) {
console.log("hzfdbg file[" + __filename + "]" + " crawlerSubscribe():newsList empty in url " + url);
return;
}
newsList.forEach(function(newsEntry) {
if(!newsEntry.title || !newsEntry.id) {
return;
}
var tags = entry.tags;
for(var i = 0; i < tags.length; i++) {
if (newsEntry.title.indexOf(tags[i]) !== -1 || entry.all) {
newsEntry.tagName = tags[i];
newsEntry.subscribeName = entry.name;
News.findOne(genQqFindCmd(site,newsEntry), function(err, result) {
if(err || result) {
return;
}
console.log("hzfdbg file[" + __filename + "]" + " crawlerSubscribe():["+newsEntry.tagName+"]"+newsEntry.title+",docid="+newsEntry.id);
startGetDetail.emit('startGetSubscribeNewsDetail', newsEntry);
}); // News.findOne
break;
}
}//for
});//forEach
});//request
}
fw.update(payload, function (err, res) {
t.ok(err, 'error returned');
t.equal(err.message, util.format('Could not find VM "%s" in VM list',
vm.uuid), 'error message');
t.done();
});
var getNewsDetail = function(entry) {
var detailLink = 'http://inews.qq.com/getQQNewsNormalHtmlContent?id=%s';
var docid = util.format("%s",entry.id);
var url = util.format(detailLink, docid);
var req = {uri: url, method: "GET", headers: headers};
if(proxyEnable) {
req.proxy = proxyUrl;
}
request(req, function (err, res, body) {
var json = data2Json(err, res, body);
if((!json) || (json.ret != 0)) {
console.log("hzfdbg file[" + __filename + "]" + " getNewsDetail():ret="+json.ret);
return;
}
var jObj = json;
var obj = entry;
News.findOne(genQqFindCmd(site, entry), function(err, result) {
if(err || result) {
return;
}
obj.docid = encodeDocID(site, docid);
obj.site = site;
obj.body = '';
obj.img = [];
jObj.content.forEach(function(item) {
if(item.type == 1) { //text
obj.body += item.value;
}
else if(item.type == 2) { //pic
obj.body += genLazyLoadHtml(entry.tagName, item.value);
obj.img[obj.img.length] = item.value;
}
else if(item.type == 3) { //video
obj.body += genLazyLoadHtml(entry.tagName, item.value.img);
obj.img[obj.img.length] = item.value.img;
}
});
obj.video = [];
obj.link = "";
if(entry.url) {
obj.link = entry.url; // http://view.inews.qq.com/a/NEW2013050300143202
}else if(entry.surl) {
obj.link = entry.surl; // http://view.inews.qq.com/a/NEW2013050300143202
}else if(jObj.url) {
obj.link = jObj.url; // http://view.inews.qq.com/a/NEW2013050300143202
}else if(jObj.surl) {
obj.link = jObj.surl; // http://view.inews.qq.com/a/NEW2013050300143202
}else {
obj.link = util.format("http://view.inews.qq.com/a/%s", docid);
}
obj.title = entry.title;
obj.ptime = entry.time;
obj.time = new Date(entry.time);
obj.marked = obj.body;
obj.created = new Date();
obj.views = 1;
obj.tags = entry.tagName;
obj.digest = genDigest(obj.body);
if(!obj.digest) {
obj.digest = obj.body;
}
obj.cover = entry.thumbnails[0];
if (obj.img[0]) {
obj.cover = obj.img[0];
}
News.insert(obj, function (err, result) {
if(err) {
console.log("hzfdbg file[" + __filename + "]" + " getNewsDetail(), News.insert():error " + err);
}
});
});//News.findOne
});//request
};
var getPhotoDetail = function(entry) {
// http://inews.qq.com/getQQNewsSimpleHtmlContent?id=PIC2013061200601000&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&sceneid=00000&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&chlid=news_photo&appver=16_android_2.7.0
var photoDetailLink = "http://inews.qq.com/getQQNewsSimpleHtmlContent?id=%s&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&sceneid=00000&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&chlid=news_photo&appver=16_android_2.7.0";
var docid = util.format("%s",entry.id);
var url = util.format(photoDetailLink, docid);
var req = {uri: url, method: "GET", headers: headers};
if(proxyEnable) {
req.proxy = proxyUrl;
}
request(req, function (err, res, body) {
var json = data2Json(err, res, body);
if((!json) || (json.ret != 0)) {
console.log("hzfdbg file[" + __filename + "]" + " getPhotoDetail():ret="+json.ret);
return;
}
var jObj = json;
var obj = entry;
News.findOne(genQqFindCmd(site, entry), function(err, result) {
if(err || result) {
return;
}
obj.docid = encodeDocID(site, docid);
obj.site = site;
obj.body = jObj.intro + jObj.content.text;
obj.img = [];
var attribute = jObj.attribute;
var img_keys = _.keys(attribute);
img_keys.forEach(function(key){
obj.img[obj.img.length] = attribute[key].url;
var imgHtml = attribute[key].desc + genLazyLoadHtml(attribute[key].desc, attribute[key].url);
obj.body = obj.body.replace("<!--" + key + "-->", imgHtml); // <!--IMG_4-->
});
obj.video = [];
obj.link = "";
if(entry.url) {
obj.link = entry.url; // http://view.inews.qq.com/a/TPC2013061100203800
}else if(entry.surl) {
obj.link = entry.surl; // http://view.inews.qq.com/a/TPC2013061100203800
}else if(jObj.url) {
obj.link = jObj.url; // http://view.inews.qq.com/a/TPC2013061100203800
}else if(jObj.surl) {
obj.link = jObj.surl; // http://view.inews.qq.com/a/TPC2013061100203800
}else {
obj.link = util.format("http://view.inews.qq.com/a/%s", docid);
}
obj.title = entry.title;
obj.ptime = entry.time;
obj.time = new Date(entry.time);
obj.marked = obj.body;
obj.created = new Date();
obj.views = 1;
obj.tags = entry.tagName;
obj.digest = genDigest(obj.body);
obj.cover = entry.thumbnails[0];
if (obj.img[0]) {
obj.cover = obj.img[0];
}
News.insert(obj, function (err, result) {
if(err) {
console.log("hzfdbg file[" + __filename + "]" + " getPhotoDetail(), News.insert():error " + err);
}
});
});//News.findOne
});//request
};
var getSubscribeNewsDetail = function(entry) {
// http://r.inews.qq.com/getSubNewsContent?id=20131129A000H600&qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&appver=16_android_3.2.1
var subscribeNewsDetailLink = 'http://r.inews.qq.com/getSubNewsContent?id=%s&qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&appver=16_android_3.2.1';
var docid = util.format("%s",entry.id);
var url = util.format(subscribeNewsDetailLink, docid);
var req = {uri: url, method: "GET", headers: headers};
if(proxyEnable) {
req.proxy = proxyUrl;
}
request(req, function (err, res, body) {
var json = data2Json(err, res, body);
if((!json) || (json.ret != 0)) {
console.log("hzfdbg file[" + __filename + "]" + " getSubscribeNewsDetail():ret="+json.ret);
return;
}
var jObj = json;
var obj = entry;
News.findOne(genQqFindCmd(site, entry), function(err, result) {
if(err || result) {
return;
}
obj.docid = encodeDocID(site, docid);
obj.site = site;
obj.body = jObj.content.text;
obj.img = [];
for(key in jObj.attribute) {
var html = genLazyLoadHtml(entry.title, jObj.attribute[key].url);
if(jObj.attribute[key].desc) {
html = html + jObj.attribute[key].desc + '<br/>';
}
obj.img[obj.img.length] = jObj.attribute[key].url;
obj.body = obj.body.replace(util.format('<!--%s-->', key), html);
}
obj.video = [];
obj.link = '';
if(entry.url) {
obj.link = entry.url;
}else {
obj.link = util.format("http://view.inews.qq.com/a/%s", entry.id);
}
obj.title = entry.title;
obj.ptime = entry.time;
obj.time = new Date(entry.time);
obj.marked = obj.body;
obj.created = new Date();
obj.views = 1;
obj.tags = entry.tagName;
obj.digest = genDigest(obj.body);
obj.cover = '';
if(entry.thumbnails_qqnews && entry.thumbnails_qqnews[0]) {
obj.cover = entry.thumbnails_qqnews[0];
}else if(entry.chlsicon) {
obj.cover = entry.chlsicon;
}else if(entry.chlicon) {
obj.cover = entry.chlicon;
}else if(jObj.card && jObj.card.icon) {
obj.cover = jObj.card.icon;
}else if(obj.img) {
obj.cover = obj.img[0];
}else {
obj.cover = '';
}
News.insert(obj, function (err, result) {
if(err) {
console.log("hzfdbg file[" + __filename + "]" + " getSubscribeNewsDetail(), News.insert():error " + err);
}
});
});//News.findOne
});//request
};
function logBadResponseError(err$2) {
logger.error('CommandHandler', format(badResponseFormat, command.message, inspect(res)));
}
/**
* This function uses the objectdiff library to diff two objects or
* literals and generates a nice-looking diff (inspired by file diffs)
* from the result.
*
* @param a {*} diff operand 1, can be object or literal.
* @param b {*} diff operand 2, can be object or literal.
* @param colors {boolean} whether to colorize the output.
* @returns {Array} empty if equal, otherwise a list of changes.
* @private
*/
function _generateObjectDiff(a, b, colors) {
var result = [];
var numberPat = /^\d+$/;
// Format strings
var changeStr = ['* %s = %s -> %s',
'\x1B[36m* %s\x1B[0m = \x1B[31m%s\x1B[0m -> \x1B[32m%s\x1B[0m'];
var changeStr2 = ['* %s -> %s',
'\x1B[36m*\x1B[0m \x1B[31m%s\x1B[0m -> \x1B[32m%s\x1B[0m'];
var removeStr = ['- %s = %s',
'\x1B[31m- %s\x1B[0m = \x1B[31m%s\x1B[0m'];
var addStr = ['+ %s = %s',
'\x1B[32m+ %s\x1B[0m = \x1B[32m%s\x1B[0m'];
var idx = colors ? 1 : 0;
// Traverse the diff object to find all changes.
function traverse(node, path) {
var keys, i;
if (node.changed === 'object change') {
keys = Object.keys(node.value);
for (i = 0; i < keys.length; i++) {
var newPath;
// If the key is a number, it's probably an array index
if (numberPat.test(keys[i])) {
newPath = '[' + keys[i] + ']';
if (path) {
newPath = path + newPath;
}
} else {
newPath = path ? path + '.' + keys[i] : keys[i];
}
traverse(node.value[keys[i]], newPath);
}
} else if (node.changed === 'primitive change') {
result.push(_lineBreak(util.format(changeStr[idx]
, path
, util.inspect(node.removed)
, util.inspect(node.added))));
} else if (node.changed === 'removed') {
result.push(_lineBreak(util.format(removeStr[idx]
, path
, util.inspect(node.value))));
} else if (node.changed === 'added') {
result.push(_lineBreak(util.format(addStr[idx]
, path
, util.inspect(node.value))));
}
}
// objectdiff only supports comparing two objects.
if (a instanceof Object && b instanceof Object) {
var diff = objectDiff.diff(a, b);
traverse(diff, '');
} else {
if (a !== b) {
result.push(_lineBreak(util.format(changeStr2[idx]
, util.inspect(a)
, util.inspect(b))));
}
}
return result;
}
var conditions = _.map(this.parseConditionObject(smth.conditions), function generateSql(condition) {
return util.format("%s#>>'{%s}' = '%s'",
_.first(condition.path),
_.tail(condition.path).join(','),
condition.value);
});
return promise.then(pass_results => {
var proposalResponses = pass_results[0];
var proposal = pass_results[1];
var header = pass_results[2];
var goodCount = 0;
for(var i in proposalResponses) {
let one_good = false;
let proposal_response = proposalResponses[i];
if( proposal_response.response && proposal_response.response.status === 200) {
console.log('transaction proposal has response status of good');
one_good = chain.verifyProposalResponse(proposal_response);
if(one_good) {
console.log(' transaction proposal signature and endorser are valid');
goodCount++;
}
} else {
//console.error('transaction proposal was bad');
}
//all_good = all_good & one_good;
}
var all_good = goodCount >= +config.endorsementPolicy.split("/")[0];
if (all_good) {
// check all the read/write sets to see if the same, verify that each peer
// got the same results on the proposal
all_good = chain.compareProposalResponseResults(proposalResponses);
console.log('compareProposalResponseResults exection did not throw an error');
if(all_good){
console.log(' All proposals have a matching read/writes sets');
}
else {
console.error(' All proposals do not have matching read/write sets');
}
}
if (all_good) {
// check to see if all the results match
console.log(util.format('Successfully sent Proposal and received ProposalResponse: Status - %s, message - "%s", metadata - "%s", endorsement signature: %s', proposalResponses[0].response.status));//, proposalResponses[0].response.message, proposalResponses[0].response.payload, proposalResponses[0].endorsement.signature));
var request = {
proposalResponses: proposalResponses,
proposal: proposal,
header: header
};
// set the transaction listener and set a timeout of 30sec
// if the transaction did not get committed within the timeout period,
// fail the test
var deployId = tx_id.getTransactionID();
var eventPromises = [];
eventhubs.forEach((eh) => {
let txPromise = new Promise((resolve, reject) => {
let handle = setTimeout(reject, 120000);
eh.registerTxEvent(deployId.toString(),
(tx, code) => {
clearTimeout(handle);
eh.unregisterTxEvent(deployId);
if (code !== 'VALID') {
console.error('The balance transfer transaction was invalid, code = ' + code);
reject();
} else {
console.log('The balance transfer transaction has been committed on peer '+ eh.ep._endpoint.addr);
resolve();
}
},
(err) => {
clearTimeout(handle);
console.log('Successfully received notification of the event call back being cancelled for '+ deployId);
resolve();
}
);
});
eventPromises.push(txPromise);
});
var sendPromise = chain.sendTransaction(request);
return Promise.all([sendPromise].concat(eventPromises))
.then((results) => {
console.debug(' event promise all complete and testing complete');
return results[0]; // the first returned value is from the 'sendPromise' which is from the 'sendTransaction()' call
}).catch((err) => {
console.log(err);
console.error('Failed to send transaction and get notifications within the timeout period.');
throw new Error('Failed to send transaction and get notifications within the timeout period.');
});
} else {
console.error('Failed to send Proposal or receive valid response. Response null or status is not 200. exiting...');
throw new Error('Failed to send Proposal or receive valid response. Response null or status is not 200. exiting...');
}
}, (err) => {
var req = http.request(options, function(res) {
logger.debug(util.format('Response, %s',pageLink));
var result = {
"remote_proxy":res.headers['remoteproxy'],
"drill_count":0,
"cookie":res.headers['Cookie'],
"url":urlinfo['url'],
//"url":res.req.path,
//"statusCode":res.statusCode,
"origin":urlinfo
};
if(result['url'].startsWith('/'))result['url'] = urlUtil.resolve(pageLink,result['url']);
result['statusCode'] = res.statusCode;
if(parseInt(res.statusCode)==301||parseInt(res.statusCode)==302){
if(res.headers['location']){
result['origin']['redirect'] = urlUtil.resolve(pageLink,res.headers['location']);
logger.debug(pageLink+' 301 Moved Permanently to '+res.headers['location']);
}
}
var compressed = /gzip|deflate/.test(res.headers['content-encoding']);
var bufferHelper = new BufferHelper();
// res.setEncoding('utf8');
res.on('data', function (chunk) {
bufferHelper.concat(chunk);
});
res.on('end', function (chunk) {
self.timeout_count--;
if(timeOuter){
clearTimeout(timeOuter);
timeOuter = false;
}
result["cost"] = (new Date()) - startTime;
logger.debug('download '+pageLink+', cost:'+result["cost"]+'ms');
var page_encoding = urlinfo['encoding'];
if(page_encoding==='auto'){
page_encoding = self.get_page_encoding(res.headers);
}
page_encoding = page_encoding.toLowerCase().replace('\-','')
if(!compressed || typeof unzip == 'undefined'){
if(urlinfo['format']=='binary'){
result["content"] = bufferHelper.toBuffer();
}else{
result["content"] = iconv.decode(bufferHelper.toBuffer(),page_encoding);//page_encoding
}
spiderCore.emit('crawled',result);
}else{
unzip(bufferHelper.toBuffer(), function(err, buff) {
if (!err && buff) {
if(urlinfo['format']=='binary'){
result["content"] = buff;
}else{
result["content"] = iconv.decode(buff,page_encoding);
}
spiderCore.emit('crawled',result);
}else{
spiderCore.emit('crawling_failure',urlinfo,'unzip failure');
}
});
}
});
});
$("#divResults .clsResult").map(function() {
var row = $(this);
// Structured as per rawHtml found @ http://athlinks.com/time.aspx?eventid=115006&courseid=156044&genderpage=A14
var claimed = row.find("img").attr("src") === "/images/icoMemberSm.gif"
? 1
: 0;
var a = row.find(":nth-child(2)").text();
var g = row.find(":nth-child(3)").text();
var o = row.find(":nth-child(4)").text();
var ago = util.format("%s/%s/%s", a, g, o);
var athlete_row = row.find(":nth-child(5)");
var athlete_ids = _s
.trim(row.find("a").attr("href"), "http://athlinks.com/result/")
.split("/");
var athlinksId = athlete_ids[athlete_ids.length - 2];
var names = _s.capitalize(_s.humanize(athlete_row.text())).split(" ");
var firstName = _.first(names);
var lastName = _.rest(names, 1).join(" ");
var m_f = row.find(":nth-child(6)").text();
var age = row.find(":nth-child(7)").text();
var bib = row.find(":nth-child(8)").text();
var swimTime = row.find(":nth-child(9)").text();
var swimPace = row.find(":nth-child(10)").text();
var swimAgo = row.find(":nth-child(11)").text();
var cycleTime = row.find(":nth-child(12)").text();
var cycleSpeed = row.find(":nth-child(13)").text();
var cycleAgo = row.find(":nth-child(14)").text();
var runTime = row.find(":nth-child(15)").text();
var runPace = row.find(":nth-child(16)").text();
var runAgo = row.find(":nth-child(17)").text();
var finalTime = row.find(":nth-child(18)").text();
rows.push({
claimed: claimed,
ago: ago,
firstName: firstName,
lastName: lastName,
athlinksId: athlinksId,
m_f: m_f,
age: age,
bib: bib,
swimTime: swimTime,
swimPace: swimPace,
swimAgo: swimAgo,
t1: "",
cycleTime: cycleTime,
cycleSpeed: cycleSpeed,
cycleAgo: cycleAgo,
t2: "",
runTime: runTime,
runPace: runPace,
runAgo: _cleanupAgo(runAgo),
finalTime: finalTime
});
});
sails.after('lifted', function () {
console.log();
sails.log.warn(util.format('Invalid blueprint prefix ("%s") configured for controller `%s` (should start with a `/`).', originalPrefix, globalId));
sails.log.warn(util.format('For now, assuming you meant: "%s".', config.prefix));
STRINGFILE.logMoreInfoLink(STRINGFILE.get('links.docs.config.blueprints'), sails.log.warn);
});
var restify = function(app, model, opts) {
var postProcess, lean, filter, contextFilter, postCreate, postDelete,
usingExpress,
options = getDefaults(),
queryOptions = {
protected: ['skip', 'limit', 'sort', 'populate', 'select', 'lean',
'$and', '$or', 'query'],//H+ exposes OR, AND and WHERE methods
current: {}
};
for (var prop in opts) {
if (opts[prop] instanceof Array) {
options[prop] = [];
for (var index in opts[prop]) {
options[prop][index] = opts[prop][index];
}
} else {
options[prop] = opts[prop];
}
}
options.private = options.private ? options.private.split(',') : [];
options.protected = options.protected ?
options.protected.split(',') : [];
model.schema.eachPath(function(name, path) {
if (path.options.access) {
switch (path.options.access.toLowerCase()) {
case 'private':
options.private.push(name);
break;
case 'protected':
options.protected.push(name);
break;
}
}
});
if (options.exclude) {
options.private = options.exclude.split(',');
console.error('Exclude is deprecated. Use private instead');
}
lean = options.lean;
filter = new Filter(model, options.private, options.protected);
postProcess = options.postProcess || function() {
};
if (options.middleware) {
if (!(options.middleware instanceof Array)) {
var m = options.middleware;
options.middleware = [ m ];
}
}
function cleanQuery(req, res, next) {
queryOptions.current = {};
var err = null;
for (var key in req.query) {
if (queryOptions.protected.indexOf(key) !== -1) {
queryOptions.current[key] = req.query[key];
delete req.query[key];
} else if (!model.schema.paths.hasOwnProperty(key)) {
var keys = key.match('.') &&
!model.schema.paths.hasOwnProperty(key) ?
key.split('.') : false;
if(!keys || !model.schema.paths.hasOwnProperty(keys[0]) ||
!model.schema.paths[keys[0]].schema.paths.hasOwnProperty(keys[1]) ||
keys.length > 2) {
if(!model.schema.virtuals.hasOwnProperty(keys[0])) {
err = createError(400);
break;
}
}
}
}
if(err) {
onError(res,next,err);
} else {
next();
}
}
usingExpress = !options.restify;
function createError(status) {
var err = new Error(http.STATUS_CODES[status]);
err.status = status;
return err;
}
function onError(res, next, err) {
if(usingExpress) {
//next(err);
res.status(err.status).send(err.message);
} else {
res.send(err.status, err.message);
}
}
options.middleware.push(cleanQuery);
var outputFn = options.outputFn ?
options.outputFn : (options.restify ?
outputRestify : outputExpress);
if(undefined === app.delete) {
app.delete = app.del;
}
var apiUri = '%s%s/%s';
var modelName;
if (options.name) {
modelName = options.name;
}
else {
modelName = model.modelName;
}
modelName = options.plural ? inflection.pluralize(modelName)
: modelName;
modelName = options.lowercase === true ? modelName.toLowerCase()
: modelName;
var uri_item = util.format(apiUri, options.prefix, options.version,
modelName);
if (uri_item.indexOf('/:id') === -1) {
uri_item += '/:id';
}
var uri_items = uri_item.replace('/:id', '');
var uri_count = uri_items + '/count';
function buildQuery(query, req) {
var options = req.query,
excludedarr = filter.getExcluded(req.access);
var arr, i, re;
for (var key in options) {
if (excludedarr.indexOf(key) !== -1) {
// caller tries to query for excluded keys. for security
// reasons, we will skip the first -1 objects (to provoke
// an error) and immediately return;
return query.skip(-1);
}
query.where(key);
var value = options[key];
if ('~' === value[0]) {
re = new RegExp(value.substring(1), 'i');
query.where(key).regex(re);
} else if ('>' === value[0]) {
if ('=' === value[1]) {
query.gte(value.substr(2));
} else {
query.gt(value.substr(1));
}
} else if ('<' === value[0]) {
if ('=' === value[1]) {
query.lte(value.substr(2));
} else {
query.lt(value.substr(1));
}
} else if ('!' === value[0] && '=' === value[1]) { //H+ for !=
query.ne(value.substr(2));
} else if ('[' === value[0] && ']' === value[value.length - 1]) {
query.in(value.substr(1, value.length - 2).split(','));
} else {
query.equals(value);
}
}
//H+ exposes Query AND, OR and WHERE methods
if (queryOptions.current.query) {
query.where(JSON.parse(queryOptions.current.query,
jsonQueryParser));
}
//TODO - as introduction of QUERY param obsoletes need of $and, $or
if (queryOptions.current.$and) {
query.and(JSON.parse(queryOptions.current.$and, jsonQueryParser));
}
if (queryOptions.current.$or) {
query.or(JSON.parse(queryOptions.current.$or, jsonQueryParser));
}
//H+ exposes Query AND, OR methods
if (queryOptions.current.skip) {
query.skip(queryOptions.current.skip);
}
if (queryOptions.current.limit) {
query.limit(queryOptions.current.limit);
}
if (queryOptions.current.sort) {
query.sort(queryOptions.current.sort);
}
var selectObj = {root: {}};
if (queryOptions.current.select) {
if (queryOptions.current.select) {
arr = queryOptions.current.select.split(',');
for (i = 0; i < arr.length; ++i) {
if (arr[i].match(/\./)) {
var subSelect = arr[i].split('.');
if (!selectObj[subSelect[0]]) {
selectObj[subSelect[0]] = {};
//selectObj.root[subSelect[0]] = 1;
}
selectObj[subSelect[0]][subSelect[1]] = 1;
} else {
selectObj.root[arr[i]] = 1;
}
}
}
query = query.select(selectObj.root);
}
if (queryOptions.current.populate) {
arr = queryOptions.current.populate.split(',');
for (i = 0; i < arr.length; ++i) {
if (!_.isUndefined(selectObj[arr[i]]) &&
!_.isEmpty(selectObj.root)) {
selectObj.root[arr[i]] = 1;
}
query = query.populate(arr[i], selectObj[arr[i]]);
}
query.select(selectObj.root);
}
return query;
}
//H+ - JSON query param parser
//TODO - improve to serve recursive logical operators
function jsonQueryParser(key, value) {
if (_.isString(value)) {
if ('~' === value[0]) { //parse RegExp
return new RegExp(value.substring(1), 'i');
} else if ('>' === value[0]) {
if ('=' === value[1]) {
return {$gte: value.substr(2)};
} else {
return {$gt: value.substr(1)};
}
} else if ('<' === value[0]) {
if ('=' === value[1]) {
return {$lte: value.substr(2)};
} else {
return {$lt: value.substr(1)};
}
} else if ('!' === value[0] && '=' === value[1]) {
return {$ne: value.substr(2)};
}
} else if (_.isArray(value)) {
if (model.schema.paths.hasOwnProperty(key)) {
return {$in: value};
}
}
return value;
}
//H+ - JSON query param parser
function ensureContentType(req, res, next) {
var ct = req.headers['content-type'];
if (-1 === ct.indexOf('application/json')) {
onError(res,next,createError(400));
} else {
next();
}
}
function createSingleObject(body) {
for (var key in body) {
var path = model.schema.path(key);
if (typeof path === 'undefined') {
continue;
}
if (path.caster !== undefined) {
if (path.caster.instance === 'ObjectID') {
if (_.isArray(body[key])) {
for (var k = 0; k < body[key].length; ++k) {
if (typeof body[key][k] === 'object') {
body[key][k] = body[key][k]._id;
}
}
} else if ((typeof body[key] === 'object') &&
(body[key] !== null)) {
body[key] = body[key]._id;
}
}
} else if ((path.instance === 'ObjectID') &&
(typeof body[key] === 'object') &&
(body[key] !== null)) {
body[key] = body[key]._id;
}
}
}
function createObject(req, res, next) {
if (!req.body) {
onError(res,next,createError(400));
return;
}
var filterOpts = { access: req.access };
req.body = filter.filterObject(req.body, filterOpts);
if (model.schema.options._id) {
delete req.body._id;
}
if (model.schema.options.versionKey) {
delete req.body[model.schema.options.versionKey];
}
var key, path;
if (_.isArray(req.body)) {
for (var i = 0; i < req.body.length; ++i) {
createSingleObject(req.body[i]);
}
} else {
createSingleObject(req.body);
}
model.create(req.body, function(err, item) {
if (err) {
res.setHeader('Content-Type', 'application/json');
var forwardErr = new Error(JSON.stringify(err));
forwardErr.status = 400;
onError(res,next,forwardErr);
} else {
var result = null;
if (_.isArray(req.body)) {
var items = Array.prototype.slice.call(arguments, 1);
result = filter.filterObject(items, filterOpts);
} else {
result = filter.filterObject(item, filterOpts);
}
postCreate(res, result, function(err) {
if (err) {
onError(res,next,createError(400));
}
else {
outputFn(res, result);
next();
}
});
}
});
}
function modifyObject(req, res, next) {
var byId = {};
byId[options.idProperty || '_id'] = req.params.id;
if (!req.body) {
onError(res,next,createError(400));
return;
}
var filterOpts = { access: req.access };
req.body = filter.filterObject(req.body, filterOpts);
delete req.body._id;
if (model.schema.options.versionKey) {
delete req.body[model.schema.options.versionKey];
}
for (var key in req.body) {
var path = model.schema.path(key);
if (typeof path === 'undefined') {
continue;
}
if (path.caster !== undefined) {
if (path.caster.instance === 'ObjectID') {
if (_.isArray(req.body[key])) {
for (var j = 0; j < req.body[key].length; ++j) {
if (typeof req.body[key][j] === 'object') {
req.body[key][j] = req.body[key][j]._id;
}
}
} else if ((typeof req.body[key] === 'object') &&
(req.body[key] !== null)) {
req.body[key] = req.body[key]._id;
}
}
} else if ((path.instance === 'ObjectID') &&
(typeof req.body[key] === 'object') &&
(req.body[key] !== null)) {
req.body[key] = req.body[key]._id;
}
}
if (options.findOneAndUpdate) {
contextFilter(model, req, function(filteredContext) {
filteredContext.findOneAndUpdate(byId, req.body, {},
function(err, item) {
if (err || !item) {
onError(res,next,createError(404));
} else {
item = filter.filterObject(item, filterOpts);
outputFn(res, item);
next();
}
});
});
} else {
contextFilter(model, req, function(filteredContext) {
filteredContext.findOne(byId, function(err, doc) {
if (err || !doc) {
onError(res,next,createError(404));
} else {
for (var key in req.body) {
doc[key] = req.body[key];
}
doc.save(function(err, item) {
if (err) {
onError(res,next,createError(404));
} else {
item = filter.filterObject(item, filterOpts);
outputFn(res, item);
next();
}
});
}
});
});
}
}
var write_middleware = options.middleware.slice(-1);
if (options.prereq) {
var allowMW = permissions.allow(usingExpress, options.prereq);
write_middleware = [allowMW].concat(write_middleware);
}
var delete_middleware = write_middleware.slice(0);
if (options.access) {
var accessMW = permissions.access(options.access);
options.middleware.push(accessMW);
write_middleware.push(accessMW);
}
write_middleware.push(ensureContentType);
if (options.contextFilter) {
contextFilter = options.contextFilter;
}
else {
contextFilter = function(model, req, done) {
done(model);
};
}
if (options.postCreate) {
postCreate = options.postCreate;
}
else {
postCreate = function(res, result, done) {
done();
};
}
if (options.postDelete) {
postDelete = options.postDelete;
}
else {
postDelete = function(res, result, done) {
done();
};
}
app.get(uri_items, options.middleware, function(req, res, next) {
contextFilter(model, req, function(filteredContext) {
buildQuery(filteredContext.find(), req).lean(lean)
.exec(function(err, items) {
if (err) {
onError(res,next,createError(400));
} else {
var populate = queryOptions.current.populate,
opts = {
populate: populate,
access: req.access
};
items = filter.filterObject(items, opts);
outputFn(res, items);
next();
}
});
});
}, postProcess);
app.get(uri_count, options.middleware, function(req, res, next) {
contextFilter(model, req, function(filteredContext) {
buildQuery(filteredContext.count(), req).exec(function(err, count) {
if (err) {
onError(res,next,createError(400));
} else {
outputFn(res, { count: count });
next();
}
});
});
}, postProcess);
app.post(uri_items, write_middleware, createObject, postProcess);
if (!options.strict) {
app.put(uri_items, write_middleware, createObject, postProcess);
}
if (!options.strict) {
// TODO: reconsider this function
app.delete(uri_items, delete_middleware, function(req, res, next) {
contextFilter(model, req, function(filteredContext) {
var results, error;
buildQuery(filteredContext.find(), req).remove(function(err) {
if (err) {
onError(res,next,createError(400));
} else {
if(usingExpress) {
res.status(200).end();
} else {
res.send(200);
}
next();
}
});
});
}, postProcess);
}
app.get(uri_item, options.middleware, function(req, res, next) {
var byId = {};
byId[options.idProperty || '_id'] = req.params.id;
contextFilter(model, req, function(filteredContext) {
buildQuery(filteredContext.findOne().and(byId), req).lean(lean)
.findOne(function(err, item) {
if (err || !item) {
onError(res,next,createError(404));
} else {
var populate = queryOptions.current.populate,
opts = {
populate: populate,
access: req.access
};
item = filter.filterObject(item, opts);
outputFn(res, item);
next();
}
});
});
}, postProcess);
if (!options.strict) {
// TODO: POST (create) doesn't make sense here
app.post(uri_item, write_middleware, modifyObject, postProcess);
}
app.put(uri_item, write_middleware, modifyObject, postProcess);
app.delete(uri_item, delete_middleware, function(req, res, next) {
var byId = {};
byId[options.idProperty || '_id'] = req.params.id;
if (options.findOneAndUpdate) {
contextFilter(model, req, function(filteredContext) {
var results, error;
filteredContext.find().and(byId).findOneAndRemove(
function(err, result) {
if (err || !result) {
onError(res,next,createError(404));
} else {
postDelete(res, [result], function(err) {
if (err) {
onError(res,next,createError(400));
}
else {
if(usingExpress) {
res.status(200).end();
} else {
res.send(200);
}
next();
}
});
}
});
});
} else {
contextFilter(model, req, function(filteredContext) {
filteredContext.findOne(byId, function(err, doc) {
if (err || !doc) {
onError(res,next,createError(404));
} else {
doc.remove(function(err, result) {
postDelete(res, [doc], function(err) {
if (err) {
onError(res,next,createError(400));
}
else {
if(usingExpress) {
res.status(200).end();
} else {
res.send(200);
}
next();
}
});
});
}
});
});
}
}, postProcess);
};
var path = require('path');
var webpack = require('webpack');
var HtmlWebpackPlugin = require('html-webpack-plugin');
var pkg = require('./package.json');
var ExtractTextPlugin = require('extract-text-webpack-plugin');
var autoprefixer = require('autoprefixer');
var util = require('util');
var webpack = require('webpack');
var DEBUG = process.env.NODE_ENV === 'development';
var TEST = process.env.NODE_ENV === 'test';
var cssBundle = path.join('css', util.format('app.%s.css', pkg.version));
var jsBundle = path.join('js', util.format('app.%s.js', pkg.version));
var jsxLoader;
var sassLoader;
var cssLoader;
var fileLoader = 'file-loader?name=[path][name].[ext]';
var htmlLoader = [
'file-loader?name=[path][name].[ext]',
'template-html-loader?' + [
'raw=true',
'engine=lodash',
'version=' + pkg.version,
'title=' + pkg.name,
'debug=' + DEBUG
].join('&')
].join('!');
function getAllSubmissionsUrl(handle){
return util.format(SUBMISSIONS_URL, handle);
}
SpiderJob.prototype.snapshot = function (page, url){
var self = this;
console.log(util.format('Do snapshot for url [%s].', url));
setTimeout(function () {
console.log(util.format('Snapshot time out is tirgger, [%s].', url));
page.evaluate(function () {
return document.documentElement.outerHTML;
},function(err, doc){
if(doc){
var filePath = buildSnapshotFileAndPath(url, self.snapshotDir);
fs.writeFile(filePath, doc, function(e){//会先清空原先的内容
if(e) throw e;
console.log(util.format('Snapshot file [%s] is created!', filePath));
});
}else{
console.error(util.format('Html snapshot on [%s] is empty', url));
}
});
self.parserPage(page);
}, 3000);
/**
* Build the file path of snapshot for url.
*/
function buildSnapshotFileAndPath(_url, snapshotDir){
var fileName, filePath,
self = this;
fileName = escapeFileName(_url);
filePath = snapshotDir;
if(filePath.indexOf('\\', filePath.length - 1) === -1){
filePath += '\\';
}
if(!fs.existsSync(filePath)){
fs.mkdirSync(filePath);
}
return filePath + fileName;
}
/**
* Escape url to suitable file name.
* @param input input url
* @return {*} output file name
*/
function escapeFileName(input){
var toEscape = '\\ / : * ? " < >',
//toEscapes = toEscape.split(' '),
toEscapes = ['\\', '\/', '\:', '\*', '\?', '"', '<', '>'],
output = input;
// toEscapes.forEach(function(item){
// // output = output.replace(new RegExp(item, 'gm'), '_');
// output = output.replace(item, '_');
// console.log(output);
// });
console.log(output);
output = output.replace(/\\/ig, '_');
output = output.replace(/\//ig, '_');
output = output.replace(/:/ig, '_');
output = output.replace(/\*/ig, '_');
output = output.replace(/\?/ig, '_');
output = output.replace(/"/ig, '_');
output = output.replace(/</ig, '_');
output = output.replace(/>/ig, '_');
console.log(output);
return output;
}
};
function getSubmissionUrl(contestID, submissionID){
return util.format(SUBMISSION_URL, contestID, submissionID);
}
return function parser(inError, body) {
/*jshint maxdepth:4 */
var code = response.statusCode
, errors = []
, errorClass
, json
, returned
;
if (code !== 200) logger.debug("Got %s as a response code from the collector.", code);
if (inError) errors.push(inError);
if (body) {
try {
json = JSON.parse(body);
// Can be super verbose, but useful for debugging.
logger.trace({response : json}, "Deserialized from collector:");
// If we get messages back from the collector, be polite and pass them along.
returned = json[RESPONSE_VALUE_NAME];
if (returned && returned.messages) {
returned.messages.forEach(function cb_forEach(element) { logger.info(element.message); });
}
/* Wait to deal with errors in the response until any messages have
* been passed along. Otherwise, ensure that there was a return
* value, raising an error if not.
*
* Some errors are only interesting if the status code indicates
* that the request went bad already, so filter out adding more
* errors when statusCode is not OK (200).
*/
var exception = json[EXCEPTION_VALUE_NAME];
if (exception) {
if (exception.message) {
errors.push(new Error(exception.message));
}
else if (code === 200 ) {
errors.push(new Error('OneAPM internal error'));
}
if (exception.error_type) errorClass = exception.error_type;
}
else if (code === 200 && returned === undefined) {
errors.push(new Error(format('No data found in response to %s.', name)));
}
}
catch (error) {
errors.push(error);
}
}
else {
errors.push(new Error(format('No body found in response to %s.', name)));
}
if (code !== 200) {
errors.push(new Error(format('Got HTTP %s in response to %s.', code, name)));
}
var error;
if (errors.length > 0) {
error = errors.shift();
error.statusCode = code;
// Preserve a consistent hidden class (cheaper than subclassing Error).
error.class = errorClass ? errorClass : undefined;
error.laterErrors = (errors.length > 0) ? errors : undefined;
}
// Raw json is useful for testing and logging.
process.nextTick(function cb_nextTick() { callback(error, returned, json); });
};
function createOutputFolder(contestName){
fs.mkdirSync(util.format(OUTPUT_FOLDER + '/%s', contestName));
}
row[key].split(',').map(function(v) { return v.trim(); }).forEach((nodeName) => {
if (nodeName.length !== 1) throw new Error(util.format('*** invalid relation node member "%s"'), nodeName);
let node = this.findNodeByName(nodeName);
if (!node) throw new Error(util.format('*** unknown relation node member "%s"'), nodeName);
relation.addMember('node', node.id, isNode[1]);
});
function doAppCall(endpoints, func,params,liveflag,cb) {
if('object'!== typeof endpoints || endpoints.status !== 200) return cb(nodejsutil.format(ERRORS.AppNotFound,params.guid));
var retdata = JSON.parse(endpoints['body']);
var callurl = (liveflag)
? retdata['hosts']['live-url']
: retdata['hosts']['development-url'];
callurl+="/cloud/"+func;
var payload = "";
try{
payload = JSON.stringify(params);
}catch(e){
return cb(" Could not send params " + e.message);
}
var urlprops = url.parse(callurl);
//sometimes there is a port appended to the host url
var portmatch = urlprops.host.match(/[:\d]*$/);
var port = (portmatch === null)? 443 : portmatch[0].toString().replace(/:/,"");
urlprops.host = urlprops.host.replace(/[:\d]*$/,"");
var headers = {};
headers["content-type"] = "application/json";
headers["content-length"] = new Buffer(payload).length ||0;
headers["accept"] = "application/json";
addAppApiKeyHeader(headers);
var httpsopts = {
host:urlprops.host,
path:urlprops.path,
port:port,
method:"POST",
headers:headers
};
//finally make call to other app
var req = https.request(httpsopts,function (res){
req.socket.setTimeout(60000);
req.connection.setTimeout(60000);
var retData="";
res.on('data', function (data){
retData+=data;
});
res.on('end',function () {
return cb(null,retData);
});
});
req.on('error', function (err){
return cb(err);
});
req.write(payload);
req.end();
};