Example #1
0
console.log=function(d){
	log_file.write(util.format(d)+'\n');
	log_stdout.write(util.format(d)+'\n');
};
Example #2
0
 fs.writeFile(filePath, doc, function(e){//会先清空原先的内容
     if(e) throw e;
     console.log(util.format('Snapshot file [%s] is created!', filePath));
 });
Example #3
0
	analyzer.on('declaration', function(rule, property, value) {
		if (value.indexOf('!important') > -1) {
			analyzer.incrMetric('importants');
			analyzer.addOffender('importants', format('%s {%s: %s}', rule.selectors.join(', '), property, value));
		}
	});
Example #4
0
function create_filename(file) {
    var ext = path.extname(file.name);
    var basename = path.basename(file.name, ext);
    var filename = util.format('%s_%d%s', basename, Date.now(), ext);
    return filename;
}
Example #5
0
 return util.format(xray.templates.map, opts.vector_layers.map(function(layer){
     var rgb = xray.color(layer.id).join(',');
     return util.format(xray.templates.layer, layer.id, rgb, rgb, rgb, rgb, rgb, layer.id, layer.id)
 }).join('\n'));
Example #6
0
downloader.prototype.downloadItAct = function(urlinfo){
    var spiderCore = this.spiderCore;
    var self = this;

    var timeOuter = false;
    var pageLink = urlinfo['url'];
    if(urlinfo['redirect'])pageLink = urlinfo['redirect'];

    var useProxy = false;
    if(urlinfo['urllib']&&spiderCore.settings['use_proxy']===true){
        if(spiderCore.spider.getDrillerRule(urlinfo['urllib'],'use_proxy')===true)useProxy=true;
    }

    if(useProxy){
        var proxyRouter = spiderCore.settings['proxy_router'].split(':');
        var __host = proxyRouter[0];
        var __port = proxyRouter[1];
        var __path =  pageLink;
    }else{
        var urlobj = urlUtil.parse(pageLink);
        var __host = urlobj['hostname'];
        var __port = urlobj['port'];
        var __path = urlobj['path'];
//        var __path = pageLink;
    }


    var startTime = new Date();
    var options = {
        'host': __host,
        'port': __port,
        'path': __path,
        'method': 'GET',
        'headers': {
            "User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like NeoCrawler) Chrome/31.0.1650.57 Safari/537.36",
            "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
            "Accept-Encoding":"gzip",
            "Accept-Language":"zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4",
            "Referer":urlinfo['referer']||'',
            "void-proxy":urlinfo['void_proxy']?urlinfo['void_proxy']:"",
            "Cookie":this.transCookieKvPair(urlinfo['cookie'])
        }
    };
    logger.debug(util.format('Request start, %s',pageLink));
    var req = http.request(options, function(res) {
        logger.debug(util.format('Response, %s',pageLink));

        var result = {
            "remote_proxy":res.headers['remoteproxy'],
            "drill_count":0,
            "cookie":res.headers['Cookie'],
            "url":urlinfo['url'],
            //"url":res.req.path,
            //"statusCode":res.statusCode,
            "origin":urlinfo
        };
        if(result['url'].startsWith('/'))result['url'] = urlUtil.resolve(pageLink,result['url']);
        result['statusCode'] = res.statusCode;
        if(parseInt(res.statusCode)==301||parseInt(res.statusCode)==302){
            if(res.headers['location']){
                result['origin']['redirect'] = urlUtil.resolve(pageLink,res.headers['location']);
                logger.debug(pageLink+' 301 Moved Permanently to '+res.headers['location']);
            }
        }

        var compressed = /gzip|deflate/.test(res.headers['content-encoding']);

        var bufferHelper = new BufferHelper();
//        res.setEncoding('utf8');

        res.on('data', function (chunk) {
            bufferHelper.concat(chunk);
        });

        res.on('end', function (chunk) {
            self.timeout_count--;
            if(timeOuter){
                clearTimeout(timeOuter);
                timeOuter = false;
            }
            result["cost"] = (new Date()) - startTime;
            logger.debug('download '+pageLink+', cost:'+result["cost"]+'ms');


            var page_encoding = urlinfo['encoding'];

            if(page_encoding==='auto'){
                page_encoding = self.get_page_encoding(res.headers);
            }

            page_encoding = page_encoding.toLowerCase().replace('\-','')
            if(!compressed || typeof unzip == 'undefined'){
                if(urlinfo['format']=='binary'){
                    result["content"] = bufferHelper.toBuffer();
                }else{
                    result["content"] = iconv.decode(bufferHelper.toBuffer(),page_encoding);//page_encoding
                }
                spiderCore.emit('crawled',result);
            }else{
                unzip(bufferHelper.toBuffer(), function(err, buff) {
                    if (!err && buff) {
                        if(urlinfo['format']=='binary'){
                            result["content"] = buff;
                        }else{
                            result["content"] = iconv.decode(buff,page_encoding);
                        }
                        spiderCore.emit('crawled',result);
                    }else{
                        spiderCore.emit('crawling_failure',urlinfo,'unzip failure');
                    }
                });
            }
        });
    });

    timeOuter = setTimeout(function(){
        if(req){
            logger.error('Cost '+((new Date())-startTime)+'ms download timeout, '+pageLink);
            req.abort();
            req=null;
            spiderCore.emit('crawling_failure',urlinfo,'download timeout');
            if(self.timeout_count++>spiderCore.settings['spider_concurrency']){logger.fatal('too much timeout, exit.');process.exit(1);}
        }
    },spiderCore.settings['download_timeout']*1000);

    req.on('error', function(e) {
        logger.error('problem with request: ' + e.message+', url:'+pageLink);
        if(timeOuter){
            clearTimeout(timeOuter);
            timeOuter = false;
        }
        if(req){
            req.abort();
            req = null;
            spiderCore.emit('crawling_failure',urlinfo,e.message);
        }
    });
    req.end();
}
Example #7
0
					sails.after('lifted', function () {
						console.log();
						sails.log.warn(util.format('Ignoring invalid blueprint prefix configured for controller `%s`.', globalId));
						sails.log.warn('`prefix` should be a string, e.g. "/api/v1".');
						STRINGFILE.logMoreInfoLink(STRINGFILE.get('links.docs.config.blueprints'), sails.log.warn);
					});
Example #8
0
exports['FWAPI-237: Ignore rules that don\'t change'] = function (t) {
    var vm1 = helpers.generateVM({ uuid: helpers.uuidNum(1) });
    var vm2 = helpers.generateVM({ uuid: helpers.uuidNum(2) });
    var vm3 = helpers.generateVM({ uuid: helpers.uuidNum(3) });
    var vm4 = helpers.generateVM({
        owner_uuid: mod_uuid.v4(),
        uuid: helpers.uuidNum(4)
    });
    var allVMs = [ vm1, vm2, vm3, vm4 ];
    var updateUUID = mod_uuid.v4();

    var payload = {
        rules: [
            {
                rule: 'FROM any TO all vms ALLOW icmp TYPE 8 CODE 0',
                enabled: true,
                global: true
            },
            {
                uuid: updateUUID,
                owner_uuid: vm1.owner_uuid,
                rule: util.format('FROM any TO vm %s ALLOW udp PORT 514',
                    vm1.uuid),
                enabled: true
            }
        ],
        vms: allVMs
    };

    var expRules = [clone(payload.rules[0]), clone(payload.rules[1])];

    var updatePayload = {
        rules: [
            {
                uuid: updateUUID,
                owner_uuid: vm1.owner_uuid,
                rule: util.format('FROM any TO vm %s ALLOW udp PORT 515',
                    vm1.uuid),
                enabled: true
            }
        ],
        vms: allVMs
    };

    async.series([
    function (cb) {
        fw.add(payload, function (err, res) {
            t.ifError(err);
            if (err) {
                return cb();
            }

            for (var i in res.rules) {
                t.ok(res.rules[i].uuid, 'rule has a uuid');
                expRules[i].uuid = res.rules[i].uuid;

                t.ok(res.rules[i].version, 'rule has a version');
                expRules[i].version = res.rules[i].version;
            }

            t.deepEqual(helpers.sortRes(res), helpers.sortRes({
                rules: clone(expRules),
                vms: [ vm1.uuid, vm2.uuid, vm3.uuid, vm4.uuid ].sort()
            }), 'rules returned');

            cb();
        });

    }, function (cb) {
        helpers.fwGetEquals(t, expRules[0], cb);

    }, function (cb) {
        helpers.fwListEquals(t, expRules, cb);

    }, function (cb) {
        var globalPayload = {
            rules: [ expRules[0] ],
            vms: allVMs
        };

        fw.update(globalPayload, function (err, res) {
            t.ifError(err);
            if (err) {
                return cb();
            }

            // Nothing's changed in the global rule, so nothing should
            // be updated.
            t.deepEqual(helpers.sortRes(res), { vms: [], rules: [] },
                'rules returned');

            cb();
        });

    }, function (cb) {
        var globalPayload = {
            rules: [ expRules[0] ],
            vms: allVMs
        };
        globalPayload.rules[0].rule =
            'FROM any TO all vms ALLOW icmp TYPE all';

        fw.update(globalPayload, function (err, res) {
            t.ifError(err);
            if (err) {
                return cb();
            }

            // This global rule affects all VMs, so they all should be updated.
            t.deepEqual(helpers.sortRes(res), {
                vms: [ vm1.uuid, vm2.uuid, vm3.uuid, vm4.uuid ],
                rules: globalPayload.rules
            }, 'rules returned');

            cb();
        });
    }, function (cb) {
        fw.update(updatePayload, function (err, res) {
            t.ifError(err);
            if (err) {
                return cb();
            }

            // This rule should only affect the VM that it mentions.
            t.deepEqual(helpers.sortRes(res), {
                vms: [ vm1.uuid ],
                rules: updatePayload.rules
            }, 'rules returned');

            cb();
        });

    }, function (cb) {
        helpers.fwGetEquals(t, updatePayload.rules[0], cb);

    }, function (cb) {
        fw.update(updatePayload, function (err, res) {
            t.ifError(err);
            if (err) {
                return cb();
            }

            // Repeating the same update again shouldn't do anything.
            t.deepEqual(helpers.sortRes(res), { vms: [], rules: [] },
                'rules returned');

            cb();
        });

    }, function (cb) {
        var vm2updated = clone(vm2);
        vm2updated.nics[0].ip = '1.2.3.4';
        var vmPayload = {
            localVMs: [ vm2updated ],
            vms: allVMs
        };

        fw.update(vmPayload, function (err, res) {
            t.ifError(err);
            if (err) {
                return cb();
            }

            // None of the rules are affected by the VM's IP changing, so only
            // the modified VM is updated.
            t.deepEqual(helpers.sortRes(res), {
                vms: [ vm2updated.uuid ],
                rules: []
            }, 'rules returned');

            cb();
        });
    }, function (cb) {
        var addPayload = {
            rules: [ {
                uuid: mod_uuid.v4(),
                rule: 'FROM all vms TO all vms ALLOW tcp PORT all',
                enabled: true,
                owner_uuid: vm1.owner_uuid
            } ],
            vms: allVMs,
            allowAdds: true
        };

        // Add a rule that affects all VMs owned by the owner of VMs 1, 2, 3
        fw.update(addPayload, function (err, res) {
            t.ifError(err);
            if (err) {
                return cb();
            }

            t.deepEqual(helpers.sortRes(res), {
                vms: [ vm1.uuid, vm2.uuid, vm3.uuid ],
                rules: addPayload.rules
            }, 'rules returned');

            cb();
        });
    }, function (cb) {
        var vm2updated = clone(vm2);
        vm2updated.nics[0].ip = '1.2.3.5';
        var vmPayload = {
            localVMs: [ vm2updated ],
            vms: allVMs
        };

        fw.update(vmPayload, function (err, res) {
            t.ifError(err);
            if (err) {
                return cb();
            }

            // Now that there's a rule that affects all VMs of one owner,
            // updating the IP address affects those VMs, so they need their
            // active firewall rules set updated.
            t.deepEqual(helpers.sortRes(res), {
                vms: [ vm1.uuid, vm2.uuid, vm3.uuid ],
                rules: []
            }, 'rules returned');

            cb();
        });
    }, function (cb) {
        var vm4updated = clone(vm4);
        vm4updated.nics[0].ip = '1.2.3.20';
        var vmPayload = {
            localVMs: [ vm4updated ],
            vms: allVMs
        };

        fw.update(vmPayload, function (err, res) {
            t.ifError(err);
            if (err) {
                return cb();
            }

            // Updating the IP address of the VM that has a different owner
            // shouldn't do anything to other VMs.
            t.deepEqual(helpers.sortRes(res), {
                vms: [ vm4updated.uuid ],
                rules: []
            }, 'rules returned');

            cb();
        });
    }

    ], function () {
        t.done();
    });


};
Example #9
0
exports['update non-existent rule'] = function (t) {
    var vm = helpers.generateVM();

    var payload = {
        rules: [
            {
                owner_uuid: vm.owner_uuid,
                rule: util.format('FROM vm %s TO any BLOCK tcp PORT 8080',
                    vm.uuid),
                uuid: mod_uuid.v4(),
                enabled: true
            }
        ],
        vms: [vm]
    };

    var expRules = [clone(payload.rules[0])];
    var vmsEnabled = {};
    var allRules;

    async.series([
    function (cb) {
        fw.update(payload, function (err) {
            t.ok(err, 'Error returned');
            return cb();
        });

    }, function (cb) {
        payload.allowAdds = true;
        fw.update(payload, function (err, res) {
            t.ifError(err);
            if (err) {
                return cb();
            }

            t.ok(res.rules[0].uuid, 'rule has a uuid');
            expRules[0].uuid = res.rules[0].uuid;

            t.ok(res.rules[0].version, 'rule has a version');
            expRules[0].version = res.rules[0].version;

            t.deepEqual(res, {
                rules: expRules,
                vms: [ vm.uuid ]
            }, 'rules returned');

            allRules = helpers.defaultZoneRules(vm.uuid);
            allRules[vm.uuid].out.tcp = [
                helpers.blockPortOutTCP('any', 8080)
            ];

            t.deepEqual(helpers.zoneIPFconfigs(4), allRules,
                'IPv4 firewall rules correct');
            t.deepEqual(helpers.zoneIPFconfigs(6), allRules,
                'IPv6 firewall rules correct');

            vmsEnabled[vm.uuid] = true;
            t.deepEqual(helpers.getIPFenabled(), vmsEnabled,
                'ipf enabled in VMs');

            cb();
        });

    }, function (cb) {
        helpers.fwGetEquals(t, expRules[0], cb);
    }

    ], function () {
        t.done();
    });
};
Example #10
0
var crawlerSubscribe = function(entry) {
  // 所有栏目列表 http://r.inews.qq.com/getCatList?qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&appver=16_android_3.2.1&version=0
  // 获取某一栏目文章ids http://r.inews.qq.com/getSubNewsIndex?qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&format=json&apptype=android&chlid=1250&appver=16_android_3.2.1
  // 获取指定ids文章 http://r.inews.qq.com/getSubNewsListItems?uid=22c4d53a-7d52-4f50-9185-90a77c7b80b0&qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&ids=20131107A000D700%2C20131106A000DE00%2C20131106A000DD00%2C20131105A000DS00%2C20131105A000DR00%2C20131104A000CW00%2C20131101A000DZ00%2C20131031A000BZ00%2C20131030A000BV00%2C20131029A000C600%2C20131028A0017800%2C20131025A0019S00%2C20131024A000CG00%2C20131023A000CM00%2C20131022A000I400%2C20131022A000I300%2C20131022A0007700%2C20131018A000EB00%2C20131017A0015800%2C20131016A000AX00&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&appver=16_android_3.2.1
  // 获取某篇文章内容 http://r.inews.qq.com/getSubNewsContent?id=20131129A000H600&qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&appver=16_android_3.2.1
  var subscribeNewsIdsLink = 'http://r.inews.qq.com/getSubNewsIndex?qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&format=json&apptype=android&chlid=%s&appver=16_android_3.2.1';
  var subscribeNewsListLink = 'http://r.inews.qq.com/getSubNewsListItems?uid=22c4d53a-7d52-4f50-9185-90a77c7b80b0&qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&ids=%s&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&appver=16_android_3.2.1'

  var url = '';
  if(entry.ids) {
    var ids = '';
    var i = 0;
    for(i=0; (i<20) && (i<entry.ids.length); i++) {
      if(i > 0) {
        ids = ids + ','
      }
      ids = ids + entry.ids[i].id;
    }
    url = util.format(subscribeNewsListLink, ids);
    entry.ids = entry.ids.slice(i);
  }else {
    url = util.format(subscribeNewsIdsLink, entry.id);
  }
  var req = {uri: url, method: "GET", headers: headers};
  if(proxyEnable) {
    req.proxy = proxyUrl;
  }
  request(req, function (err, res, body) {
    var json = data2Json(err, res, body);
    if(!json || (json.ret != 0)) {
      console.log("hzfdbg file[" + __filename + "]" + " crawlerSubscribe():JSON.parse() error");
      return;
    }

    if(entry.ids) {
      setTimeout(function() {
        crawlerSubscribe(entry);
      }, 100);
    }else if(json.ids) {
      entry.ids = json.ids;
      setTimeout(function() {
        crawlerSubscribe(entry);
      }, 100);
      return;
    }
    var newsList = json.newslist;
    if((!newsList) || (!newsList.length) || (newsList.length <= 0)) {
      console.log("hzfdbg file[" + __filename + "]" + " crawlerSubscribe():newsList empty in url " + url);
      return;
    }
    newsList.forEach(function(newsEntry) {
      if(!newsEntry.title || !newsEntry.id) {
        return;
      }
      var tags = entry.tags;
      for(var i = 0; i < tags.length; i++) {
        if (newsEntry.title.indexOf(tags[i]) !== -1 || entry.all) {
          newsEntry.tagName = tags[i];
          newsEntry.subscribeName = entry.name;

          News.findOne(genQqFindCmd(site,newsEntry), function(err, result) {
            if(err || result) {
              return;
            }
            console.log("hzfdbg file[" + __filename + "]" + " crawlerSubscribe():["+newsEntry.tagName+"]"+newsEntry.title+",docid="+newsEntry.id);
            startGetDetail.emit('startGetSubscribeNewsDetail', newsEntry);
          }); // News.findOne
          break;
        }
      }//for
    });//forEach
  });//request
}
Example #11
0
 fw.update(payload, function (err, res) {
     t.ok(err, 'error returned');
     t.equal(err.message, util.format('Could not find VM "%s" in VM list',
         vm.uuid), 'error message');
     t.done();
 });
Example #12
0
var getNewsDetail = function(entry) {
  var detailLink = 'http://inews.qq.com/getQQNewsNormalHtmlContent?id=%s';
  var docid = util.format("%s",entry.id);
  var url = util.format(detailLink, docid);
  var req = {uri: url, method: "GET", headers: headers};
  if(proxyEnable) {
    req.proxy = proxyUrl;
  }
  request(req, function (err, res, body) {
    var json = data2Json(err, res, body);
    if((!json) || (json.ret != 0)) {
      console.log("hzfdbg file[" + __filename + "]" + " getNewsDetail():ret="+json.ret);
      return;
    }
    var jObj = json;
    var obj = entry;

    News.findOne(genQqFindCmd(site, entry), function(err, result) {
      if(err || result) {
        return;
      }
      obj.docid = encodeDocID(site, docid);
      obj.site = site;
      obj.body = '';
      obj.img = [];
      jObj.content.forEach(function(item) {
        if(item.type == 1) { //text
          obj.body += item.value;
        }
        else if(item.type == 2) { //pic
          obj.body += genLazyLoadHtml(entry.tagName, item.value);
          obj.img[obj.img.length] = item.value;
        }
        else if(item.type == 3) { //video
          obj.body += genLazyLoadHtml(entry.tagName, item.value.img);
          obj.img[obj.img.length] = item.value.img;
        }
      });
      obj.video = [];
      obj.link = "";
      if(entry.url) {
        obj.link = entry.url; // http://view.inews.qq.com/a/NEW2013050300143202
      }else if(entry.surl) {
        obj.link = entry.surl; // http://view.inews.qq.com/a/NEW2013050300143202
      }else if(jObj.url) {
        obj.link = jObj.url; // http://view.inews.qq.com/a/NEW2013050300143202
      }else if(jObj.surl) {
        obj.link = jObj.surl; // http://view.inews.qq.com/a/NEW2013050300143202
      }else {
        obj.link = util.format("http://view.inews.qq.com/a/%s", docid);
      }
      obj.title = entry.title;
      obj.ptime = entry.time;
      obj.time = new Date(entry.time);
      obj.marked = obj.body;
      obj.created = new Date();
      obj.views = 1;
      obj.tags = entry.tagName;
      obj.digest = genDigest(obj.body);
      if(!obj.digest) {
        obj.digest = obj.body;
      }
      obj.cover = entry.thumbnails[0];
      if (obj.img[0]) {
        obj.cover = obj.img[0];
      }

      News.insert(obj, function (err, result) {
        if(err) {
          console.log("hzfdbg file[" + __filename + "]" + " getNewsDetail(), News.insert():error " + err);
        }
      });
    });//News.findOne
  });//request
};
Example #13
0
var getPhotoDetail = function(entry) {
  // http://inews.qq.com/getQQNewsSimpleHtmlContent?id=PIC2013061200601000&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&sceneid=00000&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&chlid=news_photo&appver=16_android_2.7.0
  var photoDetailLink = "http://inews.qq.com/getQQNewsSimpleHtmlContent?id=%s&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&sceneid=00000&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&chlid=news_photo&appver=16_android_2.7.0";
  var docid = util.format("%s",entry.id);
  var url = util.format(photoDetailLink, docid);
  var req = {uri: url, method: "GET", headers: headers};
  if(proxyEnable) {
    req.proxy = proxyUrl;
  }
  request(req, function (err, res, body) {
    var json = data2Json(err, res, body);
    if((!json) || (json.ret != 0)) {
      console.log("hzfdbg file[" + __filename + "]" + " getPhotoDetail():ret="+json.ret);
      return;
    }
    var jObj = json;
    var obj = entry;

    News.findOne(genQqFindCmd(site, entry), function(err, result) {
      if(err || result) {
        return;
      }
      obj.docid = encodeDocID(site, docid);
      obj.site = site;
      obj.body = jObj.intro + jObj.content.text;
      obj.img = [];
      var attribute = jObj.attribute;
      var img_keys = _.keys(attribute);
      img_keys.forEach(function(key){
        obj.img[obj.img.length] = attribute[key].url;
        var imgHtml = attribute[key].desc + genLazyLoadHtml(attribute[key].desc, attribute[key].url);
        obj.body = obj.body.replace("<!--" + key + "-->", imgHtml); // <!--IMG_4-->
      });
      obj.video = [];
      obj.link = "";
      if(entry.url) {
        obj.link = entry.url; // http://view.inews.qq.com/a/TPC2013061100203800
      }else if(entry.surl) {
        obj.link = entry.surl; // http://view.inews.qq.com/a/TPC2013061100203800
      }else if(jObj.url) {
        obj.link = jObj.url; // http://view.inews.qq.com/a/TPC2013061100203800
      }else if(jObj.surl) {
        obj.link = jObj.surl; // http://view.inews.qq.com/a/TPC2013061100203800
      }else {
        obj.link = util.format("http://view.inews.qq.com/a/%s", docid);
      }
      obj.title = entry.title;
      obj.ptime = entry.time;
      obj.time = new Date(entry.time);
      obj.marked = obj.body;
      obj.created = new Date();
      obj.views = 1;
      obj.tags = entry.tagName;
      obj.digest = genDigest(obj.body);
      obj.cover = entry.thumbnails[0];
      if (obj.img[0]) {
        obj.cover = obj.img[0];
      }

      News.insert(obj, function (err, result) {
        if(err) {
          console.log("hzfdbg file[" + __filename + "]" + " getPhotoDetail(), News.insert():error " + err);
        }
      });
    });//News.findOne
  });//request
};
Example #14
0
var getSubscribeNewsDetail = function(entry) {
  // http://r.inews.qq.com/getSubNewsContent?id=20131129A000H600&qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&appver=16_android_3.2.1
  var subscribeNewsDetailLink = 'http://r.inews.qq.com/getSubNewsContent?id=%s&qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&appver=16_android_3.2.1';
  var docid = util.format("%s",entry.id);
  var url = util.format(subscribeNewsDetailLink, docid);
  var req = {uri: url, method: "GET", headers: headers};
  if(proxyEnable) {
    req.proxy = proxyUrl;
  }
  request(req, function (err, res, body) {
    var json = data2Json(err, res, body);
    if((!json) || (json.ret != 0)) {
      console.log("hzfdbg file[" + __filename + "]" + " getSubscribeNewsDetail():ret="+json.ret);
      return;
    }
    var jObj = json;
    var obj = entry;

    News.findOne(genQqFindCmd(site, entry), function(err, result) {
      if(err || result) {
        return;
      }
      obj.docid = encodeDocID(site, docid);
      obj.site = site;
      obj.body = jObj.content.text;
      obj.img = [];
      for(key in jObj.attribute) {
        var html = genLazyLoadHtml(entry.title, jObj.attribute[key].url);
        if(jObj.attribute[key].desc) {
          html = html + jObj.attribute[key].desc + '<br/>';
        }
        obj.img[obj.img.length] = jObj.attribute[key].url;
        obj.body = obj.body.replace(util.format('<!--%s-->', key), html);
      }
      obj.video = [];
      obj.link = '';
      if(entry.url) {
        obj.link = entry.url;
      }else {
        obj.link = util.format("http://view.inews.qq.com/a/%s", entry.id);
      }
      obj.title = entry.title;
      obj.ptime = entry.time;
      obj.time = new Date(entry.time);
      obj.marked = obj.body;
      obj.created = new Date();
      obj.views = 1;
      obj.tags = entry.tagName;
      obj.digest = genDigest(obj.body);
      obj.cover = '';
      if(entry.thumbnails_qqnews && entry.thumbnails_qqnews[0]) {
        obj.cover = entry.thumbnails_qqnews[0];
      }else if(entry.chlsicon) {
        obj.cover = entry.chlsicon;
      }else if(entry.chlicon) {
        obj.cover = entry.chlicon;
      }else if(jObj.card && jObj.card.icon) {
        obj.cover = jObj.card.icon;
      }else if(obj.img) {
        obj.cover = obj.img[0];
      }else {
        obj.cover = '';
      }

      News.insert(obj, function (err, result) {
        if(err) {
          console.log("hzfdbg file[" + __filename + "]" + " getSubscribeNewsDetail(), News.insert():error " + err);
        }
      });
    });//News.findOne
  });//request
};
Example #15
0
 function logBadResponseError(err$2) {
     logger.error('CommandHandler', format(badResponseFormat, command.message, inspect(res)));
 }
Example #16
0
/**
 * This function uses the objectdiff library to diff two objects or
 * literals and generates a nice-looking diff (inspired by file diffs)
 * from the result.
 *
 * @param a {*} diff operand 1, can be object or literal.
 * @param b {*} diff operand 2, can be object or literal.
 * @param colors {boolean} whether to colorize the output.
 * @returns {Array} empty if equal, otherwise a list of changes.
 * @private
 */
function _generateObjectDiff(a, b, colors) {
    var result = [];
    var numberPat = /^\d+$/;

    // Format strings
    var changeStr =  ['*   %s = %s -> %s',
                      '\x1B[36m*   %s\x1B[0m = \x1B[31m%s\x1B[0m -> \x1B[32m%s\x1B[0m'];
    var changeStr2 = ['*   %s -> %s',
                      '\x1B[36m*\x1B[0m   \x1B[31m%s\x1B[0m -> \x1B[32m%s\x1B[0m'];
    var removeStr =  ['-   %s = %s',
                      '\x1B[31m-   %s\x1B[0m = \x1B[31m%s\x1B[0m'];
    var addStr =     ['+   %s = %s',
                      '\x1B[32m+   %s\x1B[0m = \x1B[32m%s\x1B[0m'];
    var idx = colors ? 1 : 0;

    // Traverse the diff object to find all changes.
    function traverse(node, path) {
        var keys, i;
        if (node.changed === 'object change') {
            keys = Object.keys(node.value);
            for (i = 0; i < keys.length; i++) {
                var newPath;
                // If the key is a number, it's probably an array index
                if (numberPat.test(keys[i])) {
                    newPath = '[' + keys[i] + ']';
                    if (path) {
                        newPath = path + newPath;
                    }
                } else {
                    newPath = path ? path + '.' + keys[i] : keys[i];
                }
                traverse(node.value[keys[i]], newPath);
            }

        } else if (node.changed === 'primitive change') {
            result.push(_lineBreak(util.format(changeStr[idx]
                                             , path
                                             , util.inspect(node.removed)
                                             , util.inspect(node.added))));

        } else if (node.changed === 'removed') {
            result.push(_lineBreak(util.format(removeStr[idx]
                                             , path
                                             , util.inspect(node.value))));

        } else if (node.changed === 'added') {
            result.push(_lineBreak(util.format(addStr[idx]
                                             , path
                                             , util.inspect(node.value))));
        }
    }

    // objectdiff only supports comparing two objects.
    if (a instanceof Object && b instanceof Object) {
        var diff = objectDiff.diff(a, b);

        traverse(diff, '');

    } else {
        if (a !== b) {
            result.push(_lineBreak(util.format(changeStr2[idx]
                                             , util.inspect(a)
                                             , util.inspect(b))));
        }
    }

    return result;
}
Example #17
0
 var conditions = _.map(this.parseConditionObject(smth.conditions), function generateSql(condition) {
   return util.format("%s#>>'{%s}' = '%s'",
     _.first(condition.path),
     _.tail(condition.path).join(','),
     condition.value);
 });
Example #18
0
    return promise.then(pass_results => {
        var proposalResponses = pass_results[0];

        var proposal = pass_results[1];
        var header   = pass_results[2];
        var goodCount = 0;
        for(var i in proposalResponses) {
            let one_good = false;
            let proposal_response = proposalResponses[i];
            if( proposal_response.response && proposal_response.response.status === 200) {
                console.log('transaction proposal has response status of good');
                one_good = chain.verifyProposalResponse(proposal_response);
                if(one_good) {
                    console.log(' transaction proposal signature and endorser are valid');
                    goodCount++;
                }
            } else {
                //console.error('transaction proposal was bad');
            }
            //all_good = all_good & one_good;
        }

        var all_good = goodCount >= +config.endorsementPolicy.split("/")[0];        
        if (all_good) {
            // check all the read/write sets to see if the same, verify that each peer
            // got the same results on the proposal
            all_good = chain.compareProposalResponseResults(proposalResponses);
            console.log('compareProposalResponseResults exection did not throw an error');
            if(all_good){
                console.log(' All proposals have a matching read/writes sets');
            }
            else {
                console.error(' All proposals do not have matching read/write sets');
            }
        }
        if (all_good) {
            // check to see if all the results match
            console.log(util.format('Successfully sent Proposal and received ProposalResponse: Status - %s, message - "%s", metadata - "%s", endorsement signature: %s', proposalResponses[0].response.status));//, proposalResponses[0].response.message, proposalResponses[0].response.payload, proposalResponses[0].endorsement.signature));
            var request = {
                proposalResponses: proposalResponses,
                proposal: proposal,
                header: header
            };

            // set the transaction listener and set a timeout of 30sec
            // if the transaction did not get committed within the timeout period,
            // fail the test
            var deployId = tx_id.getTransactionID();

            var eventPromises = [];
            eventhubs.forEach((eh) => {
                let txPromise = new Promise((resolve, reject) => {
                    let handle = setTimeout(reject, 120000);

                    eh.registerTxEvent(deployId.toString(),
                        (tx, code) => {
                            clearTimeout(handle);
                            eh.unregisterTxEvent(deployId);

                            if (code !== 'VALID') {
                                console.error('The balance transfer transaction was invalid, code = ' + code);
                                reject();
                            } else {
                                console.log('The balance transfer transaction has been committed on peer '+ eh.ep._endpoint.addr);
                                resolve();
                            }
                        },
                        (err) => {
                            clearTimeout(handle);
                            console.log('Successfully received notification of the event call back being cancelled for '+ deployId);
                            resolve();
                        }
                    );
                });

                eventPromises.push(txPromise);
            });

            var sendPromise = chain.sendTransaction(request);
            return Promise.all([sendPromise].concat(eventPromises))
            .then((results) => {

                console.debug(' event promise all complete and testing complete');
                return results[0]; // the first returned value is from the 'sendPromise' which is from the 'sendTransaction()' call

            }).catch((err) => {

                console.log(err);
                console.error('Failed to send transaction and get notifications within the timeout period.');
                throw new Error('Failed to send transaction and get notifications within the timeout period.');

            });

        } else {
            console.error('Failed to send Proposal or receive valid response. Response null or status is not 200. exiting...');
            throw new Error('Failed to send Proposal or receive valid response. Response null or status is not 200. exiting...');
        }
    }, (err) => {
Example #19
0
    var req = http.request(options, function(res) {
        logger.debug(util.format('Response, %s',pageLink));

        var result = {
            "remote_proxy":res.headers['remoteproxy'],
            "drill_count":0,
            "cookie":res.headers['Cookie'],
            "url":urlinfo['url'],
            //"url":res.req.path,
            //"statusCode":res.statusCode,
            "origin":urlinfo
        };
        if(result['url'].startsWith('/'))result['url'] = urlUtil.resolve(pageLink,result['url']);
        result['statusCode'] = res.statusCode;
        if(parseInt(res.statusCode)==301||parseInt(res.statusCode)==302){
            if(res.headers['location']){
                result['origin']['redirect'] = urlUtil.resolve(pageLink,res.headers['location']);
                logger.debug(pageLink+' 301 Moved Permanently to '+res.headers['location']);
            }
        }

        var compressed = /gzip|deflate/.test(res.headers['content-encoding']);

        var bufferHelper = new BufferHelper();
//        res.setEncoding('utf8');

        res.on('data', function (chunk) {
            bufferHelper.concat(chunk);
        });

        res.on('end', function (chunk) {
            self.timeout_count--;
            if(timeOuter){
                clearTimeout(timeOuter);
                timeOuter = false;
            }
            result["cost"] = (new Date()) - startTime;
            logger.debug('download '+pageLink+', cost:'+result["cost"]+'ms');


            var page_encoding = urlinfo['encoding'];

            if(page_encoding==='auto'){
                page_encoding = self.get_page_encoding(res.headers);
            }

            page_encoding = page_encoding.toLowerCase().replace('\-','')
            if(!compressed || typeof unzip == 'undefined'){
                if(urlinfo['format']=='binary'){
                    result["content"] = bufferHelper.toBuffer();
                }else{
                    result["content"] = iconv.decode(bufferHelper.toBuffer(),page_encoding);//page_encoding
                }
                spiderCore.emit('crawled',result);
            }else{
                unzip(bufferHelper.toBuffer(), function(err, buff) {
                    if (!err && buff) {
                        if(urlinfo['format']=='binary'){
                            result["content"] = buff;
                        }else{
                            result["content"] = iconv.decode(buff,page_encoding);
                        }
                        spiderCore.emit('crawled',result);
                    }else{
                        spiderCore.emit('crawling_failure',urlinfo,'unzip failure');
                    }
                });
            }
        });
    });
  $("#divResults .clsResult").map(function() {
    var row = $(this);

    // Structured as per rawHtml found @ http://athlinks.com/time.aspx?eventid=115006&courseid=156044&genderpage=A14
    var claimed = row.find("img").attr("src") === "/images/icoMemberSm.gif"
      ? 1
      : 0;

    var a = row.find(":nth-child(2)").text();
    var g = row.find(":nth-child(3)").text();
    var o = row.find(":nth-child(4)").text();
    var ago = util.format("%s/%s/%s", a, g, o);

    var athlete_row = row.find(":nth-child(5)");

    var athlete_ids = _s
      .trim(row.find("a").attr("href"), "http://athlinks.com/result/")
      .split("/");
    var athlinksId = athlete_ids[athlete_ids.length - 2];

    var names = _s.capitalize(_s.humanize(athlete_row.text())).split(" ");

    var firstName = _.first(names);
    var lastName = _.rest(names, 1).join(" ");

    var m_f = row.find(":nth-child(6)").text();
    var age = row.find(":nth-child(7)").text();
    var bib = row.find(":nth-child(8)").text();

    var swimTime = row.find(":nth-child(9)").text();
    var swimPace = row.find(":nth-child(10)").text();
    var swimAgo = row.find(":nth-child(11)").text();

    var cycleTime = row.find(":nth-child(12)").text();
    var cycleSpeed = row.find(":nth-child(13)").text();
    var cycleAgo = row.find(":nth-child(14)").text();

    var runTime = row.find(":nth-child(15)").text();
    var runPace = row.find(":nth-child(16)").text();
    var runAgo = row.find(":nth-child(17)").text();

    var finalTime = row.find(":nth-child(18)").text();

    rows.push({
      claimed: claimed,
      ago: ago,
      firstName: firstName,
      lastName: lastName,
      athlinksId: athlinksId,
      m_f: m_f,
      age: age,
      bib: bib,
      swimTime: swimTime,
      swimPace: swimPace,
      swimAgo: swimAgo,
      t1: "",
      cycleTime: cycleTime,
      cycleSpeed: cycleSpeed,
      cycleAgo: cycleAgo,
      t2: "",
      runTime: runTime,
      runPace: runPace,
      runAgo: _cleanupAgo(runAgo),
      finalTime: finalTime
    });
  });
Example #21
0
					sails.after('lifted', function () {
						console.log();
						sails.log.warn(util.format('Invalid blueprint prefix ("%s") configured for controller `%s` (should start with a `/`).', originalPrefix, globalId));
						sails.log.warn(util.format('For now, assuming you meant:  "%s".', config.prefix));
						STRINGFILE.logMoreInfoLink(STRINGFILE.get('links.docs.config.blueprints'), sails.log.warn);
					});
var restify = function(app, model, opts) {
    var postProcess, lean, filter, contextFilter, postCreate, postDelete,
        usingExpress,
        options = getDefaults(),
        queryOptions = {
            protected: ['skip', 'limit', 'sort', 'populate', 'select', 'lean',
                '$and', '$or', 'query'],//H+ exposes OR, AND and WHERE methods
            current: {}
        };

    for (var prop in opts) {
        if (opts[prop] instanceof Array) {
            options[prop] = [];
            for (var index in opts[prop]) {
                options[prop][index] = opts[prop][index];
            }
        } else {
            options[prop] = opts[prop];
        }
    }

    options.private = options.private ? options.private.split(',') : [];
    options.protected = options.protected ?
        options.protected.split(',') : [];

    model.schema.eachPath(function(name, path) {
        if (path.options.access) {
            switch (path.options.access.toLowerCase()) {
                case 'private':
                    options.private.push(name);
                    break;
                case 'protected':
                    options.protected.push(name);
                    break;
            }
        }
    });

    if (options.exclude) {
        options.private = options.exclude.split(',');
        console.error('Exclude is deprecated. Use private instead');
    }

    lean = options.lean;
    filter = new Filter(model, options.private, options.protected);
    postProcess = options.postProcess || function() {
    };

    if (options.middleware) {
        if (!(options.middleware instanceof Array)) {
            var m = options.middleware;
            options.middleware = [ m ];
        }
    }

    function cleanQuery(req, res, next) {
        queryOptions.current = {};
		var err = null;
        for (var key in req.query) {
            if (queryOptions.protected.indexOf(key) !== -1) {
                queryOptions.current[key] = req.query[key];
				delete req.query[key];
            } else if (!model.schema.paths.hasOwnProperty(key)) {
                var keys = key.match('.') &&
					!model.schema.paths.hasOwnProperty(key) ?
						key.split('.') : false;
                if(!keys || !model.schema.paths.hasOwnProperty(keys[0]) ||
					!model.schema.paths[keys[0]].schema.paths.hasOwnProperty(keys[1]) ||
					keys.length > 2) {
						
					if(!model.schema.virtuals.hasOwnProperty(keys[0])) {
						err = createError(400);
						break;
					}
                }
            }
        }
		
		if(err) {
			onError(res,next,err);
		} else {
	        next();
		}
    }
	
	usingExpress = !options.restify;
	
	function createError(status) {
		var err = new Error(http.STATUS_CODES[status]);
		err.status = status;
		return err;
	}
	
	function onError(res, next, err) {
		if(usingExpress) {
			//next(err);
			res.status(err.status).send(err.message);
		} else {
			res.send(err.status, err.message);
		}
	}

    options.middleware.push(cleanQuery);

    var outputFn = options.outputFn ?
        options.outputFn : (options.restify ?
        outputRestify : outputExpress);
	if(undefined === app.delete) {
	    app.delete = app.del;
	}
    var apiUri = '%s%s/%s';

    var modelName;

	if (options.name) {
		modelName = options.name;
	}
	else {
		modelName = model.modelName;
	}
	modelName = options.plural ? inflection.pluralize(modelName)
			: modelName;

    modelName = options.lowercase === true ? modelName.toLowerCase()
        : modelName;

    var uri_item = util.format(apiUri, options.prefix, options.version,
        modelName);
    if (uri_item.indexOf('/:id') === -1) {
        uri_item += '/:id';
    }
    var uri_items = uri_item.replace('/:id', '');

    var uri_count = uri_items + '/count';

    function buildQuery(query, req) {
        var options = req.query,
            excludedarr = filter.getExcluded(req.access);

        var arr, i, re;
        for (var key in options) {
            if (excludedarr.indexOf(key) !== -1) {
                // caller tries to query for excluded keys. for security
                // reasons, we will skip the first -1 objects (to provoke
                // an error) and immediately return;
                return query.skip(-1);
            }

            query.where(key);
            var value = options[key];

            if ('~' === value[0]) {
                re = new RegExp(value.substring(1), 'i');
                query.where(key).regex(re);
            } else if ('>' === value[0]) {
                if ('=' === value[1]) {
                    query.gte(value.substr(2));
                } else {
                    query.gt(value.substr(1));
                }
            } else if ('<' === value[0]) {
                if ('=' === value[1]) {
                    query.lte(value.substr(2));
                } else {
                    query.lt(value.substr(1));
                }
            } else if ('!' === value[0] && '=' === value[1]) { //H+ for !=
                query.ne(value.substr(2));
            } else if ('[' === value[0] && ']' === value[value.length - 1]) {
                query.in(value.substr(1, value.length - 2).split(','));
            } else {
                query.equals(value);
            }
        }

        //H+ exposes Query AND, OR and WHERE methods
        if (queryOptions.current.query) {
            query.where(JSON.parse(queryOptions.current.query,
                jsonQueryParser));
        }
        //TODO - as introduction of QUERY param obsoletes need of $and, $or
        if (queryOptions.current.$and) {
            query.and(JSON.parse(queryOptions.current.$and, jsonQueryParser));
        }
        if (queryOptions.current.$or) {
            query.or(JSON.parse(queryOptions.current.$or, jsonQueryParser));
        }
        //H+ exposes Query AND, OR methods

        if (queryOptions.current.skip) {
            query.skip(queryOptions.current.skip);
        }
        if (queryOptions.current.limit) {
            query.limit(queryOptions.current.limit);
        }
        if (queryOptions.current.sort) {
            query.sort(queryOptions.current.sort);
        }
        var selectObj = {root: {}};
        if (queryOptions.current.select) {

            if (queryOptions.current.select) {
                arr = queryOptions.current.select.split(',');
                for (i = 0; i < arr.length; ++i) {
                    if (arr[i].match(/\./)) {
                        var subSelect = arr[i].split('.');
                        if (!selectObj[subSelect[0]]) {
                            selectObj[subSelect[0]] = {};
                            //selectObj.root[subSelect[0]] = 1;
                        }
                        selectObj[subSelect[0]][subSelect[1]] = 1;
                    } else {
                        selectObj.root[arr[i]] = 1;
                    }
                }
            }
            query = query.select(selectObj.root);
        }
        if (queryOptions.current.populate) {
            arr = queryOptions.current.populate.split(',');
            for (i = 0; i < arr.length; ++i) {
                if (!_.isUndefined(selectObj[arr[i]]) &&
					!_.isEmpty(selectObj.root)) {
                    selectObj.root[arr[i]] = 1;
                }
                query = query.populate(arr[i], selectObj[arr[i]]);
            }
            query.select(selectObj.root);
        }
        return query;
    }

    //H+ - JSON query param parser
    //TODO - improve to serve recursive logical operators
    function jsonQueryParser(key, value) {
        if (_.isString(value)) {
            if ('~' === value[0]) { //parse RegExp
                return new RegExp(value.substring(1), 'i');
            } else if ('>' === value[0]) {
                if ('=' === value[1]) {
                    return {$gte: value.substr(2)};
                } else {
                    return {$gt: value.substr(1)};
                }
            } else if ('<' === value[0]) {
                if ('=' === value[1]) {
                    return {$lte: value.substr(2)};
                } else {
                    return {$lt: value.substr(1)};
                }
            } else if ('!' === value[0] && '=' === value[1]) {
                return {$ne: value.substr(2)};
            }
        } else if (_.isArray(value)) {
            if (model.schema.paths.hasOwnProperty(key)) {
                return {$in: value};
            }
        }
        return value;
    }

    //H+ - JSON query param parser

    function ensureContentType(req, res, next) {
        var ct = req.headers['content-type'];
        if (-1 === ct.indexOf('application/json')) {
			onError(res,next,createError(400));
        } else {
            next();
        }
    }

    function createSingleObject(body) {
        for (var key in body) {
            var path = model.schema.path(key);
            if (typeof path === 'undefined') {
                continue;
            }

            if (path.caster !== undefined) {
                if (path.caster.instance === 'ObjectID') {
                    if (_.isArray(body[key])) {
                        for (var k = 0; k < body[key].length; ++k) {
                            if (typeof body[key][k] === 'object') {
                                body[key][k] = body[key][k]._id;
                            }
                        }
                    } else if ((typeof body[key] === 'object') &&
                        (body[key] !== null)) {
                        body[key] = body[key]._id;
                    }
                }
            } else if ((path.instance === 'ObjectID') &&
                (typeof body[key] === 'object') &&
                (body[key] !== null)) {
                body[key] = body[key]._id;
            }
        }
    }

    function createObject(req, res, next) {
        if (!req.body) {
			onError(res,next,createError(400));
            return;
        }

        var filterOpts = { access: req.access };
        req.body = filter.filterObject(req.body, filterOpts);
        if (model.schema.options._id) {
            delete req.body._id;
        }
        if (model.schema.options.versionKey) {
            delete req.body[model.schema.options.versionKey];
        }

        var key, path;

        if (_.isArray(req.body)) {
            for (var i = 0; i < req.body.length; ++i) {
                createSingleObject(req.body[i]);
            }
        } else {
            createSingleObject(req.body);
        }

        model.create(req.body, function(err, item) {
            if (err) {
                res.setHeader('Content-Type', 'application/json');
				var forwardErr = new Error(JSON.stringify(err));
				forwardErr.status = 400;
				onError(res,next,forwardErr);
            } else {
                var result = null;

                if (_.isArray(req.body)) {
                    var items = Array.prototype.slice.call(arguments, 1);

                    result = filter.filterObject(items, filterOpts);
                } else {
                    result = filter.filterObject(item, filterOpts);
                }

                postCreate(res, result, function(err) {
                    if (err) {
						onError(res,next,createError(400));
					}
                    else {
                        outputFn(res, result);
                        next();
                    }
                });
            }
        });
    }

    function modifyObject(req, res, next) {
        var byId = {};
        byId[options.idProperty || '_id'] = req.params.id;
        if (!req.body) {
			onError(res,next,createError(400));
            return;
        }

        var filterOpts = { access: req.access };
        req.body = filter.filterObject(req.body, filterOpts);

        delete req.body._id;
        if (model.schema.options.versionKey) {
            delete req.body[model.schema.options.versionKey];
        }

        for (var key in req.body) {
            var path = model.schema.path(key);
            if (typeof path === 'undefined') {
                continue;
            }

            if (path.caster !== undefined) {
                if (path.caster.instance === 'ObjectID') {
                    if (_.isArray(req.body[key])) {
                        for (var j = 0; j < req.body[key].length; ++j) {
                            if (typeof req.body[key][j] === 'object') {
                                req.body[key][j] = req.body[key][j]._id;
                            }
                        }
                    } else if ((typeof req.body[key] === 'object') &&
                        (req.body[key] !== null)) {
                        req.body[key] = req.body[key]._id;
                    }
                }
            } else if ((path.instance === 'ObjectID') &&
                (typeof req.body[key] === 'object') &&
                (req.body[key] !== null)) {
                req.body[key] = req.body[key]._id;
            }
        }

        if (options.findOneAndUpdate) {
            contextFilter(model, req, function(filteredContext) {
                filteredContext.findOneAndUpdate(byId, req.body, {},
                    function(err, item) {
                        if (err || !item) {
							onError(res,next,createError(404));
                        } else {
                            item = filter.filterObject(item, filterOpts);
                            outputFn(res, item);
                            next();
                        }
                    });
            });
        } else {
            contextFilter(model, req, function(filteredContext) {
                filteredContext.findOne(byId, function(err, doc) {
                    if (err || !doc) {
						onError(res,next,createError(404));
                    } else {
                        for (var key in req.body) {
                            doc[key] = req.body[key];
                        }
                        doc.save(function(err, item) {
                            if (err) {
								onError(res,next,createError(404));
                            } else {
                                item = filter.filterObject(item, filterOpts);
                                outputFn(res, item);
                                next();
                            }
                        });
                    }
                });
            });
        }
    }

    var write_middleware = options.middleware.slice(-1);

    if (options.prereq) {
        var allowMW = permissions.allow(usingExpress, options.prereq);
        write_middleware = [allowMW].concat(write_middleware);
    }

    var delete_middleware = write_middleware.slice(0);

    if (options.access) {
        var accessMW = permissions.access(options.access);
        options.middleware.push(accessMW);
        write_middleware.push(accessMW);
    }

    write_middleware.push(ensureContentType);

    if (options.contextFilter) {
        contextFilter = options.contextFilter;
    }
    else {
        contextFilter = function(model, req, done) {
            done(model);
        };
    }
    if (options.postCreate) {
        postCreate = options.postCreate;
    }
    else {
        postCreate = function(res, result, done) {
            done();
        };
    }
    if (options.postDelete) {
        postDelete = options.postDelete;
    }
    else {
        postDelete = function(res, result, done) {
            done();
        };
    }

    app.get(uri_items, options.middleware, function(req, res, next) {
        contextFilter(model, req, function(filteredContext) {
            buildQuery(filteredContext.find(), req).lean(lean)
                .exec(function(err, items) {
                    if (err) {
						onError(res,next,createError(400));
                    } else {
                        var populate = queryOptions.current.populate,
                            opts = {
                                populate: populate,
                                access: req.access
                            };

                        items = filter.filterObject(items, opts);

                        outputFn(res, items);
                        next();
                    }
                });
        });
    }, postProcess);

    app.get(uri_count, options.middleware, function(req, res, next) {
        contextFilter(model, req, function(filteredContext) {
            buildQuery(filteredContext.count(), req).exec(function(err, count) {
                if (err) {
					onError(res,next,createError(400));
                } else {
                    outputFn(res, { count: count });
                    next();
                }
            });
        });
    }, postProcess);

    app.post(uri_items, write_middleware, createObject, postProcess);

    if (!options.strict) {
        app.put(uri_items, write_middleware, createObject, postProcess);
    }

    if (!options.strict) {
        // TODO: reconsider this function
        app.delete(uri_items, delete_middleware, function(req, res, next) {
            contextFilter(model, req, function(filteredContext) {
                var results, error;
                buildQuery(filteredContext.find(), req).remove(function(err) {
                    if (err) {
						onError(res,next,createError(400));
                    } else {
						if(usingExpress) {
	                        res.status(200).end();
						} else {
	                        res.send(200);
						}
                        next();
                    }
                });
            });
        }, postProcess);
    }

    app.get(uri_item, options.middleware, function(req, res, next) {
        var byId = {};
        byId[options.idProperty || '_id'] = req.params.id;
        contextFilter(model, req, function(filteredContext) {
            buildQuery(filteredContext.findOne().and(byId), req).lean(lean)
                .findOne(function(err, item) {

                    if (err || !item) {
						onError(res,next,createError(404));
                    } else {
                        var populate = queryOptions.current.populate,
                            opts = {
                                populate: populate,
                                access: req.access
                            };

                        item = filter.filterObject(item, opts);

                        outputFn(res, item);
                        next();
                    }
                });
        });
    }, postProcess);

    if (!options.strict) {
        // TODO: POST (create) doesn't make sense here
        app.post(uri_item, write_middleware, modifyObject, postProcess);
    }

    app.put(uri_item, write_middleware, modifyObject, postProcess);

    app.delete(uri_item, delete_middleware, function(req, res, next) {
        var byId = {};
        byId[options.idProperty || '_id'] = req.params.id;

        if (options.findOneAndUpdate) {
	        contextFilter(model, req, function(filteredContext) {
	            var results, error;
	            filteredContext.find().and(byId).findOneAndRemove(
					function(err, result) {
	                if (err || !result) {
						onError(res,next,createError(404));
	                } else {
	                    postDelete(res, [result], function(err) {
	                        if (err) {
								onError(res,next,createError(400));
	                        }
	                        else {
								if(usingExpress) {
			                        res.status(200).end();
								} else {
			                        res.send(200);
								}
	                            next();
	                        }
	                    });
	                }
	            });
	        });
		} else {
	        contextFilter(model, req, function(filteredContext) {
                filteredContext.findOne(byId, function(err, doc) {
                    if (err || !doc) {
						onError(res,next,createError(404));
                    } else {
                        doc.remove(function(err, result) {
		                    postDelete(res, [doc], function(err) {
		                        if (err) {
									onError(res,next,createError(400));
		                        }
		                        else {
									if(usingExpress) {
				                        res.status(200).end();
									} else {
				                        res.send(200);
									}
		                            next();
		                        }
		                    });
                        });
                    }
		        });
	        });
		}
    }, postProcess);
};
var path = require('path');
var webpack = require('webpack');
var HtmlWebpackPlugin = require('html-webpack-plugin');

var pkg = require('./package.json');
var ExtractTextPlugin = require('extract-text-webpack-plugin');
var autoprefixer = require('autoprefixer');
var util = require('util');
var webpack = require('webpack');


var DEBUG = process.env.NODE_ENV === 'development';
var TEST = process.env.NODE_ENV === 'test';

var cssBundle = path.join('css', util.format('app.%s.css', pkg.version));
var jsBundle = path.join('js', util.format('app.%s.js', pkg.version));

var jsxLoader;
var sassLoader;
var cssLoader;
var fileLoader = 'file-loader?name=[path][name].[ext]';
var htmlLoader = [
    'file-loader?name=[path][name].[ext]',
    'template-html-loader?' + [
        'raw=true',
        'engine=lodash',
        'version=' + pkg.version,
        'title=' + pkg.name,
        'debug=' + DEBUG
    ].join('&')
].join('!');
Example #24
0
function getAllSubmissionsUrl(handle){
    return util.format(SUBMISSIONS_URL, handle);
}
Example #25
0
SpiderJob.prototype.snapshot = function (page, url){
    var self = this;

    console.log(util.format('Do snapshot for url [%s].', url));

    setTimeout(function () {
        console.log(util.format('Snapshot time out is tirgger, [%s].', url));

        page.evaluate(function () {
            return document.documentElement.outerHTML;
        },function(err, doc){
            if(doc){
                var filePath = buildSnapshotFileAndPath(url, self.snapshotDir);

                fs.writeFile(filePath, doc, function(e){//会先清空原先的内容
                    if(e) throw e;
                    console.log(util.format('Snapshot file [%s] is created!', filePath));
                });
            }else{
                console.error(util.format('Html snapshot on [%s] is empty', url));
            }
        });

        self.parserPage(page);
    }, 3000);

    /**
     * Build the file path of snapshot for url.
     */
    function buildSnapshotFileAndPath(_url, snapshotDir){
        var fileName, filePath,
            self = this;

        fileName = escapeFileName(_url);

        filePath = snapshotDir;
        if(filePath.indexOf('\\', filePath.length - 1) === -1){
            filePath += '\\';
        }

        if(!fs.existsSync(filePath)){
            fs.mkdirSync(filePath);
        }

        return filePath + fileName;
    }

    /**
     * Escape url to suitable file name.
     * @param input input url
     * @return {*} output file name
     */
    function escapeFileName(input){
        var toEscape = '\\ / : * ? " < >',
        //toEscapes = toEscape.split(' '),
            toEscapes = ['\\', '\/', '\:', '\*', '\?', '"', '<', '>'],
            output = input;

//        toEscapes.forEach(function(item){
//            // output = output.replace(new RegExp(item, 'gm'), '_');
//            output = output.replace(item, '_');
//            console.log(output);
//        });
        console.log(output);
        output = output.replace(/\\/ig, '_');
        output = output.replace(/\//ig, '_');
        output = output.replace(/:/ig, '_');
        output = output.replace(/\*/ig, '_');
        output = output.replace(/\?/ig, '_');
        output = output.replace(/"/ig, '_');
        output = output.replace(/</ig, '_');
        output = output.replace(/>/ig, '_');
        console.log(output);

        return output;
    }
};
Example #26
0
function getSubmissionUrl(contestID, submissionID){
 return util.format(SUBMISSION_URL, contestID, submissionID);
}
Example #27
0
  return function parser(inError, body) {
    /*jshint maxdepth:4 */

    var code = response.statusCode
      , errors = []
      , errorClass
      , json
      , returned
      ;

    if (code !== 200) logger.debug("Got %s as a response code from the collector.", code);

    if (inError) errors.push(inError);

    if (body) {
      try {
        json = JSON.parse(body);

        // Can be super verbose, but useful for debugging.
        logger.trace({response : json}, "Deserialized from collector:");

        // If we get messages back from the collector, be polite and pass them along.
        returned = json[RESPONSE_VALUE_NAME];
        if (returned && returned.messages) {
          returned.messages.forEach(function cb_forEach(element) { logger.info(element.message); });
        }

        /* Wait to deal with errors in the response until any messages have
         * been passed along. Otherwise, ensure that there was a return
         * value, raising an error if not.
         *
         * Some errors are only interesting if the status code indicates
         * that the request went bad already, so filter out adding more
         * errors when statusCode is not OK (200).
         */
        var exception = json[EXCEPTION_VALUE_NAME];
        if (exception) {
          if (exception.message) {
            errors.push(new Error(exception.message));
          }
          else if (code === 200 ) {
            errors.push(new Error('OneAPM internal error'));
          }

          if (exception.error_type) errorClass = exception.error_type;
        }
        else if (code === 200 && returned === undefined) {
          errors.push(new Error(format('No data found in response to %s.', name)));
        }
      }
      catch (error) {
        errors.push(error);
      }
    }
    else {
      errors.push(new Error(format('No body found in response to %s.', name)));
    }

    if (code !== 200) {
      errors.push(new Error(format('Got HTTP %s in response to %s.', code, name)));
    }

    var error;
    if (errors.length > 0) {
      error = errors.shift();
      error.statusCode = code;
      // Preserve a consistent hidden class (cheaper than subclassing Error).
      error.class = errorClass ? errorClass : undefined;
      error.laterErrors = (errors.length > 0) ? errors : undefined;
    }

    // Raw json is useful for testing and logging.
    process.nextTick(function cb_nextTick() { callback(error, returned, json); });
  };
Example #28
0
function createOutputFolder(contestName){
    fs.mkdirSync(util.format(OUTPUT_FOLDER + '/%s', contestName));
}
Example #29
0
 row[key].split(',').map(function(v) { return v.trim(); }).forEach((nodeName) => {
     if (nodeName.length !== 1) throw new Error(util.format('*** invalid relation node member "%s"'), nodeName);
     let node = this.findNodeByName(nodeName);
     if (!node) throw new Error(util.format('*** unknown relation node member "%s"'), nodeName);
     relation.addMember('node', node.id, isNode[1]);
 });
Example #30
0
      function doAppCall(endpoints, func,params,liveflag,cb) {
        if('object'!== typeof endpoints || endpoints.status !== 200) return cb(nodejsutil.format(ERRORS.AppNotFound,params.guid));
        var retdata = JSON.parse(endpoints['body']);
        var callurl = (liveflag)
          ? retdata['hosts']['live-url']
          : retdata['hosts']['development-url'];
          callurl+="/cloud/"+func;
        var payload = "";
          try{
              payload = JSON.stringify(params);
          }catch(e){
              return cb(" Could not send params " + e.message);
          }
        var urlprops = url.parse(callurl);
        //sometimes there is a port appended to the host url
        var portmatch = urlprops.host.match(/[:\d]*$/);
        var port = (portmatch === null)? 443 : portmatch[0].toString().replace(/:/,"");
        urlprops.host = urlprops.host.replace(/[:\d]*$/,"");
        var headers = {};
        headers["content-type"]   = "application/json";
        headers["content-length"] = new Buffer(payload).length ||0;
        headers["accept"]         = "application/json";
        addAppApiKeyHeader(headers);

        var httpsopts = {
          host:urlprops.host,
          path:urlprops.path,
          port:port,
          method:"POST",
          headers:headers
       };
       //finally make call to other app
       var req = https.request(httpsopts,function (res){
           req.socket.setTimeout(60000);
           req.connection.setTimeout(60000);
           var retData="";
           res.on('data', function (data){
               retData+=data;
           });
           res.on('end',function () {
            return cb(null,retData);
           });
       });
       req.on('error', function (err){
        return cb(err);
       });
       req.write(payload);
       req.end();
     };