function ajax(options, callback) {
if (typeof options === "function") {
callback = options;
options = {};
}
options = extend(true, {}, options);
function call(fun) {
/* jshint validthis: true */
var args = Array.prototype.slice.call(arguments, 1);
if (typeof fun === typeof Function) {
fun.apply(this, args);
}
}
var defaultOptions = {
method : "GET",
headers: {},
json: true,
processData: true,
timeout: 10000,
cache: false
};
options = extend(true, defaultOptions, options);
// cache-buster, specifically designed to work around IE's aggressive caching
// see http://www.dashbay.com/2011/05/internet-explorer-caches-ajax/
if (options.method === 'GET' && !options.cache) {
var hasArgs = options.url.indexOf('?') !== -1;
options.url += (hasArgs ? '&' : '?') + '_nonce=' + uuid(16);
}
function onSuccess(obj, resp, cb) {
if (!options.binary && !options.json && options.processData &&
typeof obj !== 'string') {
obj = JSON.stringify(obj);
} else if (!options.binary && options.json && typeof obj === 'string') {
try {
obj = JSON.parse(obj);
} catch (e) {
// Probably a malformed JSON from server
call(cb, e);
return;
}
}
if (Array.isArray(obj)) {
obj = obj.map(function (v) {
var obj;
if (v.ok) {
return v;
} else if (v.error && v.error === 'conflict') {
obj = errors.REV_CONFLICT;
obj.id = v.id;
return obj;
} else if (v.missing) {
obj = errors.MISSING_DOC;
obj.missing = v.missing;
return obj;
}
});
}
call(cb, null, obj, resp);
}
function onError(err, cb) {
var errParsed, errObj, errType, key;
try {
errParsed = JSON.parse(err.responseText);
//would prefer not to have a try/catch clause
for (key in errors) {
if (errors.hasOwnProperty(key) && errors[key].name === errParsed.error) {
errType = errors[key];
break;
}
}
errType = errType || errors.UNKNOWN_ERROR;
errObj = errors.error(errType, errParsed.reason);
} catch (e) {
for (var key in errors) {
if (errors.hasOwnProperty(key) && errors[key].status === err.status) {
errType = errors[key];
break;
}
}
errType = errType || errors.UNKNOWN_ERROR;
errObj = errors.error(errType);
}
call(cb, errObj);
}
if (process.browser && typeof XMLHttpRequest !== 'undefined') {
var timer, timedout = false;
var xhr = new XMLHttpRequest();
xhr.open(options.method, options.url);
xhr.withCredentials = true;
if (options.json) {
options.headers.Accept = 'application/json';
options.headers['Content-Type'] = options.headers['Content-Type'] ||
'application/json';
if (options.body && options.processData && typeof options.body !== "string") {
options.body = JSON.stringify(options.body);
}
}
if (options.binary) {
xhr.responseType = 'arraybuffer';
}
var createCookie = function (name, value, days) {
var expires = "";
if (days) {
var date = new Date();
date.setTime(date.getTime() + (days * 24 * 60 * 60 * 1000));
expires = "; expires=" + date.toGMTString();
}
document.cookie = name + "=" + value + expires + "; path=/";
};
for (var key in options.headers) {
if (key === 'Cookie') {
var cookie = options.headers[key].split('=');
createCookie(cookie[0], cookie[1], 10);
} else {
xhr.setRequestHeader(key, options.headers[key]);
}
}
if (!("body" in options)) {
options.body = null;
}
var abortReq = function () {
timedout = true;
xhr.abort();
call(onError, xhr, callback);
};
xhr.onreadystatechange = function () {
if (xhr.readyState !== 4 || timedout) {
return;
}
clearTimeout(timer);
if (xhr.status >= 200 && xhr.status < 300) {
var data;
if (options.binary) {
data = createBlob([xhr.response || ''], {
type: xhr.getResponseHeader('Content-Type')
});
} else {
data = xhr.responseText;
}
call(onSuccess, data, xhr, callback);
} else {
call(onError, xhr, callback);
}
};
if (options.timeout > 0) {
timer = setTimeout(abortReq, options.timeout);
xhr.onprogress = function () {
clearTimeout(timer);
timer = setTimeout(abortReq, options.timeout);
};
if (xhr.upload) { // does not exist in ie9
xhr.upload.onprogress = xhr.onprogress;
}
}
xhr.send(options.body);
return {abort: abortReq};
} else {
if (options.json) {
if (!options.binary) {
options.headers.Accept = 'application/json';
}
options.headers['Content-Type'] = options.headers['Content-Type'] ||
'application/json';
}
if (options.binary) {
options.encoding = null;
options.json = false;
}
if (!options.processData) {
options.json = false;
}
return request(options, function (err, response, body) {
if (err) {
err.status = response ? response.statusCode : 400;
return call(onError, err, callback);
}
var error;
var content_type = response.headers['content-type'];
var data = (body || '');
// CouchDB doesn't always return the right content-type for JSON data, so
// we check for ^{ and }$ (ignoring leading/trailing whitespace)
if (!options.binary && (options.json || !options.processData) &&
typeof data !== 'object' &&
(/json/.test(content_type) ||
(/^[\s]*\{/.test(data) && /\}[\s]*$/.test(data)))) {
data = JSON.parse(data);
}
if (response.statusCode >= 200 && response.statusCode < 300) {
call(onSuccess, data, response, callback);
}
else {
if (options.binary) {
data = JSON.parse(data.toString());
}
if (data.reason === 'missing') {
error = errors.MISSING_DOC;
} else if (data.reason === 'no_db_file') {
error = errors.error(errors.DB_MISSING, data.reason);
} else if (data.error === 'conflict') {
error = errors.REV_CONFLICT;
} else {
error = errors.error(errors.UNKNOWN_ERROR, data.reason, data.error);
}
error.status = response.statusCode;
call(callback, error);
}
});
}
}
function getGames( sport, onSuccess, onError ) {
if ( gameList[ sport ].length > 0 && (new Date() - gameListInstant[ sport ] < CACHE_INTERVAL ) ) {
getGamesFromList();
}
else {
request( 'http://api.thescore.com/' + sport + '/schedule?utc_offset=-18000', gotSchedule);
}
function gotSchedule( error, response, body ) {
try {
var schedule = JSON.parse( body );
gameList[ sport ] = schedule.current_group.event_ids;
gameListInstant[ sport ] = new Date();
getGamesFromList();
}
catch( e ) {
onError();
}
}
function getGamesFromList() {
request( 'http://api.thescore.com/' + sport + '/events?id.in=' + gameList[ sport ].join('%2C'), gotGames );
}
function gotGames( error, response, body ) {
try {
var games = JSON.parse( body );
for ( var i = 0; i < games.length; i++ ) {
var game = games[ i ];
if ( game.status === 'pre_game' ) {
game.game_status = moment( new Date( game.game_date ) ).tz('America/Chicago').format('h:mm A');
}
else {
game.game_status = game.box_score.progress.string;
}
}
games.sort( function(a,b) {
if ( a.status !== 'postponed' && b.status === 'postponed' ) {
return -1;
}
if ( a.status === 'postponed' && b.status !== 'postponed' ) {
return 1;
}
if ( a.status !== 'final' && b.status === 'final' ) {
return -1;
}
if ( a.status === 'final' && b.status !== 'final' ) {
return 1;
}
var a_moment = moment( new Date( a.game_date ) );
var b_moment = moment( new Date( b.game_date ) );
return a_moment.diff( b_moment );
});
onSuccess( games );
}
catch( e ) {
onError();
}
}
}
return function (req, res) {
if (!config.get('planName') || !config.get('key') || !config.get('gamiHost') || typeof config.get('getLoginUid') != 'function') {
if (config.get('errorHandler') && typeof(config.get('errorHandler')) == 'function') {
config.get('errorHandler')(res);
} else {
res.json(500, {
error : 'Proxy error !'
})
}
}
//use white url list for access control
var planName = config.get('planName');
var validURLWhilteList = {};
validURLWhilteList['\/service\/plan\/' + config.get('planName') + '\/user\/{uid}\/?'] = 'GET,PUT';//retrieve & update login user's profile
validURLWhilteList['\/service\/plan\/' + config.get('planName') + '\/user\/{uid}\/defaultTitle/?'] = 'PUT'; //update login user's title
validURLWhilteList['\/service\/plan\/' + config.get('planName') + '\/user\/{uid}\/pic/?'] = 'PUT'; //update login user's picture
validURLWhilteList['\/service\/plan\/' + config.get('planName') + '\/user\/{uid}\/leaderboard(/?|/.+)'] = 'GET'; //get login user's leaderboard
validURLWhilteList['\/service\/plan\/' + config.get('planName') + '\/user\/(.+)\/?'] = 'GET'; //get any user's profile
validURLWhilteList['\/service\/plan\/' + config.get('planName') + '\/user\/(.+)\/pic\/?'] = 'GET'; //get any user's picture
validURLWhilteList['\/service\/plan\/' + config.get('planName') + '\/var(/?|/.+)'] = 'GET'; //get variable definition
validURLWhilteList['\/service\/plan\/' + config.get('planName') + '\/deed(/?|/.+)'] = 'GET'; //get deed definition
validURLWhilteList['\/service\/plan\/' + config.get('planName') + '\/mission(/?|/.+)'] = 'GET'; //get mission definition
validURLWhilteList['\/trigger\/plan\/' + config.get('planName') + '\/mission(/?|/.+)'] = 'GET'; //get any user's mission history
validURLWhilteList['\/cometd(/?|/.+)'] = 'POST'; //cometd notification
var self = this;
var uid = config.get('getLoginUid').call(self, req);
if(uid == null){
console.warn("cannot find login user to update resource - return 403");
res.json(403, {
error : 'unauthorize !'
})
}else{
var matched = false;
for(var url in validURLWhilteList){
var matchUrl = url.replace('{uid}', uid);
if(req.url.match(matchUrl) && validURLWhilteList[url].indexOf(req.method) >= 0){
matched = true;
break;
}
}
if(!matched){
console.warn("request URL(" + req.url + ") is not authorized for user(" + uid + ") - return 403");
res.json(403, {
error : 'unauthorize !'
})
}
}
//console.log('[proxy]original url:' + req.url);
if (req.url.indexOf('/proxy/') >= 0)
req.url = req.url.substring(req.url.indexOf('/proxy/') + '/proxy'.length);
if (req.url.indexOf('?') < 0) {
req.url += '?';
}
req.url += '&key=' + config.get('key') + '&tenantId=' + config.get('tenantId');
var servicePath = 'https://' + config.get('gamiHost');
if (config.get('gamiPort'))
servicePath += ':' + config.get('gamiPort');
url = servicePath + req.url;
if(req.method ==='GET'){
//console.log('[proxy]GET from:' + url);
req.pipe(request(url)).pipe(res);
}else if(req.method ==='POST'){
// console.log('[proxy]POST to:' + url);
// console.log('[proxy]BODY is: ' + JSON.stringify(req.body))
req.pipe(request.post(url, {body:JSON.stringify(req.body)})).pipe(res);
}else if(req.method ==='PUT'){
//console.log('[proxy]PUT to:' + url);
req.pipe(request.put(url, {body:JSON.stringify(req.body)})).pipe(res);
}
};
var fs = require("fs");
var options = {
headers: {'User-Agent': 'Wikilipsum'},
json:true
};
options["url"] = "http://en.wikipedia.org/w/api.php?action=query&list=random&format=json&rnnamespace=0&rnlimit=1";
request(options, function (error, response, body) {
var pageId = body.query.random[0].id;
options["url"] = "http://en.wikipedia.org/w/api.php?action=parse&prop=text&format=json&pageid=" + pageId;
request(options, function (error, response, body) {
var permalink = body.parse.title.toLowerCase().replace(/\W/g, '-').replace(/-+/g, '-');
var content = htmlToText.fromString(body.parse.text["*"], {tables: true});
content = content.replace(/\[.+\]/g, "");
content = content.replace(/\/wiki\/File.+\.\w\w\w/g, "");
var page = "---\n";
page += "layout: post\n";
page += 'title: "' + body.parse.title + '"\n';
page += 'permalink: /' + permalink + '\n';
page += "---\n";
page += content;
var date = new Date().toISOString().substring(0, 10);
fs.writeFile('../_posts/' + date + '-' + permalink + '.md', page);
});
});
var request = require('request');
request('http://www.google.com', function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body) // Print the google web page.
}
})
return new Promise(function(resolve, reject) {
var headers = {
'content-type': 'application/json',
};
if (bot.config && bot.config.studio_token) {
options.uri = options.uri + '?access_token=' + bot.config.studio_token;
} else if (controller.config && controller.config.studio_token) {
options.uri = options.uri + '?access_token=' + controller.config.studio_token;
} else {
// console.log('DEBUG: Making an unathenticated request to stats api');
}
options.headers = headers;
var now = new Date();
if (options.now) {
now = options.now;
}
var stats_body = {};
stats_body.botHash = botHash(bot);
if (bot.type == 'slack' && bot.team_info) {
stats_body.team = md5(bot.team_info.id);
}
if (bot.type == 'ciscospark' && message && message.original_message && message.original_message.orgId) {
stats_body.team = md5(message.original_message.orgId);
}
stats_body.channel = options.form.channel;
stats_body.user = options.form.user;
stats_body.type = options.form.type;
stats_body.time = now;
stats_body.meta = {};
stats_body.meta.user = options.form.user;
stats_body.meta.channel = options.form.channel;
if (options.form.final_thread) {
stats_body.meta.final_thread = options.form.final_thread;
}
if (bot.botkit.config.clientId) {
stats_body.meta.app = md5(bot.botkit.config.clientId);
}
stats_body.meta.timestamp = options.form.timestamp;
stats_body.meta.bot_type = options.form.bot_type;
stats_body.meta.conversation_length = options.form.conversation_length;
stats_body.meta.status = options.form.status;
stats_body.meta.type = options.form.type;
stats_body.meta.command = options.form.command;
options.form = stats_body;
stats_body.meta.timestamp = options.now || now;
request(options, function(err, res, body) {
if (err) {
return reject(err);
}
var json = null;
try {
json = JSON.parse(body);
} catch (e) {
}
if (!json || json == null) {
return reject('Response from Botkit Studio API was empty or invalid JSON');
} else if (json.error) {
return reject(json.error);
} else {
resolve(json);
}
});
});
iotAgentLib.clearAll(function() {
request(provisionOptions, function(error, response, body) {
done();
});
});
var getSubscribeNewsDetail = function(entry) {
// http://r.inews.qq.com/getSubNewsContent?id=20131129A000H600&qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&appver=16_android_3.2.1
var subscribeNewsDetailLink = 'http://r.inews.qq.com/getSubNewsContent?id=%s&qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&appver=16_android_3.2.1';
var docid = util.format("%s",entry.id);
var url = util.format(subscribeNewsDetailLink, docid);
var req = {uri: url, method: "GET", headers: headers};
if(proxyEnable) {
req.proxy = proxyUrl;
}
request(req, function (err, res, body) {
var json = data2Json(err, res, body);
if((!json) || (json.ret != 0)) {
console.log("hzfdbg file[" + __filename + "]" + " getSubscribeNewsDetail():ret="+json.ret);
return;
}
var jObj = json;
var obj = entry;
News.findOne(genQqFindCmd(site, entry), function(err, result) {
if(err || result) {
return;
}
obj.docid = encodeDocID(site, docid);
obj.site = site;
obj.body = jObj.content.text;
obj.img = [];
for(key in jObj.attribute) {
var html = genLazyLoadHtml(entry.title, jObj.attribute[key].url);
if(jObj.attribute[key].desc) {
html = html + jObj.attribute[key].desc + '<br/>';
}
obj.img[obj.img.length] = jObj.attribute[key].url;
obj.body = obj.body.replace(util.format('<!--%s-->', key), html);
}
obj.video = [];
obj.link = '';
if(entry.url) {
obj.link = entry.url;
}else {
obj.link = util.format("http://view.inews.qq.com/a/%s", entry.id);
}
obj.title = entry.title;
obj.ptime = entry.time;
obj.time = new Date(entry.time);
obj.marked = obj.body;
obj.created = new Date();
obj.views = 1;
obj.tags = entry.tagName;
obj.digest = genDigest(obj.body);
obj.cover = '';
if(entry.thumbnails_qqnews && entry.thumbnails_qqnews[0]) {
obj.cover = entry.thumbnails_qqnews[0];
}else if(entry.chlsicon) {
obj.cover = entry.chlsicon;
}else if(entry.chlicon) {
obj.cover = entry.chlicon;
}else if(jObj.card && jObj.card.icon) {
obj.cover = jObj.card.icon;
}else if(obj.img) {
obj.cover = obj.img[0];
}else {
obj.cover = '';
}
News.insert(obj, function (err, result) {
if(err) {
console.log("hzfdbg file[" + __filename + "]" + " getSubscribeNewsDetail(), News.insert():error " + err);
}
});
});//News.findOne
});//request
};
var getPhotoDetail = function(entry) {
// http://inews.qq.com/getQQNewsSimpleHtmlContent?id=PIC2013061200601000&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&sceneid=00000&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&chlid=news_photo&appver=16_android_2.7.0
var photoDetailLink = "http://inews.qq.com/getQQNewsSimpleHtmlContent?id=%s&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&sceneid=00000&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&chlid=news_photo&appver=16_android_2.7.0";
var docid = util.format("%s",entry.id);
var url = util.format(photoDetailLink, docid);
var req = {uri: url, method: "GET", headers: headers};
if(proxyEnable) {
req.proxy = proxyUrl;
}
request(req, function (err, res, body) {
var json = data2Json(err, res, body);
if((!json) || (json.ret != 0)) {
console.log("hzfdbg file[" + __filename + "]" + " getPhotoDetail():ret="+json.ret);
return;
}
var jObj = json;
var obj = entry;
News.findOne(genQqFindCmd(site, entry), function(err, result) {
if(err || result) {
return;
}
obj.docid = encodeDocID(site, docid);
obj.site = site;
obj.body = jObj.intro + jObj.content.text;
obj.img = [];
var attribute = jObj.attribute;
var img_keys = _.keys(attribute);
img_keys.forEach(function(key){
obj.img[obj.img.length] = attribute[key].url;
var imgHtml = attribute[key].desc + genLazyLoadHtml(attribute[key].desc, attribute[key].url);
obj.body = obj.body.replace("<!--" + key + "-->", imgHtml); // <!--IMG_4-->
});
obj.video = [];
obj.link = "";
if(entry.url) {
obj.link = entry.url; // http://view.inews.qq.com/a/TPC2013061100203800
}else if(entry.surl) {
obj.link = entry.surl; // http://view.inews.qq.com/a/TPC2013061100203800
}else if(jObj.url) {
obj.link = jObj.url; // http://view.inews.qq.com/a/TPC2013061100203800
}else if(jObj.surl) {
obj.link = jObj.surl; // http://view.inews.qq.com/a/TPC2013061100203800
}else {
obj.link = util.format("http://view.inews.qq.com/a/%s", docid);
}
obj.title = entry.title;
obj.ptime = entry.time;
obj.time = new Date(entry.time);
obj.marked = obj.body;
obj.created = new Date();
obj.views = 1;
obj.tags = entry.tagName;
obj.digest = genDigest(obj.body);
obj.cover = entry.thumbnails[0];
if (obj.img[0]) {
obj.cover = obj.img[0];
}
News.insert(obj, function (err, result) {
if(err) {
console.log("hzfdbg file[" + __filename + "]" + " getPhotoDetail(), News.insert():error " + err);
}
});
});//News.findOne
});//request
};
var uri = Conf.proto+'://'+Conf.host+':'+Conf.port;
var requestOptions = {
uri: uri,
method: 'PUT',
headers: {}
};
var header = Hawk.client.header(uri, 'PUT',
{
credentials: Conf.credentials,
ext: 'some-app-data'
});
requestOptions.headers.Authorization = header.field;
Request(requestOptions, function (err, res, body) {
if (err) { console.log(err); }
var isValid =
Hawk.client.authenticate(
res,
Conf.credentials,
header.artifacts,
{ payload: body });
if (Conf.debug)
console.log(res.statusCode+': '+body+
(isValid ? ' (valid)' : ' (invalid)'));
});
// return a "request" client object or an event emitter matching the same behaviour for unsupported protocols
// the callback should be called with a "request" response object or an event emitter matching the same behaviour too
function createClient(xhr) {
const flag = xhr[xhrSymbols.flag];
const properties = xhr[xhrSymbols.properties];
const urlObj = new URL(flag.uri);
const uri = urlObj.href;
const ucMethod = flag.method.toUpperCase();
const { requestManager } = flag;
if (urlObj.protocol === "file:") {
const response = new EventEmitter();
response.statusCode = 200;
response.rawHeaders = [];
response.headers = {};
response.request = { uri: urlObj };
const filePath = urlObj.pathname
.replace(/^file:\/\//, "")
.replace(/^\/([a-z]):\//i, "$1:/")
.replace(/%20/g, " ");
const client = new EventEmitter();
const readableStream = fs.createReadStream(filePath, { encoding: null });
readableStream.on("data", chunk => {
response.emit("data", chunk);
client.emit("data", chunk);
});
readableStream.on("end", () => {
response.emit("end");
client.emit("end");
});
readableStream.on("error", err => {
client.emit("error", err);
});
client.abort = function () {
readableStream.destroy();
client.emit("abort");
};
if (requestManager) {
const req = {
abort() {
properties.abortError = true;
xhr.abort();
}
};
requestManager.add(req);
const rmReq = requestManager.remove.bind(requestManager, req);
client.on("abort", rmReq);
client.on("error", rmReq);
client.on("end", rmReq);
}
process.nextTick(() => client.emit("response", response));
return client;
}
if (urlObj.protocol === "data:") {
const response = new EventEmitter();
response.request = { uri: urlObj };
const client = new EventEmitter();
let buffer;
try {
const parsed = parseDataURL(uri);
const contentType = parsed.mimeType.toString();
buffer = parsed.body;
response.statusCode = 200;
response.rawHeaders = ["Content-Type", contentType];
response.headers = { "content-type": contentType };
} catch (err) {
process.nextTick(() => client.emit("error", err));
return client;
}
client.abort = () => {
// do nothing
};
process.nextTick(() => {
client.emit("response", response);
process.nextTick(() => {
response.emit("data", buffer);
client.emit("data", buffer);
response.emit("end");
client.emit("end");
});
});
return client;
}
const requestHeaders = {};
for (const header in flag.requestHeaders) {
requestHeaders[header] = flag.requestHeaders[header];
}
if (getRequestHeader(flag.requestHeaders, "referer") === null) {
requestHeaders.Referer = flag.referrer;
}
if (getRequestHeader(flag.requestHeaders, "user-agent") === null) {
requestHeaders["User-Agent"] = flag.userAgent;
}
if (getRequestHeader(flag.requestHeaders, "accept-language") === null) {
requestHeaders["Accept-Language"] = "en";
}
if (getRequestHeader(flag.requestHeaders, "accept") === null) {
requestHeaders.Accept = "*/*";
}
const crossOrigin = flag.origin !== urlObj.origin;
if (crossOrigin) {
requestHeaders.Origin = flag.origin;
}
const options = {
uri,
method: flag.method,
headers: requestHeaders,
gzip: true,
maxRedirects: 21,
followAllRedirects: true,
encoding: null,
strictSSL: flag.strictSSL,
proxy: flag.proxy,
forever: true
};
if (flag.auth) {
options.auth = {
user: flag.auth.user || "",
pass: flag.auth.pass || "",
sendImmediately: false
};
}
if (flag.cookieJar && (!crossOrigin || flag.withCredentials)) {
options.jar = wrapCookieJarForRequest(flag.cookieJar);
}
const { body } = flag;
const hasBody = body !== undefined &&
body !== null &&
body !== "" &&
!(ucMethod === "HEAD" || ucMethod === "GET");
if (hasBody && !flag.formData) {
options.body = body;
}
if (hasBody && getRequestHeader(flag.requestHeaders, "content-type") === null) {
requestHeaders["Content-Type"] = "text/plain;charset=UTF-8";
}
function doRequest() {
try {
const client = request(options);
if (hasBody && flag.formData) {
const form = client.form();
for (const entry of body) {
form.append(entry.name, entry.value, entry.options);
}
}
return client;
} catch (e) {
const client = new EventEmitter();
process.nextTick(() => client.emit("error", e));
return client;
}
}
let client;
const nonSimpleHeaders = Object.keys(flag.requestHeaders)
.filter(header => !simpleHeaders.has(header.toLowerCase()));
if (crossOrigin && (!simpleMethods.has(ucMethod) || nonSimpleHeaders.length > 0 || properties.uploadListener)) {
client = new EventEmitter();
const preflightRequestHeaders = [];
for (const header in requestHeaders) {
// the only existing request headers the cors spec allows on the preflight request are Origin and Referrer
const lcHeader = header.toLowerCase();
if (lcHeader === "origin" || lcHeader === "referrer") {
preflightRequestHeaders[header] = requestHeaders[header];
}
}
preflightRequestHeaders["Access-Control-Request-Method"] = flag.method;
if (nonSimpleHeaders.length > 0) {
preflightRequestHeaders["Access-Control-Request-Headers"] = nonSimpleHeaders.join(", ");
}
preflightRequestHeaders["User-Agent"] = flag.userAgent;
flag.preflight = true;
const preflightOptions = {
uri,
method: "OPTIONS",
headers: preflightRequestHeaders,
followRedirect: false,
encoding: null,
pool: flag.pool,
strictSSL: flag.strictSSL,
proxy: flag.proxy,
forever: true
};
const preflightClient = request(preflightOptions);
preflightClient.on("response", resp => {
// don't send the real request if the preflight request returned an error
if (resp.statusCode < 200 || resp.statusCode > 299) {
client.emit("error", new Error("Response for preflight has invalid HTTP status code " + resp.statusCode));
return;
}
// don't send the real request if we aren't allowed to use the headers
if (!validCORSPreflightHeaders(xhr, resp, flag, properties)) {
setResponseToNetworkError(xhr);
return;
}
const realClient = doRequest();
realClient.on("response", res => {
for (const header in resp.headers) {
if (preflightHeaders.has(header)) {
res.headers[header] = Object.prototype.hasOwnProperty.call(res.headers, header) ?
mergeHeaders(res.headers[header], resp.headers[header]) :
resp.headers[header];
}
}
client.emit("response", res);
});
realClient.on("data", chunk => client.emit("data", chunk));
realClient.on("end", () => client.emit("end"));
realClient.on("abort", () => client.emit("abort"));
realClient.on("request", req => {
client.headers = realClient.headers;
client.emit("request", req);
});
realClient.on("redirect", () => {
client.response = realClient.response;
client.emit("redirect");
});
realClient.on("error", err => client.emit("error", err));
client.abort = () => {
realClient.abort();
};
});
preflightClient.on("error", err => client.emit("error", err));
client.abort = () => {
preflightClient.abort();
};
} else {
client = doRequest();
}
if (requestManager) {
const req = {
abort() {
properties.abortError = true;
xhr.abort();
}
};
requestManager.add(req);
const rmReq = requestManager.remove.bind(requestManager, req);
client.on("abort", rmReq);
client.on("error", rmReq);
client.on("end", rmReq);
}
return client;
}
ImdbParser.prototype.findById = function(id) {
if (id == undefined) {
console.error('id фильма не передан');
}
var url = this.urlPrefix + id;
request(url, function (error, response, html) {
if (!error) {
if (response.statusCode == 200) {
var $ = cheerio.load(html);
var title, release, rating, description;
var json = {};
$('.summary_text').filter(function () {
var data = $(this);
description = data.text().trim();
json.description = description;
});
$('.title_wrapper').filter(function () {
var data = $(this);
title = data.children().first().text().trim();
release = data.children().last().children().last().text().trim();
json.title = title;
json.release = release;
});
$('.ratingValue').filter(function () {
var data = $(this);
rating = data.text().trim().split(' ');
json.rating = rating[0].split('/');
});
json.id = function () {
var crypto = require('crypto');
var name = json.title;
return crypto.createHash('md5').update(name).digest('hex');
}();
if (json.title != '' && json.rating != '') {
var instanceOfMovie = new Movies();
userDataObject.findOne({name: argv['user']}, function (err, result) {
if (!err) {
if (!result) {
throw Error('User not found');
} else {
instanceOfMovie.setDescription(json.description);
if (json.rating != undefined) {
instanceOfMovie.setRatingImdb(json.rating);
} else {
instanceOfMovie.setRatingImdb(0);
}
instanceOfMovie.setReleaseDate(json.release);
instanceOfMovie.setTitle(json.title);
instanceOfMovie.setCurrentDate();
instanceOfMovie.setImdbId(id);
instanceOfMovie.setUserId(result._id);
var movieToDb = new movieDataObject(instanceOfMovie);
//jsonfile.writeFileSync(file, movieToDb);
fs.appendFile(file, JSON.stringify(movieToDb),function(err){
if(err)
console.error(err);
console.log('Appended!');
});
movieToDb.save(function (err) {
if (err) {
return handleError(err);
} else {
console.log(clc.blue('Фильм успешно сохранен в БД'));
}
});
}
} else {
console.log(clc.redBright('Ошибка получения информации о фильме'));
}
}
);
}
}
}
});
};
function() {
request('http://localhost:8001/', function(error) {
expect(error).not.toBe(null);
done();
});
}
CameioTask.prototype.fetchCodepen = function() {
var self = this;
var codepenUrl = this.template.split('?')[0].split('#')[0];
var wwwPath = path.join(this.targetPath, 'www');
console.log('\nDownloading Codepen:'.bold, codepenUrl);
var qHTML = Q.defer();
var qCSS = Q.defer();
var qJS = Q.defer();
var proxy = process.env.PROXY || null;
request({ url: codepenUrl + '.html', proxy: proxy }, function(err, res, html) {
if(!err && res && res.statusCode === 200) {
html = html || '';
if(html.indexOf('<!DOCTYPE html>') < 0) {
html = '<!DOCTYPE html>\n' + html;
}
var resources = ' <link href="css/style.css" rel="stylesheet">\n' +
' <script src="js/app.js"></script>\n';
if(self.isCordovaProject) {
resources += ' <script src="cordova.js"></script>\n';
}
resources += ' </head>';
html = html.replace(/<\/head>/i, '\n' + resources);
html = self.convertTemplates(html);
fs.writeFileSync(path.join(wwwPath, 'index.html'), html, 'utf8');
}
qHTML.resolve();
});
request({ url: codepenUrl + '.css', proxy: proxy }, function(err, res, css) {
if(!err && res && res.statusCode === 200) {
css = css || '';
var cssPath = path.join(wwwPath, 'css');
if(!fs.existsSync(cssPath)) {
fs.mkdirSync(cssPath);
}
css = css.replace("cursor: url('http://cameioframework.com/img/finger.png'), auto;", '');
fs.writeFileSync(path.join(cssPath, 'style.css'), css, 'utf8');
}
qCSS.resolve();
});
request({ url: codepenUrl + '.js', proxy: proxy }, function(err, res, js) {
if(!err && res && res.statusCode === 200) {
js = js || '';
var jsPath = path.join(wwwPath, 'js');
if(!fs.existsSync(jsPath)) {
fs.mkdirSync(jsPath);
}
fs.writeFileSync(path.join(jsPath, 'app.js'), js, 'utf8');
}
qJS.resolve();
});
return Q.all([qHTML.promise, qCSS.promise, qJS.promise]);
};
setInterval(function() {
var url = process.env.deployUrl;
var resolved = http_url.resolve(url, "/pulse");
http_request(resolved, function (error, response, data) { });
}, 1000);
var getNewsDetail = function(entry) {
var detailLink = 'http://inews.qq.com/getQQNewsNormalHtmlContent?id=%s';
var docid = util.format("%s",entry.id);
var url = util.format(detailLink, docid);
var req = {uri: url, method: "GET", headers: headers};
if(proxyEnable) {
req.proxy = proxyUrl;
}
request(req, function (err, res, body) {
var json = data2Json(err, res, body);
if((!json) || (json.ret != 0)) {
console.log("hzfdbg file[" + __filename + "]" + " getNewsDetail():ret="+json.ret);
return;
}
var jObj = json;
var obj = entry;
News.findOne(genQqFindCmd(site, entry), function(err, result) {
if(err || result) {
return;
}
obj.docid = encodeDocID(site, docid);
obj.site = site;
obj.body = '';
obj.img = [];
jObj.content.forEach(function(item) {
if(item.type == 1) { //text
obj.body += item.value;
}
else if(item.type == 2) { //pic
obj.body += genLazyLoadHtml(entry.tagName, item.value);
obj.img[obj.img.length] = item.value;
}
else if(item.type == 3) { //video
obj.body += genLazyLoadHtml(entry.tagName, item.value.img);
obj.img[obj.img.length] = item.value.img;
}
});
obj.video = [];
obj.link = "";
if(entry.url) {
obj.link = entry.url; // http://view.inews.qq.com/a/NEW2013050300143202
}else if(entry.surl) {
obj.link = entry.surl; // http://view.inews.qq.com/a/NEW2013050300143202
}else if(jObj.url) {
obj.link = jObj.url; // http://view.inews.qq.com/a/NEW2013050300143202
}else if(jObj.surl) {
obj.link = jObj.surl; // http://view.inews.qq.com/a/NEW2013050300143202
}else {
obj.link = util.format("http://view.inews.qq.com/a/%s", docid);
}
obj.title = entry.title;
obj.ptime = entry.time;
obj.time = new Date(entry.time);
obj.marked = obj.body;
obj.created = new Date();
obj.views = 1;
obj.tags = entry.tagName;
obj.digest = genDigest(obj.body);
if(!obj.digest) {
obj.digest = obj.body;
}
obj.cover = entry.thumbnails[0];
if (obj.img[0]) {
obj.cover = obj.img[0];
}
News.insert(obj, function (err, result) {
if(err) {
console.log("hzfdbg file[" + __filename + "]" + " getNewsDetail(), News.insert():error " + err);
}
});
});//News.findOne
});//request
};
// Dependencies:
var request = require('request'); // Snatches html from urls
var cheerio = require('cheerio'); // Scrapes our html
// Make a request call to grab the html body from the site of your choice
// Notice: the page's html gets saved as the callback's third arg
request('https://www.imgur.com', function (error, response, html) {
// Load the html into cheerio and save it to a var.
// '$' becomes a shorthand for cheerio's selector commands,
// much like jQuery's '$'.
var $ = cheerio.load(html);
// an empty array to save the data that we'll scrape
var result = [];
// Select each instance of the html body that you want to scrape.
// NOTE: Cheerio selectors function similarly to jQuery's selectors,
// but be sure to visit the package's npm page to see how it works.
$('.hover p').each(function(i, element){
// Scrape information from the web page, put it in an object
// and add it to the result array.
var text = $(this).text();
result.push(text);
});
console.log(result);
});
var crawlerSubscribe = function(entry) {
// 所有栏目列表 http://r.inews.qq.com/getCatList?qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&appver=16_android_3.2.1&version=0
// 获取某一栏目文章ids http://r.inews.qq.com/getSubNewsIndex?qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&format=json&apptype=android&chlid=1250&appver=16_android_3.2.1
// 获取指定ids文章 http://r.inews.qq.com/getSubNewsListItems?uid=22c4d53a-7d52-4f50-9185-90a77c7b80b0&qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&ids=20131107A000D700%2C20131106A000DE00%2C20131106A000DD00%2C20131105A000DS00%2C20131105A000DR00%2C20131104A000CW00%2C20131101A000DZ00%2C20131031A000BZ00%2C20131030A000BV00%2C20131029A000C600%2C20131028A0017800%2C20131025A0019S00%2C20131024A000CG00%2C20131023A000CM00%2C20131022A000I400%2C20131022A000I300%2C20131022A0007700%2C20131018A000EB00%2C20131017A0015800%2C20131016A000AX00&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&appver=16_android_3.2.1
// 获取某篇文章内容 http://r.inews.qq.com/getSubNewsContent?id=20131129A000H600&qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&appver=16_android_3.2.1
var subscribeNewsIdsLink = 'http://r.inews.qq.com/getSubNewsIndex?qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&format=json&apptype=android&chlid=%s&appver=16_android_3.2.1';
var subscribeNewsListLink = 'http://r.inews.qq.com/getSubNewsListItems?uid=22c4d53a-7d52-4f50-9185-90a77c7b80b0&qqnetwork=wifi&store=118&hw=Xiaomi_MI2&devid=1366805394774330052&ids=%s&mac=c4%253A6a%253Ab7%253Ade%253A4d%253A24&apptype=android&appver=16_android_3.2.1'
var url = '';
if(entry.ids) {
var ids = '';
var i = 0;
for(i=0; (i<20) && (i<entry.ids.length); i++) {
if(i > 0) {
ids = ids + ','
}
ids = ids + entry.ids[i].id;
}
url = util.format(subscribeNewsListLink, ids);
entry.ids = entry.ids.slice(i);
}else {
url = util.format(subscribeNewsIdsLink, entry.id);
}
var req = {uri: url, method: "GET", headers: headers};
if(proxyEnable) {
req.proxy = proxyUrl;
}
request(req, function (err, res, body) {
var json = data2Json(err, res, body);
if(!json || (json.ret != 0)) {
console.log("hzfdbg file[" + __filename + "]" + " crawlerSubscribe():JSON.parse() error");
return;
}
if(entry.ids) {
setTimeout(function() {
crawlerSubscribe(entry);
}, 100);
}else if(json.ids) {
entry.ids = json.ids;
setTimeout(function() {
crawlerSubscribe(entry);
}, 100);
return;
}
var newsList = json.newslist;
if((!newsList) || (!newsList.length) || (newsList.length <= 0)) {
console.log("hzfdbg file[" + __filename + "]" + " crawlerSubscribe():newsList empty in url " + url);
return;
}
newsList.forEach(function(newsEntry) {
if(!newsEntry.title || !newsEntry.id) {
return;
}
var tags = entry.tags;
for(var i = 0; i < tags.length; i++) {
if (newsEntry.title.indexOf(tags[i]) !== -1 || entry.all) {
newsEntry.tagName = tags[i];
newsEntry.subscribeName = entry.name;
News.findOne(genQqFindCmd(site,newsEntry), function(err, result) {
if(err || result) {
return;
}
console.log("hzfdbg file[" + __filename + "]" + " crawlerSubscribe():["+newsEntry.tagName+"]"+newsEntry.title+",docid="+newsEntry.id);
startGetDetail.emit('startGetSubscribeNewsDetail', newsEntry);
}); // News.findOne
break;
}
}//for
});//forEach
});//request
}
it('should update the status in the Context Broker', function(done) {
request(commandOptions, function(error, response, body) {
contextBrokerMock.done();
done();
});
});
console.log(ele);
});
console.log("\n");
};
var data = JSON.parse(originalData);
processData(data);
// Process Data From Requested JSON
var url1 = "http://foureyes.github.io/csci-ua.0480-fall2014-002/homework/02/2014-06-15-heat-spurs.json";
var url2 = "http://foureyes.github.io/csci-ua.0480-fall2014-002/homework/02/2014-04-09-thunder-clippers.json";
request(url1,function(err, response, body){
if(err){
return console.error('requst failed:', err);
}
if (!err && response.statusCode == 200) {
processData(JSON.parse(body));
}
});
request(url2,function(err, response, body){
if(err){
return console.error('requst failed:', err);
}
if (!err && response.statusCode == 200) {
processData(JSON.parse(body));
}
});
function unpack() {
var stream;
var size = {
file: 0,
gunzip: 0,
xml: 0
};
var todo = [];
function chunked(chunk) {
size.file += chunk.length;
if (size.file > maxsize.file) {
var err = new RangeError('Upload size should not exceed ' + numeral(maxsize.file).format('0b') + '.');
stream.emit('error', err);
}
}
gunzip.on('data', function(chunk) {
size.gunzip += chunk.length;
if (size.gunzip > maxsize.gunzip) {
var err = new RangeError('Unzipped size should not exceed ' + numeral(maxsize.gunzip).format('0b') + '.');
gunzip.emit('error', err);
}
});
parser.on('entry', function(entry) {
var parts = [];
var filepath = entry.props.path.split('/').slice(1).join('/');
entry.on('data', function(chunk) {
if (path.basename(filepath).toLowerCase() == 'project.xml') {
size.xml += chunk.length;
if (size.xml > maxsize.xml) {
var err = new RangeError('Unzipped project.xml size should not exceed ' + numeral(maxsize.xml).format('0b') + '.');
parser.emit('error', err);
}
}
parts.push(chunk);
});
entry.on('end', function() {
var buffer = Buffer.concat(parts);
if (path.basename(filepath).toLowerCase() == 'project.xml') {
xml = buffer.toString();
if (unpacked) return load();
} else if (!unpacked && entry.type === 'Directory') {
todo.push(function(next) { fs.mkdir(base + '/' + filepath, next); });
} else if (!unpacked && entry.type === 'File') {
todo.push(function(next) { fs.writeFile(base + '/' + filepath, buffer, next); });
}
});
});
parser.on('end', function() {
// Load was called early via parser. Do nothing.
if (unpacked && xml) return;
// Package unpacked but no project.xml. Call load to error our.
if (unpacked) return load();
todo.push(function(next) { fs.writeFile(base + '/.unpacked', '', next); });
var next = function(err) {
if (err && err.code !== 'EEXIST') return callback(err);
if (todo.length) {
todo.shift()(next);
} else {
unpacked = true;
load();
}
};
next();
});
gunzip.on('error', error);
parser.on('error', error);
switch(uri.protocol) {
case 'tm2z:':
// The uri from unpacker has already been pulled
// down from S3.
stream = fs.createReadStream(uri.pathname)
.on('data', chunked)
.pipe(gunzip)
.pipe(parser)
.on('error', error);
break;
case 'tm2z+http:':
uri.protocol = 'http:';
stream = request({ uri: uri })
.on('data', chunked)
.pipe(gunzip)
.pipe(parser)
.on('error', error);
break;
}
};
self.request = function(opts) {
// console.log("OPTS",opts);
if (useCache(opts)) {
var cacheData = self.cache[opts.uri];
//If a query has already been made to self URL, don't callback again
if (cacheData) {
// Make sure we actually have cached data, and not just a note
// that the page was already crawled
if (_.isArray(cacheData)) {
self.onContent(null,opts,cacheData[0],true);
} else {
release(opts);
}
return;
}
}
if (opts.debug) {
console.log(opts.method+" "+opts.uri+" ...");
}
// Cloning keeps the opts parameter clean:
// - some versions of "request" apply the second parameter as a
// property called "callback" to the first parameter
// - keeps the query object fresh in case of a retry
// Doing parse/stringify instead of _.clone will do a deep clone and remove functions
var ropts = JSON.parse(JSON.stringify(opts));
if (!ropts.headers) ropts.headers={};
if (ropts.forceUTF8) {
if (!ropts.headers["Accept-Charset"] && !ropts.headers["accept-charset"]) ropts.headers["Accept-Charset"] = 'utf-8;q=0.7,*;q=0.3';
if (!ropts.encoding) ropts.encoding=null;
}
if (typeof ropts.encoding === 'undefined') {
ropts.headers["Accept-Encoding"] = "gzip";
ropts.encoding = null;
}
if (ropts.userAgent) {
ropts.headers["User-Agent"] = ropts.userAgent;
}
if (ropts.proxies && ropts.proxies.length) {
ropts.proxy = ropts.proxies[0];
}
var requestArgs = ["uri","url","qs","method","headers","body","form","json","multipart","followRedirect","followAllRedirects",
"maxRedirects","encoding","pool","timeout","proxy","auth","oauth","strictSSL","jar","aws"];
var req = request(_.pick.apply(this,[ropts].concat(requestArgs)), function(error,response,body) {
if (error) return self.onContent(error, opts);
response.uri = opts.uri;
// Won't be needed after https://github.com/mikeal/request/pull/303 is merged
if (response.headers['content-encoding'] && response.headers['content-encoding'].toLowerCase().indexOf('gzip') >= 0) {
zlib.gunzip(response.body, function (error, body) {
if (error) return self.onContent(error, opts);
if (!opts.forceUTF8) {
response.body = body.toString(req.encoding);
} else {
response.body = body;
}
self.onContent(error,opts,response,false);
});
} else {
self.onContent(error,opts,response,false);
}
});
};
var FeedParser = require('feedparser')
, request = require('request');
var rssUrl = 'http://rss.rtbf.be/article/rss/highlight_rtbfinfo_info-accueil.xml';
request(rssUrl)
.pipe(new FeedParser([]))
.on('error', function(error) {
// always handle errors
})
.on('meta', function (meta) {
// do something
})
.on('readable', function () {
// do something else, then do the next thing
var stream = this, item;
while (item = stream.read()) {
console.log('Got article: %s', item.title);
console.log(' Description: %s', item.description);
}
})
router.get('/search', function (req, res, next) {
// build the url for the search query
var url = 'https://www.dr.dk/mu/search/programcard?FreeText="'+req.query.query+'"';
request(url, function (error, response, body) {
if (!error && response.statusCode == 200) {
var json = JSON.parse(body);
// filter for entries that has the 'Assets'-key wherein there is at least one VideoResource.
var filtered = _.filter(json.Data, function(result){
if (result.Assets &&
result.Assets.length &&
typeof _.findWhere(result.Assets, { 'Kind': 'VideoResource' } !== 'undefined')
) {
return result;
}
});
// Now we need to build our own data objects with the data that we're
// interested in. Also we need to fetch the video uris for the videoresource
// assets. They're located at another address, so we need to fetch them.
var buildDataObjects = function () {
var fulfilled;
var deferreds = [];
_.forEach(filtered, function(result) {
_.forEach(result.Assets, function(asset) {
if (asset.Kind !== 'VideoResource') return;
var deferred = Q.defer();
// if there's an image for this entry, we should get the uri for it
var imgUrl = _.findWhere(result.Assets, { 'Kind' : 'Image' });
if (imgUrl) {
imgUrl = imgUrl.Uri;
}
request({
url : asset.Uri
}, function(error, response, body) {
var json = JSON.parse(body);
// we need to get the link where target says HLS
var hdsObj = _.findWhere(json.Links, { 'Target' : 'HLS' });
// and this is where we build the object with the data we're interested in
deferred.resolve({
CreatedTime: result.CreatedTime,
Title: result.Title,
Description: result.Description,
ImageUrl : imgUrl,
Slug: result.Slug,
CastUrl: hdsObj.Uri
});
});
deferreds.push(deferred.promise);
});
});
return Q.all(deferreds);
}
// now all we need to do is wait for all our fetching
Q.fcall( buildDataObjects ).then(function(data) {
// so let's sort for created time by descending order
var data = _.sortByOrder(data, 'CreatedTime', false);
// and make sure that we only have unique entries
data = _.uniq(data, 'Title');
// and present them for our caller
res.json(data);
}).catch(function (error) {
console.log(error);
}).done();
}
});
});
setInterval(function() {
LOG.info("BuildWatcher", "Sending request to ", url);
request(url, function (error, response, body) {
if (error || response.statusCode != 200) {
LOG.error("BuildWatcher", "Could not load Jenkins data");
return;
}
var result = JSON.parse(body);
if(!result.jobs) {
LOG.error("BuildWatcher", "No jobs were present in JSON response");
LOG.error(body);
return;
}
var currentStatus;
var jobStatus = [];
result.jobs.forEach(function(job) {
if(job.color == "red") {
jobStatus.push(STATUSES.FAILING);
} else if(job.color == "yellow") {
jobStatus.push(STATUSES.UNSTABLE);
} else if(job.color.substr(job.color.length - "_anime".length) == "_anime") {
jobStatus.push(STATUSES.BUILDING);
} else if(job.color == "blue") {
jobStatus.push(STATUSES.PASSING);
}
});
if(jobStatus.indexOf(STATUSES.FAILING) != -1) {
LOG.info("BuildWatcher", "Builds are failing");
currentStatus = STATUSES.FAILING;
} else if(jobStatus.indexOf(STATUSES.UNSTABLE) != -1) {
LOG.info("BuildWatcher", "Builds are unstable");
currentStatus = STATUSES.UNSTABLE;
} else if(jobStatus.indexOf(STATUSES.BUILDING) != -1) {
LOG.info("BuildWatcher", "Builds are building");
currentStatus = STATUSES.BUILDING;
} else if(jobStatus.indexOf(STATUSES.PASSING) != -1) {
LOG.info("BuildWatcher", "Builds are passing");
currentStatus = STATUSES.PASSING;
}
if(lastStatus != currentStatus) {
lastStatus = currentStatus;
if(currentStatus == STATUSES.FAILING) {
this.emit("failed");
} else if(currentStatus == STATUSES.BUILDING) {
this.emit("building");
} else if(currentStatus == STATUSES.PASSING) {
this.emit("passed");
} else if(currentStatus == STATUSES.UNSTABLE) {
this.emit("unstable");
}
}
}.bind(this));
}.bind(this), checkInterval);
function tsla_poll( vid, long_vid, token ) {
if (long_vid == undefined || token == undefined) {
console.log('Error: undefined vehicle_id (' + long_vid +') or token (' + token +')' );
console.log('Exiting...');
process.exit(1);
} else {
request(
{
'uri': s_url + long_vid +'/?values=' + argv.values,
'method' : 'GET',
'auth': {
'user': creds.username,
'pass': token
},
'timeout' : 125000 // a bit more than the expected 2 minute max long poll
},
function( error, response, body) {
if ( error ) { // HTTP Error
if (!argv.silent) { util.log( error ); }
// put short delay to avoid stack overflow
setTimeout(function() {
tsla_poll( vid, long_vid, token ); // poll again
}, 1000);
} else if (response.statusCode == 200) { // HTTP OK
if (!argv.silent)
{
if (body===undefined)
{
util.log('undefined');
}
else if (body===null)
{
util.log('null');
}
else
{
util.log(body);
}
}
tsla_poll( vid, long_vid, token ); // poll again
} else if ( response.statusCode == 401) { // HTTP AUTH Failed
if (!argv.silent) {
util.log('WARN: HTTP 401: Unauthorized - token has likely expired, getting a new one');
}
initstream();
} else {
if (!argv.silent) {
util.log('Problem with request:');
util.log(' Response status code = ' + response.statusCode );
util.log(' Error code = ' + error);
util.log('Polling again...');
}
// put short delay to avoid stack overflow
setTimeout(function() {
tsla_poll( vid, long_vid, token ); // poll again
}, 1000);
}
}
).on('data', function(data) {
var d, vals, i, record, doc;
if (argv.db) {
d = data.toString().trim();
vals = d.split(/[,\n\r]/);
for (i = 0; i < vals.length; i += nFields) {
record = vals.slice(i, nFields);
doc = { 'ts': +vals[i], 'record': record };
collectionS.insert(doc, { 'safe': true }, function(err,docs) {
if(err) util.log(err);
});
// collectionS.find({ 'ts': +vals[i]}).toArray(function(err, exist){
// try {
// if (err || exist == null || exist.length == 0) { // only write entry if it doesn't already exist
// collectionS.insert(doc, { 'safe': true }, function(err,docs) {
// if(err) util.log(err);
// });
// } else {
// util.log("had data, not writing it");
// }
// } catch (innerError) {
// console.dir(innerError);
// }
// });
}
} else {
stream.write(data);
}
});
}
}
function getGamesFromList() {
request( 'http://api.thescore.com/' + sport + '/events?id.in=' + gameList[ sport ].join('%2C'), gotGames );
}
var cheerio = require('cheerio');
var request = require('request');
var filenameArray = [];
request("http://substack.net/images/", function(err, response, body) {
if (!err && response.statusCode == 200) {
$ = cheerio.load(body);
$("td").each(function () {
var code = $("code",$(this));
var a = $("a",$(this));
var ahref = a.attr('href');
process.stdout.write(code.html() + ",");
process.stdout.write(a.html() + ",");
filenameArray = (ahref.split('.'));
process.stdout.write(filenameArray[1] + "\n");
});
}
});
watchProcStore.getWatchProcessorMap(function(procMap) {
logger.debug("Selecting next available processors from ", procMap);
processorObj = nextLeastLoadedProcessor(procMap);
// processorObj = nextRandomProcessor(procMap);
logger.debug("Next available processor is ", processorObj);
//@TODO make HTTP request to Processor
var options = {
method: 'POST',
url: 'http://' + processorObj.url + '/watchtaskprocessor/watchtask',
json: watchTask
};
return request(options, function(err, res, body) {
if (err) {
logger.error("Error in task allocation of task: ", watchTask.name, " to processor: ", processorObj.url, " error: ", err);
callback(null, {
processor: processorObj.url,
watchtask: watchTask,
result: false,
status: undefined,
error: err,
body: body
});
return false;
} else {
if (res === undefined || res.statusCode === undefined) {
var myError = new Error('My custom error!');
logger.error("Error in task allocation of task: ", watchTask.name, " to processor: ", processorObj.url, " returned with out any status ");
callback(null, {
processor: processorObj.url,
watchtask: watchTask,
result: false,
status: undefined,
error: err,
body: body
});
return false;
} else if (res.statusCode >= 200 && res.statusCode <= 299) {
logger.info("Successfully allocated task: ", watchTask.name, " to processor: ", processorObj.url, " response ", body);
var watchTaskObj = {
"name": watchTask.name,
"type": watchTask.type,
"subFrom": watchTask.subFrom,
"pubTo": watchTask.pubTo,
"watchName": watchTask.watchName,
"orgsite": watchTask.orgsite
}
watchProcStore.registerWatchTask(processorObj.url, watchTaskObj);
callback(null, {
processor: processorObj.url,
watchtask: watchTask,
result: true,
status: res.statusCode,
error: err,
body: body
});
return true;
}
}
});
});
it('keeps status code', function(done) {
request('http://localhost:8000/devices', function(error, res, body) {
expect(res.statusCode).toEqual('201');
done();
});
});