test('should respond to TRACE with parsed params received', async () => {
let response = await request({ method: 'trace', url: url + '/api/x', form: { key: 'someKey', value: 'someValue' }, resolveWithFullResponse: true })
expect(response.statusCode).toEqual(200)
let body = await toJson(response.body)
expect(body.receivedParams.key).toEqual('someKey')
expect(body.receivedParams.value).toEqual('someValue')
})
.then(party => {
if (req.body.image) {
const filename = `static/images/parties/${party.id}.jpg`;
request(req.body.image).pipe(fs.createWriteStream(filename));
}
res.end();
})
return Promise.resolve().then(() => {
const parsedURL = new URL(url);
url = parsedURL.href;
options = normalizeFromURLOptions(options);
const requestOptions = {
resolveWithFullResponse: true,
encoding: null, // i.e., give me the raw Buffer
gzip: true,
headers: {
"User-Agent": options.userAgent,
Referer: options.referrer,
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en"
},
jar: wrapCookieJarForRequest(options.cookieJar)
};
return request(url, requestOptions).then(res => {
const parsedContentType = parseContentType(res.headers["content-type"]);
const transportLayerEncodingLabel = parsedContentType && parsedContentType.get("charset");
options = Object.assign(options, {
url: res.request.href + parsedURL.hash,
contentType: res.headers["content-type"],
referrer: res.request.getHeader("referer"),
[transportLayerEncodingLabelHiddenOption]: transportLayerEncodingLabel
});
return new JSDOM(res.body, options);
});
});
// Try to make a request with the proxy
async function checkProxy(proxy, config) {
const options = {
uri: config.googleAdress,
timeout: config.proxyRequestTimeout,
proxy: proxy.getUrl(),
resolveWithFullResponse: true
};
try {
const response = await request(options);
if (response.statusCode !== VALID_HTTP_STATUS) {
log.debug(`Check Proxy - Invalid status code for request through the proxy : ${ proxy.getUrl() } : ${ response.statusCode }`);
proxy.valid = false;
return;
}
log.debug(`Check Proxy - valid proxy request to Google : ${ proxy.getUrl() }`);
proxy.valid = true;
} catch (e) {
log.debug(`Check Proxy - Error during request through the proxy : ${ proxy.getUrl() } : ${ e }`);
proxy.valid = false;
}
}
/**
* Query the Propsd endpoint
*
* @return {Promise<Object>}
* @private
*/
_query() {
return rp({
uri: this.path,
json: true
}).then((data) => {
assert(typeof data === 'object');
// Keys are the result of filtering the Propsd property set by the prefix, then removing the prefix from the
// key leaving an object of `identity: secret`.
return data;
}).then((data) => {
return Object.keys(data)
.filter((key) => key.startsWith(this.prefix))
.reduce((obj, key) => {
let resultKey = key;
if (this.prefix) {
// Only replace the prefix and delimiter if they exist
resultKey = key.replace(this.prefix, '');
}
obj[resultKey] = data[key];
return obj;
}, {});
}).catch((err) => {
this.emit('error', err);
});
}
exports.getPageData = async options => {
const pageContent = (await request({
method: "GET",
uri: options.url,
resolveWithFullResponse: true,
encoding: null,
headers: {
"User-Agent": options.userAgent
}
})).body.toString();
const jsdomOptions = {
url: options.url,
virtualConsole: new VirtualConsole(),
userAgent: options.userAgent,
pretendToBeVisual: true,
runScripts: "outside-only",
resources: "usable"
};
if (options.browserWidth && options.browserHeight) {
jsdomOptions.beforeParse = function (window) {
window.outerWidth = window.innerWidth = options.browserWidth;
window.outerHeight = window.innerHeight = options.browserHeight;
};
}
let dom;
try {
dom = new JSDOM(pageContent, jsdomOptions);
options.insertSingleFileComment = true;
options.insertFaviconLink = true;
const win = dom.window;
const doc = win.document;
let scripts = (await Promise.all(SCRIPTS.concat(options.browserScripts).map(scriptPath => fs.readFileSync(require.resolve(scriptPath)).toString()))).join("\n");
const fileContents = {
"/lib/hooks/content/content-hooks-web.js": fs.readFileSync(require.resolve("../../lib/hooks/content/content-hooks-web.js")).toString(),
"/lib/hooks/content/content-hooks-frames-web.js": fs.readFileSync(require.resolve("../../lib/hooks/content/content-hooks-frames-web.js")).toString(),
};
scripts = scripts + ";this.singlefile.lib.getFileContent = filename => (" + JSON.stringify(fileContents) + ")[filename];";
dom.window.eval(scripts);
if (dom.window.document.readyState == "loading") {
await new Promise(resolve => win.document.onload = resolve);
}
win.Element.prototype.getBoundingClientRect = undefined;
executeFrameScripts(doc, scripts);
if (!options.saveRawPage && !options.removeFrames) {
options.frames = await win.singlefile.lib.frameTree.content.frames.getAsync(options);
}
options.win = win;
options.doc = doc;
options.removeHiddenElements = false;
const SingleFile = getSingleFileClass(win);
const singleFile = new SingleFile(options);
await singleFile.run();
return singleFile.getPageData();
} finally {
if (dom && dom.window) {
dom.window.close();
}
}
};
Promise.resolve().then(function() {
if (req.param('provider') === 'facebook') { // Verify access token
return request({
url: `https://graph.facebook.com/${req.body.identifier}?access_token=${req.body.accessToken}`
});
}
return Promise.resolve();
}).then(function() {
test('should respond to OPTIONS with only HTTP headers', async () => {
let response = await request({ method: 'options', url: url + '/api/cacheTest', resolveWithFullResponse: true })
expect(response.statusCode).toEqual(200)
expect(response.headers['access-control-allow-methods']).toEqual('HEAD, GET, POST, PUT, PATCH, DELETE, OPTIONS, TRACE')
expect(response.headers['access-control-allow-origin']).toEqual('*')
expect(response.headers['content-length']).toEqual('0')
expect(response.body).toEqual('')
})
async function action(type,value,format,meta) {
if (type === 'Str') return rp({
uri: value,
json: true
}).then(function (data) {
return Str(data.places[0]["post code"]);
})
}
async function getQuestionUrl(courseInstanceUrl) {
const questionsUrl = courseInstanceUrl + '/instructor/questions';
const body = await request({uri: questionsUrl, jar: cookies});
const $ = cheerio.load(body);
const elemList = $(`td a:contains("${questionTitle}")`);
assert(elemList.length == 1);
return serverUrl + elemList[0].attribs.href;
}
const downloadThumb = async photo => {
const url = `flickr-imgs/${photo.id}.${photo.url_n.split('.').slice(-1)[0]}`;
console.log(`Downloading thumb for ${photo.title}`);
const resp = await rp({url: photo.url_n, encoding: null}); // get photo as Buffer
pages[url] = {contents: resp};
photo.thumb = '/' + url;
return photo;
};
module.exports.postModel = (url, token, trainingRun) =>
request({
method: 'POST',
json: true,
url: `${url}/v3/models`,
auth: { bearer: token },
body: trainingRun
})
module.exports.postTrainingDefinition = (url, token, trainingDefinition) =>
request({
method: 'POST',
json: true,
url: `${url}/v3/ml_assets/training_definitions`,
auth: { bearer: token },
body: trainingDefinition
})
Sonos.prototype.deviceDescription = async function () {
debug('Sonos.deviceDescription()')
return request('http://' + this.host + ':' + this.port + '/xml/device_description.xml')
.then(Helpers.ParseXml)
.then(result => {
return result.root.device
})
}
await Utils.parallel(range(concurrency), async () => {
let response = await rp(url(_url), { resolveWithFullResponse: true })
// Status Code
let code = response.statusCode
if (!statusCodes[code]) {
statusCodes[code] = 0
}
statusCodes[code]++
})
module.exports.authenticate = (url, username, password) =>
request({
method: 'GET',
json: true,
url: `${url}/v3/identity/token`,
auth: { user: username, pass: password }
}).then(body => {
return body.token
})
return new Promise((resolve, reject) => {
const url = 'http://api.mathjs.org/v1/?expr=' + encodeURIComponent(expr)
request(url).then(num => {
resolve(noExponents(num))
}).catch(() => {
reject(new Error(speech.calc.error))
})
})
test.serial('/error status code', async t => {
const errorSpy = await interceptError()
const err = await t.throws(rp(url('/error')))
t.true(err.statusCode === 500)
t.true(err.response.body.includes('An error occurred in the application and your page could not be served'))
release()
t.true(errorSpy.calledOnce)
t.true(errorSpy.args[0][0].message.includes('Error mouahahah'))
})
test.serial('/error-midd', async t => {
const errorSpy = await interceptError()
const err = await t.throws(rp(url('/error-midd')))
t.is(err.statusCode, 505)
t.true(err.response.body.includes('Middleware Error'))
release()
// Don't display error since redirect returns a noopApp
t.true(errorSpy.notCalled)
})
let saveUrlToFile = async (path, urlToLoad) => {
try {
let fileName = null;
if (path) {
fileName = path + '/' + urlToLoad.replace(/^.+\/([^/]+)$/, '$1');
let exists = await fileExists(fileName);
if (exists) {
console.log(`Download "${urlToLoad}" to "${fileName}" skipped - file exists`);
return fileName;
}
console.log(`Download "${urlToLoad}" to "${fileName}"...`);
}
else {
console.log(`Download "${urlToLoad}"...`);
}
let host = urlToLoad.match(/\/\/([^/]+)\//);
if (host) {
host = host[1];
}
let res = await rp({
url: encodeURI(urlToLoad),
encoding: null,
headers: {
'User-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.104 Safari/537.36',
'Referer': url,
'Cache-Control': 'no-cache',
'Host': host,
},
});
if (!path) {
// If no PATH - no save required -> return readed body
return res.toString('utf8');
}
fs.writeFileSync(fileName, res);
console.log(`Done${fileName ? ` "${fileName}"` : ''}: ${res.length} bytes`);
return fileName;
}
catch (err) {
console.error(`saveUrlToFile(${path}, ${urlToLoad})`, err);
throw err;
}
};
login() {
let options = {
uri: this.baseUrl + '/guest?login_id=' + this.hydraId,
json: true,
jar: true
};
return rp(options);
}
getCameraList() {
let options = {
uri: this.baseUrl + '/readCamera',
json: true,
jar: true
};
return rp(options);
}
getSnapshotLink(cameraStreamId) {
let options = {
uri: this.baseUrl + '/snapshot?streamID=' + cameraStreamId,
json: true,
jar: true
};
return rp(options);
}
async function getMosquesFromJAKIM() {
const response = await request({
uri: "https://www.e-solat.gov.my/index.php",
qs: {
r: 'esolatApi/nearestMosque',
lat: 2.91667,
long: 101.7,
// lat: 1.498652,
// long: 103.787032,
dist: 2500,
},
json: true,
});
console.log("Downloaded JAKIM data.");
const items = [];
for (let item of response.locationData) {
items.push({
id: item.no_daftar,
name: item.nama_masjid.split(",")[0].trim(),
address: item.alamat.replace(/\n/g, "").replace(/\r/g, "").replace("KG.", "Kampung") + " " + item.poskod + ", Malaysia",
phone: item.tel.trim() || undefined,
directions: item.lokasi.trim() || undefined,
location: {
latitude: parseFloat(item.latitud),
longitude: parseFloat(item.longitud),
},
provision: item.kemudahan.trim() || undefined,
desc: item.sejarah.trim() || undefined,
});
}
let addressCache = {};
const addressCacheFile = 'cache/reverseGeocodeCache.json';
if (fs.existsSync(addressCacheFile)) {
addressCache = require('./' + addressCacheFile);
}
for (let item of items) {
if (!addressCache[item.location.latitude + ',' + item.location.longitude]) {
try {
let address = await utils.getAddressFromLatLng(item.location.latitude, item.location.longitude);
addressCache[item.location.latitude + ',' + item.location.longitude] = address;
item.address = address;
} catch (e) {
console.warn("Cannot reverse geocode: " + item.address + " (" + item.location.latitude + ", " + item.location.longitude + ")");
}
} else {
item.address = addressCache[item.location.latitude + ',' + item.location.longitude];
}
}
fs.writeFileSync(addressCacheFile, JSON.stringify(addressCache, null, 2));
return items;
}
.then((response) => {
const cookie = response.headers['set-cookie'][0];
return rp({
uri: `http://${HOSTNAME}/api/monitoring/status`,
headers: {
cookie,
},
});
})
.get((req, res, next) => {
const url = decodeURIComponent(req.query.url);
request(url).then(html => {
html = html.substring(html.indexOf('<ul class="photos">') + 19);
html = html.substring(0, html.indexOf('</ul>'));
const images = html.match(/http:\/\/cffc.se\/thumbnail\/thumb\/\d+\/small\.jpg/g);
res.json({ images });
})
.catch(() => res.status(404).end());
});
module.exports.putTrainingDefinition = (url, token, trainingZip) =>
trainingZip.pipe(
request({
method: 'PUT',
json: true,
url: url,
auth: { bearer: token },
headers: { 'content-type': 'application/octet-stream' }
})
)
robot.respond(/pour blood down our gullets/i, async msg => {
if (!Admin.verify(robot, msg)) return;
try {
await bloodSet.truncate();
const $ = await request({
uri: "https://blood-down-the-gullet.obsidianportal.com/adventure-log",
transform: body => cheerio.load(body),
});
const tracks = [];
$(".adventure-log-post .post-content").each((i, e) => {
$(e)
.find("p")
.each((j, p) => {
const pt = $(p).text();
const rx = /[0-9]+\s*[.-]\s*["']?(.+)\n/gi;
let m;
while ((m = rx.exec(pt)) !== null) {
tracks.push(m[1].replace(/["']$/, ""));
}
});
$(e)
.find("ol li")
.each((j, li) => {
tracks.push($(li).text());
});
});
for (const track of tracks) {
await bloodSet.add(msg.message.user.name, track, []);
await new Promise((resolve, reject) => {
robot.markov.modelNamed("bloodkov", model => {
model.learn(track, err => {
if (err) return reject(err);
resolve();
});
});
});
}
msg.reply(
`:metal: Our gullets have accepted ${tracks.length} tracks. :metal:`
);
} catch (err) {
msg.reply(
`I couldn't get to the page on Obsidian portal:\n\`\`\`\n${
err.stack
}\n\`\`\`\n`
);
}
});
return __awaiter(this, void 0, void 0, function* () {
try {
if (helper.isUrl(this.url) && helper.isXml(this.url)) {
return yield request(this.url);
}
}
catch (error) {
console.error(`Error on Request Catch : ${error}`);
throw error;
}
});
async fetchScreenshotInfo(cbtScreenshotId) {
return request({
method: 'GET',
uri: CbtApi.getFullUrl_(`/screenshots/${cbtScreenshotId}`),
auth: {
username: API_USERNAME,
password: API_AUTHKEY,
},
json: true, // Automatically stringify the request body and parse the response body as JSON
});
}