function getJar() {
var jar = new CookieJar();
if (document.cookie && document.cookie.length) {
var ca = document.cookie.split(';');
var cValue;
for (var i = 0; i < ca.length; i++) {
cValue = ca[i];
while (cValue.charAt(0) == ' ') cValue = cValue.substring(1);
jar.setCookies(cValue, accessInfo);
}
}
return jar;
}
.forEach(cookieStr => {
const cookie = new Cookie(cookieStr);
if (!cookie.domain) {
cookie.domain = domain;
}
if (!cookie.path) {
cookie.path = path;
}
cookieJar.setCookie(cookie);
});
requestCookies = uri => {
const parsedURI = new URL(uri),
domain = parsedURI.hostname,
path = parsedURI.pathname,
secure = parsedURI.protocol === 'https:',
script = false,
accessInfo = new CookieAccessInfo(domain, path, secure, script),
cookies = cookieJar.getCookies(accessInfo);
return cookies && cookies.map(cookie => cookie.toValueString());
},
Request.prototype.request = function(){
if (this.req) return this.req;
const options = {};
try {
const query = qs.stringify(this.qs, {
indices: false,
strictNullHandling: true,
});
if (query) {
this.qs = {};
this._query.push(query);
}
this._finalizeQueryString();
} catch (e) {
return this.emit('error', e);
}
let url = this.url;
const retries = this._retries;
// Capture backticks as-is from the final query string built above.
// Note: this'll only find backticks entered in req.query(String)
// calls, because qs.stringify unconditionally encodes backticks.
let queryStringBackticks;
if(url.indexOf('`') > -1) {
const queryStartIndex = url.indexOf("?");
if(queryStartIndex !== -1) {
const queryString = url.substr(queryStartIndex + 1);
queryStringBackticks = queryString.match(/`|\%60/g);
}
}
// default to http://
if (0 != url.indexOf('http')) url = `http://${url}`;
url = parse(url);
// See https://github.com/visionmedia/superagent/issues/1367
if(queryStringBackticks) {
let i = 0;
url.query = url.query.replace(/\%60/g, () => queryStringBackticks[i++]);
url.search = `?${url.query}`;
url.path = url.pathname + url.search;
}
// support unix sockets
if (/^https?\+unix:/.test(url.protocol) === true) {
// get the protocol
url.protocol = `${url.protocol.split('+')[0]}:`;
// get the socket, path
const unixParts = url.path.match(/^([^/]+)(.+)$/);
options.socketPath = unixParts[1].replace(/%2F/g, '/');
url.path = unixParts[2];
}
// options
options.method = this.method;
options.port = url.port;
options.path = url.path;
options.host = url.hostname;
options.ca = this._ca;
options.key = this._key;
options.pfx = this._pfx;
options.cert = this._cert;
options.passphrase = this._passphrase;
options.agent = this._agent;
// Allows request.get('https://1.2.3.4/').set('Host', 'example.com')
if (this._header['host']) {
options.servername = this._header['host'].replace(/:[0-9]+$/,'');
}
// initiate request
const mod = this._enableHttp2 ? exports.protocols['http2:'].setProtocol(url.protocol) : exports.protocols[url.protocol];
// request
const req = (this.req = mod.request(options));
// set tcp no delay
req.setNoDelay(true);
if ('HEAD' != options.method) {
req.setHeader('Accept-Encoding', 'gzip, deflate');
}
this.protocol = url.protocol;
this.host = url.host;
// expose events
req.once('drain', () => { this.emit('drain'); });
req.on('error', err => {
// flag abortion here for out timeouts
// because node will emit a faux-error "socket hang up"
// when request is aborted before a connection is made
if (this._aborted) return;
// if not the same, we are in the **old** (cancelled) request,
// so need to continue (same as for above)
if (this._retries !== retries) return;
// if we've received a response then we don't want to let
// an error in the request blow up the response
if (this.response) return;
this.callback(err);
});
// auth
if (url.auth) {
const auth = url.auth.split(':');
this.auth(auth[0], auth[1]);
}
if (this.username && this.password) {
this.auth(this.username, this.password);
}
for (const key in this.header) {
if (this.header.hasOwnProperty(key))
req.setHeader(key, this.header[key]);
}
// add cookies
if (this.cookies) {
if(this._header.hasOwnProperty('cookie')) {
// merge
const tmpJar = new CookieJar.CookieJar();
tmpJar.setCookies(this._header.cookie.split(';'));
tmpJar.setCookies(this.cookies.split(';'));
req.setHeader('Cookie',tmpJar.getCookies(CookieJar.CookieAccessInfo.All).toValueString());
} else {
req.setHeader('Cookie', this.cookies);
}
}
return req;
};
setCookie = cookie => cookieJar.setCookie(cookie),
getCookies = ({domain, path = '/', secure = true, script = false}) =>
cookieJar.getCookies(new CookieAccessInfo(domain, path, secure, script)),
newAgent = () => {
let logPages = true,
logDir,
userAgent = USER_AGENTS.Mechanize;
const history = newHistory(),
cookieJar = new CookieJar(),
addResponseCookie = ({cookieString, domain, path}) => {
const cookieStringSplitter = /[:](?=\s*[a-zA-Z0-9_-]+\s*[=])/g;
cookieString
.split(cookieStringSplitter)
.forEach(cookieStr => {
const cookie = new Cookie(cookieStr);
if (!cookie.domain) {
cookie.domain = domain;
}
if (!cookie.path) {
cookie.path = path;
}
cookieJar.setCookie(cookie);
});
},
addResponseCookies = ({response, uri, page}) => {
const domain = uri.hostname,
path = uri.pathname;
page.search('//head/meta[@http-equiv="Set-Cookie"]')
.forEach(meta => {
if (meta.attr('content') && meta.attr('content').value()) {
addResponseCookie({
cookieString: meta.attr('content').value(),
domain,
path
});
}
});
let cookieHeaders =
response.headers && response.headers['set-cookie'] || [];
if (!Array.isArray(cookieHeaders)) {
cookieHeaders = [cookieHeaders];
}
cookieHeaders.forEach(header => {
if (header) {
addResponseCookie({cookieString: header, domain, path});
}
});
},
requestHeaders = options => {
const headers = {
'User-Agent': userAgent,
Accept: '*/*'
},
referer = options.referer || history.currentPage() ||
newPage({response: {'content-type': 'text/html'}, agent});
Object.assign(headers, options.headers);
let refererURI;
if (typeof (referer) === 'string') {
refererURI = referer;
} else if (referer) {
refererURI = referer.uri;
}
if (refererURI) {
const parsedURI = new URL(refererURI);
headers.Referer = refererURI;
headers.Origin = parsedURI.protocol + '//' + parsedURI.host;
}
return headers;
},
requestCookies = uri => {
const parsedURI = new URL(uri),
domain = parsedURI.hostname,
path = parsedURI.pathname,
secure = parsedURI.protocol === 'https:',
script = false,
accessInfo = new CookieAccessInfo(domain, path, secure, script),
cookies = cookieJar.getCookies(accessInfo);
return cookies && cookies.map(cookie => cookie.toValueString());
},
getCookieString = uri => {
const cookies = requestCookies(uri);
return cookies && cookies.join('; ');
},
requestOptions = options => {
// uri
// baseUrl
// method
// headers
// qs
// qsParseOptions
// qsStringifyOptions
// useQuerystring
// body
// form
// formData
// multipart
// preambleCRLF
// postambleCRLF
// json
// jsonReviver
// jsonReplacer
// auth
// oauth
// hawk
// aws
// httpSignature
// followRedirect
// followAllRedirects
// followOriginalHttpMethod
// maxRedirects
// removeRefererHeader
// encoding
// gzip
// jar
// agent
// agentClass
// agentOptions
// forever
// pool
// timeout
// localAddress
// proxy
// strictSSL
// tunnel
// proxyHeaderWhiteList
// proxyHeaderExclusiveList
// time
// har
const reqOptions = {
headers: requestHeaders(options),
body: options.body,
followAllRedirects: options.followAllRedirects
};
if (reqOptions.headers.Referer) {
reqOptions.uri =
new URL(options.uri, reqOptions.headers.Referer).toString();
} else {
reqOptions.uri = options.uri.toString();
}
reqOptions.jar = cookieJar;
reqOptions.method = options.verb && options.verb.toUpperCase() || 'GET';
if (options.encoding !== 'undefined') {
reqOptions.encoding = options.encoding;
}
reqOptions.encoding = null;
reqOptions.resolveWithFullResponse = true;
return reqOptions;
},
logPage = async ({body, uri, response}) => {
const contentType = response.headers['content-type'],
ext = mime.extension(contentType &&
contentType.split(/[ \t]*;[ \t]*/)[0]),
timestamp = moment.utc().format('YYYYMMDDHHmmssSSSSSS'),
filename = path.join(
logDir,
timestamp + '_' +
path.basename(uri, path.extname(uri))) +
(ext ? '.' + ext : ''),
encoding = 'utf8';
await writeFile(filename, body, {encoding});
return filename;
},
fetchPage = async options => {
const reqOptions = requestOptions(options),
uri = reqOptions.uri,
response = await request(reqOptions);
if (reqOptions.encoding === null) {
const body = response.body;
if (body[0] === 0xEF && body[1] === 0xBB && body[2] === 0xBF) {
// UTF-8
const body2 = Buffer.allocUnsafe(body.length - 3);
body.copy(body2, 0, 3);
response.body = body2.toString('utf8');
} else if (body[0] === 0xFE && body[1] === 0xFF) {
// UTF-16 big-endian
body.swap16();
const body2 = Buffer.allocUnsafe(body.length - 2);
body.copy(body2, 0, 2);
response.body = body2.toString('utf16le');
} else if (body[0] === 0xFF && body[1] === 0xFE) {
// UTF-16 little-endian
const body2 = Buffer.allocUnsafe(body.length - 2);
body.copy(body2, 0, 2);
response.body = body2.toString('utf16le');
} else {
// UTF-8
response.body = body.toString('binary');
}
}
if (options.fixCharset) {
response.body = response.body.replace('charset=utf-16le', 'utf-8');
if (response.body.match(/charset=windows-1252/)) {
response.body = windows1252.decode(response.body);
}
}
const page = newPage({
uri,
response,
body: response.body,
code: response.statusCode,
agent
});
addResponseCookies({
response,
uri: new URL(uri),
page
});
history.push(page);
if (logPages && logDir) {
await logPage({body: response.body, uri, response});
}
return page;
},
submit = ({form, button, headers, followAllRedirects}) => {
const action = button && button.action || form.action || '',
enctype = button && button.enctype || form.enctype,
method = button && button.method || form.method;
let verb, params, body,
uri = action && querystring.unescape(action) ||
form.page && form.page.uri,
contentType = enctype, requestHeaders = {};
if (button) {
form.addButtonToQuery(button);
}
if (method && method.toLowerCase() === 'post') {
if (contentType === 'multipart/form-data') {
contentType += '; boundary=' + form.boundary;
}
verb = 'post';
body = form.requestData(enctype);
requestHeaders = {
'Content-Type': contentType,
'Content-Length': body.length.toString()
};
} else {
verb = 'get';
uri = uri.replace(/\?[!#$&-;=?-[\]_a-z~]*$/, '');
params = form.buildQuery();
}
return fetchPage({
verb,
uri,
headers: Object.assign(requestHeaders, headers),
referer: form.page,
followAllRedirects,
params,
body
});
},
setUserAgent = agentAlias => userAgent = USER_AGENTS[agentAlias],
getCookies = ({domain, path = '/', secure = true, script = false}) =>
cookieJar.getCookies(new CookieAccessInfo(domain, path, secure, script)),
setCookie = cookie => cookieJar.setCookie(cookie),
agent = {
get: fetchPage,
getCookies,
setCookie,
setUserAgent,
submit,
userAgent: () => userAgent
};
cookieJar.getCookieString = getCookieString;
return Object.freeze(agent);
};