function normalizeURL(link, baseLink, disallowHttp = false) {
try {
const url = new URL(link, baseLink);
// Only fetch http and https links
if (url.protocol !== "http:" && url.protocol !== "https:") {
return null;
}
if (disallowHttp && url.protocol === "http:") {
return null;
}
// Do not fetch links without hostname or ones that contain authorization
if (!url.hostname || url.username || url.password) {
return null;
}
// Drop hash from the url, if any
url.hash = "";
return url.toString();
} catch (e) {
// if an exception was thrown, the url is not valid
}
return null;
}
s3Url(key) {
const host = this.pjson.oclif.update.s3.host;
if (!host)
throw new Error('no s3 host is set');
const url = new url_1.URL(host);
url.pathname = path.join(url.pathname, key);
return url.toString();
}
before(async function(){
this.htmlResponse = await helpers.makeRequest("http://localhost:8071/");
var instrURL = helpers.extractInstructionsURL(this.htmlResponse.body);
var endpoint = new URL(instrURL, "http://localhost:8071");
this.mutationResponse = await helpers.makeRequest(endpoint.toString(), false);
this.mutations = helpers.decodeMutations(this.mutationResponse.body);
});
_cleanUrl(input) {
const url = new URL.URL(input);
for (const name of url.searchParams.keys()) {
if (name.toLowerCase().startsWith('utm_')) {
url.searchParams.delete(name);
}
}
return url.toString();
}
get url() {
if (this.opts.basicAuth) {
const u = new URL(this.opts.url);
u.username = this.opts.basicAuth.username || '';
u.password = this.opts.basicAuth.password || '';
return u.toString();
}
return this.opts.url;
}
_updateQueryString(obj) {
const url = this.url;
const queryString = qs.stringify(obj);
if (url) {
const urlObj = new urlModule.URL(url);
if (queryString.length > 0) {
urlObj.search = `?${queryString}`;
} else {
urlObj.search = '';
}
this.url = urlObj.toString();
} else {
this._query = queryString;
}
}
return (registryUrl ? rxjs_1.of(registryUrl) : getNpmConfigOption('registry', scope, true)).pipe(operators_1.map(partialUrl => {
if (!partialUrl) {
partialUrl = 'https://registry.npmjs.org/';
}
const partial = url.parse(partialUrl);
let fullUrl = new url.URL(`http://${partial.host}/${packageName.replace(/\//g, '%2F')}`);
try {
const registry = new url.URL(partialUrl);
registry.pathname = (registry.pathname || '')
.replace(/\/?$/, '/' + packageName.replace(/\//g, '%2F'));
fullUrl = new url.URL(url.format(registry));
}
catch (_a) { }
logger.debug(`Getting package.json from '${packageName}' (url: ${JSON.stringify(fullUrl)})...`);
return fullUrl.toString();
}), operators_1.concatMap(fullUrl => {
constructor(config = {}) {
const baseUrl = new URL(config.host || config.baseUrl || 'http://localhost:5678');
delete baseUrl.path;
delete baseUrl.hash;
this._request = rp.defaults({
baseUrl: baseUrl.toString(),
headers: {
'User-Agent': 'Teraslice Client',
Accept: 'application/json'
},
simple: false,
resolveWithFullResponse: true,
json: true
});
}
const getAutodropURL = ({ config, params = {} }) => {
const {
deviceID,
deviceKey,
} = config
const url = new URL(config.apiURL || DEFAULT_URL)
url.search = new URLSearchParams({
name: deviceID,
key: deviceKey,
...params,
})
return url.toString()
}
set path(value) {
const url = this.url;
if (url) {
if (url.startsWith(this.assetGraph.root)) {
this.url =
this.assetGraph.root +
value.replace(/^\//, '').replace(/\/?$/, '/') +
(this.fileName || '');
} else {
const urlObj = new urlModule.URL(url);
urlObj.pathname = value.replace(/\/?$/, '/') + (this.fileName || '');
this.url = urlObj.toString();
}
} else if (this.isInline) {
throw new Error('Cannot update the path of an inline asset');
}
}
function getYouTubeIFrameSrc(string) {
const url = getUrl(string)
let id = url.searchParams.get('v')
if (url.host === 'youtu.be') {
id = url.pathname.slice(1)
}
const embedUrl = new URL(`https://www.youtube-nocookie.com/embed/${id}?rel=0`)
url.searchParams.forEach((value, name) => {
if (name === 'v') {
return
}
if (name === 't') {
name = 'start'
value = getTimeValueInSeconds(value)
}
embedUrl.searchParams.append(name, value)
})
return embedUrl.toString()
}
constructor(opts) {
if ( ! opts) {
opts = {};
}
this.opts = _.extend(
{
debug: false,
database: DATABASE,
password: '',
basicAuth: null,
isUseGzip: false,
config: {
// session_id : Date.now(),
session_timeout : 60,
output_format_json_quote_64bit_integers : 0,
enable_http_compression : 0
}
},
opts
);
let url = opts.url || opts.host || URI,
port = opts.port || PORT;
if ( ! url.match(/^https?/)) {
url = 'http://' + url;
}
const u = new URL(url);
if (u.protocol === 'https:' && (port === 443 || port === 8123)) {
u.port = '';
} else if (port) {
u.port = port;
}
this.opts.url = u.toString();
this.opts.username = this.opts.user || this.opts.username || USERNAME;
}
const LinkSearchableProperty = ({
$c,
entityType,
searchField,
searchValue,
text = searchValue,
}: Props) => {
searchField = searchValue === '*' ? '-' + searchField : searchField;
const url = new URL($c.req.uri);
url.pathname = '/search';
url.search =
'query=' +
encodeURIComponent(
searchField + ':"' +
escapeLuceneValue(searchValue) + '"',
) +
'&type=' + encodeURIComponent(entityType) +
'&limit=25&method=advanced';
return <a href={url.toString()}>{text}</a>;
};
/**
* Recursively request individual stories by ID.
*
* @param {Number} feedId - The feed unique identifier provided to the listener.
* @param {String} feedUrl - The URL provided to the listener.
* @param {fetchCallback} - The callback provided to the listener.
* @param {String} fetchId - The unique identifier for this fetch.
* @param {Number[]} itemIds - A list of story IDs.
*/
function getStories(feedId, feedUrl, callback, fetchId, itemIds) {
if (itemIds.length === 0) {
callback(null);
return;
}
const endpoint = new url.URL(`/v0/item/${itemIds[0]}.json`, baseUrl);
needle.get(endpoint.toString(), {
open_timeout: 5000,
response_timeout: 10000,
follow_max: 2,
parse_response: true
}, (err, res, body) => {
if (err) {
callback(err);
return;
}
if (res.statusCode !== 200) {
callback(new Error(`Story endpoint responded with ${res.statusCode}`));
return;
}
transformStory.call(this, feedId, feedUrl, fetchId, body);
// This is redundant with the earlier check, but avoids
// a timeout when processing the last item in the id list.
if (itemIds.length === 1) {
callback(null);
return;
}
setTimeout(() => {
getStories.call(this, feedId, feedUrl, callback, fetchId, itemIds.slice(1));
}, 1000);
});
}
module.exports = function (feedId, feedUrl, callback = () => {}) {
const fetchId = crypto.pseudoRandomBytes(10).toString('hex');
// Only the topstories feed is supported at this time.
const endpoint = new url.URL('/v0/topstories.json', baseUrl);
endpoint.search = querystring.stringify({
limitToFirst: itemLimit,
orderBy: '"$key"'
});
needle.get(endpoint.toString(), {
open_timeout: 5000,
response_timeout: 10000,
follow_max: 2,
parse_response: true
}, (err, res) => {
if (err) {
// Status code zero is used to indicate fetch failure.
this.emit('stats-fetch', feedId, fetchId, 0, false);
callback(err);
return;
}
if (res.statusCode !== 200) {
this.emit('stats-fetch', feedId, fetchId, res.statusCode, false);
callback(new Error(`${baseUrl.hostname} responded with ${res.statusCode}`));
return;
}
getStories.call(this, feedId, feedUrl, callback, fetchId, res.body);
this.emit('feed-update', feedId, {
updated: new Date()
});
});
};
let value = new urlModule.URL(url)[urlPropertyName];
if (urlPropertyName === 'port') {
if (value) {
value = parseInt(value, 10);
} else {
value = undefined;
}
}
return value;
}
},
set(value) {
const url = this.url;
if (url) {
const urlObj = new urlModule.URL(url);
urlObj[urlPropertyName] = value;
this.url = urlObj.toString();
} else if (this.isInline) {
throw new Error(
`Cannot update the ${urlPropertyName} of an inline asset`
);
}
}
});
}
Object.assign(Asset.prototype, {
/**
* Boolean Property that's true for all Asset instances. Avoids
* reliance on the `instanceof` operator.
set url(url) {
if (!this.isExternalizable) {
throw new Error(
`${this.toString()} cannot set url of non-externalizable asset`
);
}
const oldUrl = this._url;
if (url && !/^[a-z+]+:/.test(url)) {
// Non-absolute
const baseUrl =
oldUrl ||
(this.assetGraph && this.baseAsset && this.baseUrl) ||
(this.assetGraph && this.assetGraph.root);
if (!baseUrl) {
throw new Error(
`Cannot find base url for resolving new url of ${
this.urlOrDescription
} to non-absolute: ${url}`
);
}
if (/^\/\//.test(url)) {
// Protocol-relative
url = urlTools.resolveUrl(baseUrl, url);
} else if (/^\//.test(url)) {
// Root-relative
if (/^file:/.test(baseUrl) && /^file:/.test(this.assetGraph.root)) {
url = urlTools.resolveUrl(this.assetGraph.root, url.substr(1));
} else {
url = urlTools.resolveUrl(baseUrl, url);
}
} else {
// Relative
url = urlTools.resolveUrl(baseUrl, url);
}
}
if (url !== oldUrl) {
const existingAsset = this.assetGraph._urlIndex[url];
if (existingAsset) {
// Move the existing asset at that location out of the way
let nextSuffixToTry = 1;
let newUrlForExistingAsset;
do {
const urlObj = new URL(url);
urlObj.pathname = urlObj.pathname.replace(
/([^/]*?)(\.[^/]*)?$/,
($0, $1, $2) => `${$1}-${nextSuffixToTry}${$2 || ''}`
);
newUrlForExistingAsset = urlObj.toString();
nextSuffixToTry += 1;
} while (this.assetGraph._urlIndex[newUrlForExistingAsset]);
existingAsset.url = newUrlForExistingAsset;
}
this._url = url;
this._query = undefined;
this._updateUrlIndex(url, oldUrl);
if (url) {
this.incomingInlineRelation = undefined;
if (!urlEndsWithSlashRegExp.test(url)) {
const pathname = urlTools.parse(url).pathname;
this._extension = pathModule.extname(pathname);
this._fileName = pathModule.basename(pathname);
this._baseName = pathModule.basename(pathname, this._extension);
}
}
if (this.assetGraph) {
for (const incomingRelation of this.assetGraph.findRelations({
to: this
})) {
incomingRelation.refreshHref();
}
for (const relation of this.externalRelations) {
relation.refreshHref();
}
}
}
}
function makeUrl(s) {
const url = new URL(s, baseUrl);
return url.toString();
}
// Flags: --expose-internals
'use strict';
require('../common');
const URL = require('url').URL;
const assert = require('assert');
const urlToOptions = require('internal/url').urlToOptions;
// Tests below are not from WPT.
const url = new URL('http://*****:*****@foo.bar.com:21/aaa/zzz?l=24#test');
const oldParams = url.searchParams; // for test of [SameObject]
// To retrieve enumerable but not necessarily own properties,
// we need to use the for-in loop.
const props = [];
for (const prop in url) {
props.push(prop);
}
// See: https://url.spec.whatwg.org/#api
// https://heycam.github.io/webidl/#es-attributes
// https://heycam.github.io/webidl/#es-stringifier
const expected = ['toString',
'href', 'origin', 'protocol',
'username', 'password', 'host', 'hostname', 'port',
'pathname', 'search', 'searchParams', 'hash', 'toJSON'];
assert.deepStrictEqual(props, expected);
// href is writable (not readonly) and is stringifier
assert.strictEqual(url.toString(), url.href);
return new Promise(function (resolve, reject) {
var data;
var headers = Object.assign({}, config.headers);
var fullUrl = config.url;
if (config.data) {
// GET and HEAD do not support body in request.
if (config.method === 'GET' || config.method === 'HEAD') {
if (!validator.isObject(config.data)) {
return reject(createError(config.method + " requests cannot have a body", config));
}
// Parse URL and append data to query string.
var parsedUrl = new url.URL(fullUrl);
var dataObj = config.data;
for (var key in dataObj) {
if (dataObj.hasOwnProperty(key)) {
parsedUrl.searchParams.append(key, dataObj[key]);
}
}
fullUrl = parsedUrl.toString();
}
else if (validator.isBuffer(config.data)) {
data = config.data;
}
else if (validator.isObject(config.data)) {
data = Buffer.from(JSON.stringify(config.data), 'utf-8');
if (typeof headers['Content-Type'] === 'undefined') {
headers['Content-Type'] = 'application/json;charset=utf-8';
}
}
else if (validator.isString(config.data)) {
data = Buffer.from(config.data, 'utf-8');
}
else {
return reject(createError('Request data must be a string, a Buffer or a json serializable object', config));
}
// Add Content-Length header if data exists
if (data) {
headers['Content-Length'] = data.length.toString();
}
}
var parsed = url.parse(fullUrl);
var protocol = parsed.protocol || 'https:';
var isHttps = protocol === 'https:';
var port = parsed.port;
if (!port) {
port = isHttps ? '443' : '80';
}
var options = {
hostname: parsed.hostname,
port: port,
path: parsed.path,
method: config.method,
agent: config.httpAgent,
headers: headers,
};
var transport = isHttps ? https : http;
var req = transport.request(options, function (res) {
if (req.aborted) {
return;
}
// Uncompress the response body transparently if required.
var respStream = res;
var encodings = ['gzip', 'compress', 'deflate'];
if (encodings.indexOf(res.headers['content-encoding']) !== -1) {
// Add the unzipper to the body stream processing pipeline.
var zlib = require('zlib');
respStream = respStream.pipe(zlib.createUnzip());
// Remove the content-encoding in order to not confuse downstream operations.
delete res.headers['content-encoding'];
}
var response = {
status: res.statusCode,
headers: res.headers,
request: req,
data: undefined,
config: config,
};
var responseBuffer = [];
var boundary = getMultipartBoundary(res.headers);
if (boundary) {
var dicer = require('dicer');
var multipartParser = new dicer({ boundary: boundary });
multipartParser.on('part', function (part) {
var tempBuffers = [];
part.on('data', function (partData) {
tempBuffers.push(partData);
});
part.on('end', function () {
responseBuffer.push(Buffer.concat(tempBuffers));
});
});
multipartParser.on('finish', function () {
response.data = null;
response.multipart = responseBuffer;
finalizeRequest(resolve, reject, response);
});
respStream.pipe(multipartParser);
}
else {
respStream.on('data', function (chunk) {
responseBuffer.push(chunk);
});
respStream.on('error', function (err) {
if (req.aborted) {
return;
}
reject(enhanceError(err, config, null, req));
});
respStream.on('end', function () {
var responseData = Buffer.concat(responseBuffer).toString();
response.data = responseData;
finalizeRequest(resolve, reject, response);
});
}
});
// Handle errors
req.on('error', function (err) {
if (req.aborted) {
return;
}
reject(enhanceError(err, config, null, req));
});
if (config.timeout) {
// Listen to timeouts and throw an error.
req.setTimeout(config.timeout, function () {
req.abort();
reject(createError("timeout of " + config.timeout + "ms exceeded", config, 'ETIMEDOUT', req));
});
}
// Send the request
req.end(data);
});