/**
* Parse and extract infos
* @param {String} giturl
* @return {Object} { host, ref, filepath }
*/
static parseUrl(giturl) {
if (!Git.isUrl(giturl)) {
return null;
}
giturl = giturl.slice(GIT_PREFIX.length);
const uri = new URI(giturl);
const ref = uri.fragment() || null;
uri.fragment(null);
// Extract file inside the repo (after the .git)
const fileParts = uri.path().split('.git');
let filepath = fileParts.length > 1 ? fileParts.slice(1).join('.git') : '';
if (filepath[0] == '/') {
filepath = filepath.slice(1);
}
// Recreate pathname without the real filename
uri.path(fileParts[0] + '.git');
return {
host: uri.toString(),
ref,
filepath
};
}
browser.pipeline.addHandler((b, request, response) => {
const url = new URI(response.url);
if (url.path() === '/config.js') {
return new zombie.Response(`angular.module('main').constant('st2Config', {
hosts: [{
name: 'Test',
url: 'https://${process.env.ST2_HOST}/api',
auth: 'https://${process.env.ST2_HOST}/auth',
}],
});`);
}
if (url.path().indexOf('/reamaze.js') >= 0) {
return new zombie.Response('');
}
if (url.host() === process.env.ST2_HOST) {
response._url = url.host('example.com').toString();
request.url = response.url;
}
response.headers.set('access-control-allow-origin', '*');
return response;
});
exports.getSeleniumStatusUrl = function(seleniumArgs) {
var host = 'localhost',
port,
statusURI,
processType,
nodeConfigArg = seleniumArgs.indexOf('-nodeConfig'),
config,
portArg = seleniumArgs.indexOf('-port'),
hostArg = seleniumArgs.indexOf('-host'),
processType = this.getRunningProcessType(seleniumArgs);
// If node config path is pass via -nodeConfig, we have to take settings from there,
// and override them with possible command line options, as later ones have higher priority
if (nodeConfigArg !== -1) {
// Load node configuration and parse it
config = JSON.parse(fs.readFileSync(seleniumArgs[nodeConfigArg + 1], 'utf8'));
if (config['host']) {
host = config['host'];
}
if (config['port']) {
port = config['port'];
}
// If processType is defined, then it was specified via command line options,
// we ignore the value defined in the config file, otherwise we take it from the file
if (!processType && config['role']) {
processType = parseRole(config['role']);
}
}
// Overrode port and host if they were specified
if (portArg !== - 1) {
port = seleniumArgs[portArg + 1];
}
if (hostArg !== - 1) {
host = seleniumArgs[hostArg + 1];
}
var statusURI = new URI('http://' + host);
var nodeStatusAPIPath = '/wd/hub/status';
var hubStatusAPIPath = '/grid/api/hub';
switch (processType) {
case PROCESS_TYPES.STANDALONE:
statusURI.path(nodeStatusAPIPath);
break;
case PROCESS_TYPES.GRID_HUB:
statusURI.path(hubStatusAPIPath);
break;
case PROCESS_TYPES.GRID_NODE:
statusURI.path(nodeStatusAPIPath);
break;
default:
throw 'ERROR: Trying to run selenium in an unknown way.';
}
// Running non-default port if it was specified or default one if it was not
statusURI.port(port || getDefaultPort(processType));
return statusURI.toString();
}
module.exports = function parse(line) {
// well formed inputs:
// http://vector.mapzen.com/osm/all/1/1/1.json
// http://vector.mapzen.com/osm/v0.9/all/1/1/1.json
// http://vector.mapzen.com/osm/tilejson/mapbox.json
// http://tile.mapzen.com/mapzen/vector/v1/tilejson/mapbox.json
// http://tile.mapzen.com/mapzen/vector/v1/all/1/1/1.mvt
// http://tile.mapzen.com/mapzen/terrain/v1/normal/0/0/0.png
// http://tile.mapzen.com/mapzen/terrain/v1/skadi/N25/N25W136.hgt.gz
var fields = line.split(' ');
var timestamp = fields[0].substring(5);
var status = fields[3];
var size = fields[4];
var totalMs = fields[5];
var server = fields[6];
var hostname = fields[7];
var path = fields[8];
var isXonacatl = fields[9] && fields[9] !== '(null)';
var uri = new URI(path || '');
var key = keyFromUri(uri);
var parsedPath;
try {
parsedPath = parsePath(hostname, uri.path());
} catch (err) {
parsedPath = {};
}
if (!parsedPath.api) {
// always default to vector-tiles service so something gets set if
// url parsing fails
parsedPath.api = 'vector-tiles';
}
return {
ts: new Date(timestamp),
api: parsedPath.api,
key: key,
status: status,
origin: 'fastly',
cacheHit: (server === 'App' || server === 'Layers') ? 'MISS' : 'HIT',
size: size,
total_ms: totalMs,
server: server,
path: path,
layer: parsedPath.layer,
x: parsedPath.x,
y: parsedPath.y,
z: parsedPath.z,
hostname: hostname,
format: parsedPath.format,
version: parsedPath.version,
is_xonacatl: isXonacatl,
duplicate: server === 'App' || isXonacatl,
tilesize: parsedPath.tilesize
};
};
item.isSelected = function() {
// item.href() might be relative, in which
// case we should let the browser resolve
// what the full path should be
tmpLink.attr("href", item.href());
var href = new URI(tmpLink[0].href);
var itemPath = trimLeading(href.path(), '/');
if (itemPath === '') {
// log.debug("nav item: ", item.id, " returning empty href, can't be selected");
return false;
}
var current = new URI();
var path = trimLeading(current.path(), '/');
var query = current.query(true);
var mainTab = query['main-tab'];
var subTab = query['sub-tab'];
if (itemPath !== '' && !mainTab && !subTab) {
if (item.isSubTab && _.startsWith(path, itemPath)) {
return true;
}
if (item.tabs) {
var answer = _.any(item.tabs, function(subTab) {
var answer = subTab.isSelected();
return answer;
});
if (answer) {
return true;
}
}
}
var answer = false;
if (item.isSubTab) {
if (!subTab) {
answer = _.startsWith(path, itemPath);
} else {
answer = subTab === item.id;
}
} else {
if (!mainTab) {
answer = _.startsWith(path, itemPath);
} else {
answer = mainTab === item.id;
}
}
return answer;
};
function gotoNavItem(item) {
if (item && item.href) {
var href = trimLeading(item.href(), documentBase($document));
var uri = new URI(href);
var search = _.merge($location.search(), uri.query(true));
log.debug("Going to item id: ", item.id, " href: ", uri.path(), " query: ", search);
$timeout(function() {
$location.path(uri.path()).search(search);
}, 10);
}
}
_getRequestURI: function(path) {
var parsed = URI.parse(path);
var uri = new URI(this.apiURL);
uri.path(parsed.path);
if (!_.isUndefined(parsed.query)) {
uri.query(parsed.query);
}
return uri.toString();
},
browser.pipeline.addHandler(function(browser, request, response) {
var url = new URI(response.url);
if (url.path() === '/config.js') {
return new zombie.Response('angular.module(\'main\').constant(\'st2Config\', {})');
}
if (url.host() === process.env.ST2_HOST) {
response._url = url.host('example.com').toString();
request.url = response.url;
}
return response;
});
subItem.href = function() {
var uri = new URI(subItem.oldHref());
if (uri.path() === "") {
return "";
}
uri.search(function(search) {
_.merge(search, uri.query(true));
if (!search['main-tab']) {
search['main-tab'] = item.id;
}
search['sub-tab'] = subItem.id;
});
return uri.toString();
};
Git.parseUrl = function(giturl) {
var ref, uri, fileParts, filepath;
if (!Git.isUrl(giturl)) return null;
giturl = giturl.slice(GIT_PREFIX.length);
uri = new URI(giturl);
ref = uri.fragment() || null;
uri.fragment(null);
// Extract file inside the repo (after the .git)
fileParts = uri.path().split('.git');
filepath = fileParts.length > 1? fileParts.slice(1).join('.git') : '';
if (filepath[0] == '/') filepath = filepath.slice(1);
// Recreate pathname without the real filename
uri.path(_.first(fileParts)+'.git');
return {
host: uri.toString(),
ref: ref,
filepath: filepath
};
};
exports.buildOrchestration = (name, beforeTimestamp, method, url, requestHeaders, requestContent, res, body) => {
let uri = new URI(url)
return {
name: name,
request: {
method: method,
headers: requestHeaders,
body: requestContent,
timestamp: beforeTimestamp,
path: uri.path(),
querystring: uri.query()
},
response: {
status: res.statusCode,
headers: res.headers,
body: body,
timestamp: new Date()
}
}
}
item.href = function() {
var oldHref = item.oldHref();
if (!oldHref) {
log.debug("Item: ", item.id, " returning null for href()");
return "";
}
var uri = new URI(oldHref);
if (uri.path() === "") {
return "";
}
uri.search(function(search) {
if (!search['main-tab']) {
search['main-tab'] = item.id;
}
_.merge(search, uri.query(true));
if (!search['sub-tab'] && item.tabs && item.tabs.length > 0) {
var sorted = sortByRank(item.tabs);
search['sub-tab'] = sorted[0].id;
}
});
return uri.toString();
};
var server = http.createServer(function (req, res) {
var uri = new URIjs(req.url);
var path = uri.path();
// Add implicit trailing slash
var key = (path.lastIndexOf('/') === (path.length - 1) ? path : path + '/').toLowerCase();
var target = redirects[key];
if (target) {
var target_uri = new URIjs(target).query(uri.query());
res.writeHead(301, {
'Location': target_uri.href(),
'Cache-Control': 'max-age=' + TWO_YEARS_IN_SECONDS
});
res.end('');
}
else {
res.writeHead(302, {
'Location': 'https://www.busbud.com/blog'
});
res.end('');
}
}).listen(PORT);
return new Promise((res, rej) => {
const crawler = new Crawler(site.hostname())
const urls = []
crawler.initialProtocol = site.protocol()
crawler.initialPort = site.port()
|| (site.protocol() === 'http' ? 80 : null)
|| (site.protocol() === 'https' ? 443 : null)
|| 80
crawler.initialPath = site.path()
crawler.interval = 500
crawler.ignoreInvalidSSL = true
crawler.parseHTMLComments = true
crawler.parseScriptTags = true
crawler.downloadUnsupported = false
crawler.maxDepth = 0
crawler.ignoreInvalidSSL = true
crawler.timeout = 10000
if (proxy) {
const proxyInfo = proxy.match(/^(\w+\:\/\/)?(([^\:]+)\:([^\@]+)\@)?([^\:\/]+)\:(\d+)$/)
crawler.useProxy = true
crawler.proxyHostname = proxyInfo[5]
crawler.proxyPort = Number(proxyInfo[6], 10)
crawler.proxyUser = proxyInfo[3]
crawler.proxyPass = proxyInfo[4]
}
crawler.addFetchCondition((parsedURL, queueItem) => {
return /\/[^\/\.]*(\.html)?(\/)?(\?.*)?$/i.test(parsedURL.path) && !urls.find(url => url === parsedURL.path)
})
crawler
.on('crawlstart', (queueItem, options) => {
console.log('Start Crawling: ', site.href())
})
.on('fetchredirect', (queueItem, parsedURL, response) => {
console.log('Redirect found: ', queueItem.url)
if (response.headers.location.indexOf(parsedURL.host)) {
console.log('Using Redirected: ', response.headers.location)
}
})
.on('fetchdisallowed', (queueItem) => {
console.log('Fetch disallowed: ', queueItem.url)
})
.on('fetchheaders', (queueItem, response) => {
if (response.statusCode < 300 && response.statusCode >= 200) {
console.log('Crawled header: ', queueItem.path)
} else {
console.log('Header redirected', queueItem.path)
}
})
.on('fetchcomplete', (queueItem) => {
console.log('Fetch complete:', queueItem.path)
urls.push(queueItem.path)
})
.on('complete', () => {
console.log('Finish crawling.', urls)
res(urls)
})
.on('discoverycomplete', (error, list) => {
console.log('discoverycomplete', list.length)
})
// Errors
.on('fetcherror', (queueItem, response) => {
console.log('Error:', queueItem.url, response.status)
})
.on('fetchtimeout', (queueItem, crawlerTimeoutValue) => {
console.log('Timeout:', queueItem.url)
})
.on('fetchclienterror', (queueItem, error) => {
console.log('Fetch client error:', queueItem.url, error)
})
.on('queueerror', (queueItem, error) => {
console.log('Queue error:', queueItem.url, error)
})
.on('cookieerror', (queueItem, error) => {
console.log('Cookie error:', queueItem.url, error)
})
.on('fetchdataerror', (queueItem, response) => {
console.log('Fetch Data error:', queueItem.url, response.status)
})
.on('robotstxterror', (error) => {
console.log('robotstxterror:', error.message)
})
.start()
})
const monitoringData = MonitoringData.findOne({ apiId });
if (monitoringData && monitoringData.responses) {
// eslint-disable-next-line no-useless-escape
const responses = monitoringData.responses.filter((obj) => {
let result = false;
if (obj.date >= lastDate && obj.date <= startDate) {
result = true;
}
return result;
});
if (monitoringData) {
return responses;
}
}
return [];
},
apiStatusCode (code) {
return code === '200';
},
apiUrlPath () {
const monitoringSettings = MonitoringSettings.findOne({ apiId: this.api._id });
let path;
if (monitoringSettings) {
const apiUrl = new URI(monitoringSettings.url);
path = apiUrl.path();
}
return path;
},
});