getFileExtension: function( media ) {
let extension;
if ( ! media ) {
return;
}
const isString = 'string' === typeof media;
const isFileObject = 'File' in window && media instanceof window.File;
if ( isString ) {
let filePath;
if ( isUri( media ) ) {
filePath = url.parse( media ).pathname;
} else {
filePath = media;
}
extension = path.extname( filePath ).slice( 1 );
} else if ( isFileObject ) {
extension = path.extname( media.name ).slice( 1 );
} else if ( media.extension ) {
extension = media.extension;
} else {
const pathname = url.parse( media.URL || media.file || media.guid || '' ).pathname || '';
extension = path.extname( pathname ).slice( 1 );
}
return extension;
},
router.get('/new/:proto//:url', function(req, res, next) {
// url checking
var error = {
"error":"Wrong url format, make sure you have a valid protocol and real site."
};
var validUrl = require('valid-url');
var in_url = req.params.proto + "//" + req.params.url;
if (!validUrl.isUri(in_url)){
res.end(JSON.stringify(error));
return;
}
// get a random no.
var c = Math.floor(Math.random()*1000);
var data = {
"original_url" : in_url,
"short_url" : "https://req-header-parser-dishant15.c9users.io/" + c.toString(),
};
var collection = req.db.collection("urls");
collection.insert({"_id":c,"url":in_url}, function(err,result){
if(err){
res.end(JSON.stringify(error));
} else {
res.end(JSON.stringify(data));
}
});
});
app.get("/new/:link(*)", function(req, res) {
var arg = req.param('link');
if (validUrl.isUri(arg)){
var entry = {
_id: id,
original: arg,
shorter: "https://shorterer.herokuapp.com/" + id
}
mongo.connect(url, function(err, db) {
var collection = db.collection(col);
collection.insert(entry, function(err, data) {
console.log(JSON.stringify(entry));
db.close();
}); // end insert()
}); // end mongo.connect()
} else {
var entry = { error: arg + " is not a Valid URL! Please try again..."}
}
var id = new ObjectID();
res.end(JSON.stringify(entry, null, ' '));
}) // end app.get(/new/:link)
.then(function(response) {
if (!validUrl.isUri(response.repository)) {
this.ui.writeError('Your answer does not appear to be a URL');
return promptForRepoName.call(this);
}
return response.repository;
}.bind(this));
function (msg, callback) {
var sender = msg.sender.id;
if (msg.message && msg.message.text) {
var text = normalize(msg.message.text);
debug.log('Received: ' + text);
if (validUrl.isUri(text)) {
core.process(text).then(
function (shortened_url) {
sendTextMessage(sender, shortened_url);
callback();
},
function (err) {
callback(err);
}
);
}
else {
sendTextMessage(sender, 'No no this is not a valid URL :(');
callback();
}
}
else {
callback();
}
},
async parseSource() {
// 解析传入的请求源
Logger.instance.info('[acgd] Start to parse source: %s', this.argv.source);
if (libValidUrl.isUri(this.argv.source)) {
// 给予的资源参数是一个链接,即需要下载的albumUrl
this.works.push(this.argv.source);
} else if (await libFsp.exists(this.argv.source)) {
// 给予的资源参数是一个本地文件,即需要下载的内容列表,每个URL一行
let fileContent = await libFsp.readFile(this.argv.source);
let lines = fileContent.toString().match(/^.*((\r\n|\n|\r)|$)/gm);
lines.forEach((line) => {
line = line.replace(/(\r\n|\n|\r|$)/gm, "");
if (line == '') {
// empty line, ignore it
} else if (!libValidUrl.isUri(line)) {
Logger.instance.error('[acgd] Line data in source file is invalid: %s', line);
return;
}
this.works.push(line);
});
} else {
// 非法资源
Logger.instance.error('[acgd] Invalid source: %s', this.argv.source);
process.exit(1);
}
Logger.instance.info('[acgd] Works parsed: ', this.works);
}
webapp.post("/", function(req,res){
console.log("/new")
var resp = res
var candidate = req.body.url
console.log(candidate)
if( validUrl.isUri(candidate) ){
var client = new MetaInspector(candidate, { timeout: 5000 });
client.on("fetch", function(){
console.log("Description: " + client.description);
//imainWindow.loadURL(candidate);
refresh.emit('event',candidate)
resp.send({title:client.title, url:candidate, description: client.description})
});
client.on("error", function(err){
resp.send("ERR");
console.log(error);
});
client.fetch();
}
else {
console.log("Invalid URL: " + candidate)
resp.send("ERR");
}
})
// Extracts proxy configuration parameters from the request
function extractProxyConfig (req, res, next) {
// Retrieve and validate the destination URL
const uri = req.query.uri
debug.settings(`Proxy request for ${uri}`)
if (!validUrl.isUri(uri)) {
return res.status(400).send(`Invalid URL passed: ${uri || '(none)'}`)
}
// Parse the URL and retrieve its host's IP address
const { protocol, host, hostname, path } = url.parse(uri)
if (isIp(hostname)) {
addProxyConfig(null, hostname)
} else {
dns.lookup(hostname, addProxyConfig)
}
// Verifies and adds the proxy configuration to the request
function addProxyConfig (error, hostAddress) {
// Ensure the host is not a local IP
if (error || LOCAL_IP_RANGES.some(r => ipRange(hostAddress, r))) {
return res.status(400).send(`Cannot proxy ${uri}`)
}
req.destination = { path, target: `${protocol}//${host}` }
next()
}
}
function resolveSpotifyTrackQuery(query) {
if (validUrl.isUri(query)) {
var trackId, matches;
if (matches = query.match(/^spotify:track:([0-9A-Za-z]+)$/)) {
trackId = matches[1];
} else {
trackId = path.basename(url.parse(query).pathname);
}
return spotify.getTrack(trackId).then(data => data.body);
}
if (query.indexOf(' - ') !== -1) {
var pieces = query.split(' - ');
query = 'artist:' + pieces[0].trim()
+ ' track:' + pieces[1].trim();
}
return spotify.searchTracks(query)
.then(data => {
var results = data.body.tracks.items;
if (results.length === 0) {
throw "Could not find that track.";
}
return results[0];
});
}
module.exports.update = function (req, res) {
var id = req.params.book_id;
var url = db.escape(req.body.url);
var name = db.escape(req.body.name);
var description = db.escape(req.body.description);
var keywords = db.escape(req.body.keywords);
var uid = req.session.uid;
if (validUrl.isUri(url))
{
console.log('Looks like an URI');
}
else
{
console.log('Not a URI');
req.reportedError = {message: 'URI is inlaid', name: 'Bad Url given', status: 403};
renderIndex(req, res);
}
var queryString = 'UPDATE Bookmarks SET url = ' + url + ', name = ' + name + ', description = '
+ description + ', keywords = ' + keywords + ' WHERE id = ' + id + ' AND userid = ' + uid;
db.query(queryString, function (err) {
if (err)
{
req.reportedError = err;
console.error(err);
}
renderIndex(req, res);
});
};
return new Promise((resolve, reject) => {
// empty
if ( ! site) {
return reject(new Error('missing url'));
}
// not valid
if ( ! validUrl.isUri(site)) {
return reject(new Error('bad url'));
}
// serious check
let opt = url.parse(site);
opt.headers = {
'User-Agent': process.env.USER_AGENT,
};
opt.method = 'HEAD';
let start = Date.now();
let req = opt.protocol.indexOf('https') > -1 ? https : http;
let allowStatusCode = [200, 301, 302];
try {
req.request(opt, function(res) {
if (allowStatusCode.indexOf(res.statusCode) === -1) {
return reject(new Error('bad url'));
}
resolve(start);
}).on('error', function() {
reject(new Error('bad url'));
}).end();
} catch(err) {
reject(new Error('bad url'));
}
});
export async function combineSource(fileRoot, sources) {
let fn, importLine, ir;
var matches = [];
ir = /^(?:import){1}(.+){0,1}\s['"](.+)['"];/gm;
let match = null;
for (const fileName of Object.keys(sources)) {
const source = sources[fileName].content;
while((match = ir.exec(source))) {
matches.push(match);
}
for(let match of matches) {
importLine = match[0];
const extra = match[1] ? match[1] : '';
if(validUrl.isUri(fileRoot)) {
fn = url.resolve(fileRoot, match[2]);
} else {
fn = match[2];
}
try {
// resolve anything other than remix_tests.sol & tests.sol
if(fn.localeCompare('remix_tests.sol') != 0 && fn.localeCompare('tests.sol') != 0) {
let subSorce = {};
const response = await resolveImports(fileRoot, fn);
sources[fileName].content = sources[fileName].content.replace(importLine, 'import' + extra + ' \'' + response.filename + '\';');
subSorce[response.filename] = { content: response.content };
sources = Object.assign(await combineSource(response.fileRoot, subSorce), sources);
}
} catch (e) {
throw e;
}
}
}
return sources;
}
app.get('/new/*', function (req, res) {
let id;
if (validUrl.isUri(req.params[0])) {
mongoClient.connect(url, function(err, db) {
if (err) throw err;
db.collection('url').find().sort({_id: 1}).toArray(function (err, docs) {
if (err) throw err;
id = docs.length;
let data = {
id: id,
origin_url: req.params[0],
short_url: 'https://' + req.headers.host + '/' + id
};
db.collection('url').insert(data);
res.json({
origin_url: data.origin_url,
short_url: data.short_url
});
db.close();
});
});
} else {
res.json({
"error": "Wrong url format, make sure you have a valid protocol and real site."
});
}
});
module.exports.insert = function (req, res) {
console.log('This i sa test bl bla bla');
var sqlInsert = sqlQuery.insert();
var newBookmark = {};
newBookmark.url = req.body.url;
newBookmark.name = req.body.name;
newBookmark.folderId = req.body.folderId;
newBookmark.description = req.body.description;
newBookmark.keywords = req.body.keywords;
newBookmark.favorite = 0;
newBookmark.isFolder = 0;
newBookmark.userId = req.session.uid;
if (validUrl.isUri(newBookmark.url))
{
console.log('Looks like an URI');
}
else
{
console.log('Not a URI');
req.reportedError = {message: 'URI is inlaid', name: 'Bad Url given', status: 403};
renderIndex(req, res);
return;
}
var queryString = sqlInsert.into('Bookmarks').set(newBookmark).build();
console.log(queryString);
var query = Bookmarks.create(newBookmark);
query.then(()=> renderIndex(req, res));
query.catch((error) => {
req.reportedError = {message: error.message, name: 'Failed to create bookmark', status: 400};
renderIndex(req, res);
});
};
exports.isUrlValid = function (url) {
if (!(validUrl.isUri(url))) {
return false;
} else {
return true;
}
};
this.saveUrlAlias = function(req,res){
var url = req.params[0];
if(ValidUrl.isUri(url)){
var origin = UrlModule.format({
protocol: req.protocol,
host: req.get('host')
});
//Validate URL
var codeLength = Math.floor(Math.random() * (5-2+1)) + 2;
var randomCode ="";
var possible = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
for(var i = 0; i < codeLength; i++) {
randomCode += possible.charAt(Math.floor(Math.random() * possible.length));
}
var urlAlias = new UrlAliases({ url: url,code:randomCode});
urlAlias.save(function(err,result){
if(err){
throw err;
}
res.json({"original_url":url,"short_url": origin + "/" + randomCode});
});
}else{
res.json({"error":"URL invalid"});
}
};
constructor(url) {
this.tree = {
nodes: {
values: [],
lastNodeId: 0,
push(currentUrl) {
const newNode = {
id: this.lastNodeId,
url: currentUrl,
isExplored: false,
};
this.values.push(newNode);
this.lastNodeId += 1;
},
get(currentUrl) {
return this.values.filter(node => node.url === currentUrl)[0];
},
},
concat(newData) {
const newConcatValues = _.uniq(this.nodes.values.concat(newData.nodes.values));
this.nodes.values = newConcatValues;
return this;
},
};
// TODO : verify the status code of the homepage
if (!validUrl.isUri(url)) {
throw new Error('Please, set a valid url');
}
this.seedUrl = url;
this.tree.nodes.push(url);
}
handlers.results = (request, reply) => {
/* validate the url */
if (!validUrl.isUri(request.payload.url))
return reply.view('badInput');
/* Fetch the url and then analyze it */
Rp({
uri: request.payload.url,
transform: (body) => {
return Cheerio.load(body);
}
}).then(($) => {
let data = {};
// let content = $('.section-inner').text(); // medium specific selector
let content = $('body');
data.url = URL.parse(request.payload.url);
data.counts = TextStats.stats(content);
data.analysis = Readability.stats(data.counts, content);
return reply.view('results', data);
}).catch((err) => {
console.log(err);
});
};
updatePreview: function() {
var url = this.$imageUrl.val();
if (valid.isUri(url)) {
this.$previewContainer.show();
this.$uploadImage.val('');
this.$preview.attr('src', url);
}
},
return yield srcs.map(function *(src) {
var file = yield (uri.isUri(src) ? this.importUrl(src) : this.importFile(src));
yield fs.mkdir(this.resolve(file.name));
yield fs.writeFile(this.resolve(file.name), file.buffer);
return file.name;
}, this);
shim.fetch = async function(url, options = null) {
const validatedUrl = urlValidator.isUri(url);
if (!validatedUrl) throw new Error('Not a valid URL: ' + url);
return shim.fetchWithRetry(() => {
return nodeFetch(url, options)
}, options);
}
exports.isUrlValid = function(url){
console.log('in isUrlValid');
console.log('url is' + url);
if (!(validUrl.isUri(url))){
return false;
}else{
return true
}
};
function isValid(url){
if (validUrl.isUri(url)){
console.log('Looks like an URI');
return 1;
} else {
console.log('Not a URI');
return 0;
}
}
const recuveHistoricalRemote = (url) =>{
if (validUrl.isUri(url)) {
serviceRecuve(url)
console.log('Espere mientras importamos El historical...')
} else {
console.log(`${chalk.red('[error]')} Please enter a params valid URL`)
process.exit()
}
}
linkFiltering(link) {
return (
validUrl.isUri(link) && // is a proper URL
link.indexOf('#') === -1 && // is not an anchor
link.indexOf('.pdf') === -1 && // is not a pdf
link.indexOf('.jpg') === -1 && // is not a jpg
link.indexOf('.png') === -1 && // is not a png
link.search(this.seedUrl) === 0
); // is in the same domain
}
lines.forEach((line) => {
line = line.replace(/(\r\n|\n|\r|$)/gm, "");
if (line == '') {
// empty line, ignore it
} else if (!libValidUrl.isUri(line)) {
Logger.instance.error('[acgd] Line data in source file is invalid: %s', line);
return;
}
this.works.push(line);
});
api.capture = function capture (req, res, next) {
console.log('request to capture website: ', req.query.url);
if (!req.query.url || !validUrl.isUri(req.query.url)) {
return next(new RErrors.BadRequestError('Invalid URL: ' + req.query.url));
}
worker(req.query.url, function (err, results) {
res.json(results);
return next();
});
}
app.get('/new/:url*', function (req, res) {
var trimmedUrl = req.url.slice(5)
var checkedURL = validUrl.isUri(trimmedUrl)
var short = process.env.APPURL + shortid.generate()
urlPairModel.find({shortUrl: short})
var searchResult
console.log(trimmedUrl)
if (checkedURL == undefined) {
res.send("Please provide a valid URL.")
}
//Does the URL exist in the database already? If not, make a new entry
//and return JSON
else {
var checkedURL = sanitize(checkedURL)
urlPairModel.find({ longUrl: checkedURL }, function(err, query) {
if (err) throw err;
else if (query[0] == undefined) {
var urlPair = new urlPairModel({
'longUrl': checkedURL,
'shortUrl': short
})
urlPair.save(function (err) {
if (err) throw(err)
console.log('Test url saved')
})
urlPairModel.find({longUrl: checkedURL}, function(err, confirmQuery) {
if (err) throw(err)
res.send(JSON.stringify({longUrl: confirmQuery[0].longUrl, shortUrl: confirmQuery[0].shortUrl}))
})
}
//Otherwise, return the known value from the database.
else {
console.log('Known URL')
res.send(JSON.stringify({longUrl: query[0].longUrl, shortUrl: query[0].shortUrl}))
}
});
}
})
it('should correctly produce a signed spinpapi url', function(done) {
spinitron = new spinitron({
station: 'ksdt',
userid: 'testing',
secret: 'testing'
});
var request = spinitron.generateRequest('getSong', {songId: 2});
should.exist(request);
should.exist(validurl.isUri(request));
done();
});
app.get('/new/*',function(request,response){
var sequence = 0;
var req_url = request.path.replace('/new/','');
if(!validUrl.isUri(url)){
response.json({Error : "URL Invalid"});
}else{
var sequence = Math.floor((Math.random()*1000)+1);
Urls.insertOne({"original_url":req_url, "short_url":original_url+sequence});
response.json({"original_url":req_url, "short_url":original_url+sequence});
response.end();
}
});