socket.on('checkMix', function(data) {
scrape.request('http://youtube.com/watch?v='+data.youtube.videoId, function (err, $) {
if (err) return console.error(err);
$('.related-playlist a').each(function (el) {
title = el.find('span.title').first();
if (title.text.search('YouTube Mix') === 0) {
socket.emit('onMixFound', el.attribs.href.split('list=')[1]);
}
});
});
});
command: function (bot, msg) {
scrape.request('http://www.cheese.com', function (err, $) {
if (err) {
return
}
$('#cheese-of-day').each(function (div) {
var a = div.find('a').first()
var h4 = div.find('h4').first()
bot.say(msg.to, msg.nick + ': the cheese of the day is ' + h4.text + ' https://www.cheese.com' + a.attribs.href)
})
})
}
exports.get_news = function (req, res) {
var headlines = [];
scrape.request('http://economictimes.indiatimes.com/headlines.cms', function (err, $) {
if (err) return console.error(err);
$('ul.headlineData li').each(function (li) {
li.find('a').each(function (a) {
headlines.push({
url : a.attribs.href,
text : a.text
})
})
})
res.send(headlines)
});
// unirest.get('http://news.google.com/news?q=india+finance&output=rss', function (data) {
// res.send(data)
// })
}
return new Promise((resolve, reject) => {
const url = urlFormat({ protocol, hostname, query: { s: query } })
scrape.request(url, function (err, $) {
if (err) return reject(err)
var blogposts = []
$('article.post').each(div => {
var url = div.find('a').first().attribs.href
var title = div.find('h2.post-title').first().text
var author = div.find('p.author').first().text
var date = div.find('time.post-date').first().text
blogposts.push({
url: url,
title: title,
author: author,
date: date
})
})
resolve(blogposts)
})
})
obj.resources = json.resources;
},
_preProcessors(json, obj) {
obj.html_pre_processor = json.html_pre_processor;
obj.css_pre_processor = json.css_pre_processor;
obj.js_pre_processor = json.js_pre_processor;
},
getPenProperties(url, callback) {
scrape.request(url, (err, $) => {
let properties = {};
if (err) return callback(err);
let penValue = JSON.parse(html.decode($('input#init-data').first().attribs.value));
let resource = JSON.parse(penValue.__pen);
this._externalResources(resource, properties);
this._preProcessors(resource, properties);
callback(null, properties);
});
},
downloadFile(url, file, fn) {
http.get(`${util.parseUrl(url)}.${file}`, (res) => {
let buffer = '';
res
.on('data', (chunk) => {
buffer += chunk;
})
.on('end', () => {