connection.query(sql, function(err,rows) {
if (err) throw err;
connection.release();
if(!rows.length){
console.log(id + ' is not in the database, it will first get the player');
crawler.queueURL(host + url);
} else {
console.log(id + ' is in the database, it will start to get the transfer');
crawler.queueURL(host + url.replace('profil','korrektur'));
}
});
connection.query("SELECT profile_uri FROM `transfermarkt_team` WHERE profile_uri IS NOT NULL AND owner_id = 0", function(err,rows) {
if (err) throw err;
for (var i = rows.length - 1; i >= 0; i--) {
crawler.queueURL(host + rows[i].profile_uri);
};
crawler.start();
});
$('img').each(function (index, img) {
var toQueueUrl = $(img).attr('src');
if (!toQueueUrl) return;
toQueueUrl = url.resolve(queueItem.url, toQueueUrl);
crawler.queueURL(toQueueUrl, queueItem, { content: 'image' });
});
connection.query("SELECT DISTINCT taking_team_id FROM transfermarket_transfer WHERE taking_team_id NOT IN (SELECT id FROM `transfermarket_team`)", function(err,rows) {
if (err) throw err;
connection.release();
for (var i = rows.length - 1; i >= 0; i--) {
crawler.queueURL(host + '/arsenal-fc/startseite/verein/' + rows[i].taking_team_id);
};
});
connection.query("SELECT profile_uri FROM `transfermarkt_player` WHERE height < 100 AND profile_uri IS NOT NULL ORDER BY id DESC", function(err,rows) {
if (err) throw err;
for (var i = rows.length - 1; i >= 0; i--) {
crawler.queueURL(host + rows[i].profile_uri);
};
crawler.start();
});
connection.query("SELECT profile_uri FROM `transfermarkt_team` WHERE profile_uri IS NOT NULL AND id IN ('2779','44247','26099','11952','3561')", function(err,rows) {
if (err) throw err;
for (var i = rows.length - 1; i >= 0; i--) {
crawler.queueURL(host + rows[i].profile_uri);
};
crawler.start();
});
crawler.on("fetchcomplete",function(queueItem, responseBuffer, response){
console.log("Completed fetching resource:", queueItem.path);
var decoder = new StringDecoder('utf8'),$,
next;
if(/^\/\S+?\/startseite\/verein\/\d+?$/i.test(queueItem.path)){//competition
next = this.wait();
$ = cheerio.load(decoder.write(responseBuffer));
$('a.spielprofil_tooltip').each(function(i,el){
crawler.queueURL(host + $(el).attr('href'));
})
Nation.get_nation_by_team($).then(function(){
return Team.get_team($)
}).then(function(){
next();
})
};
if(/^\/\S+\/profil\/spieler\/\d{1,9}$/.test(queueItem.path)){//competition
next = this.wait();
$ = cheerio.load(decoder.write(responseBuffer));
crawler.queueURL(host + queueItem.path.replace(/^\/\S+\/profil\/spieler\/(\d{1,9})$/,'$1'));
Nation.get_nation_by_player_transfer($)/*.then(function(){
Nation.get_nation_by_player_transfer(cheerio.load(decoder.write(responseBuffer)));
})*/.then(function(){
return Player.get_player($)
}).then(function(){
return Team.get_team_by_transfers($)
}).then(function(){
return Transfer.get_trasfer_from_transfers($)
}).then(function(){
next();
}).catch(function(err){
console.log(err);
})
};
}).on('complete',function(){
connection.query("SELECT profile_uri FROM `transfermarkt_player` WHERE date_of_birth = '0000-00-00' AND profile_uri IS NOT NULL ORDER BY id ASC", function(err,rows) {
if (err) throw err;
for (var i = rows.length - 1; i >= 0; i--) {
crawler.queueURL(host + rows[i].profile_uri);
};
connection.release();
crawler.start();
});
connection.query("SELECT profile_uri FROM `transfermarkt_team` WHERE profile_uri IS NOT NULL AND id IN (SELECT DISTINCT team_id FROM transfermarket_team_player)", function(err,rows) {
if (err) throw err;
for (var i = rows.length - 1; i >= 0; i--) {
crawler.queueURL(host + rows[i].profile_uri);
};
connection.release();
crawler.start();
});
connection.query("SELECT transfermarket_competition.uri FROM `competition` JOIN `nation` ON competition.nation_id = nation.id JOIN `transfermarket_nation` ON nation.full_name = transfermarket_nation.name JOIN `transfermarket_competition` ON transfermarket_competition.nation_id = transfermarket_nation.id WHERE transfermarket_competition.competition_name IN (SELECT name FROM `competition`)", function(err,rows) {
if (err) throw err;
connection.release();
for (var i = rows.length - 1; i >= 0; i--) {
var path = rows[i].uri;
path = path.replace('startseite','gesamtspielplan');
crawler.queueURL(host + path);
};
crawler.start();
});
connection.query("SELECT profile_uri FROM transfermarket_team", function(err,rows) {
if (err) throw err;
connection.release();
for (var i = rows.length - 1; i >= 0; i--) {
var path = rows[i].profile_uri;
path = path.replace(/(^\/\S+?\/startseite\/verein\/\d+?)(\/saison_id\/\d{4})?$/,'$1')
crawler.queueURL(host + path);
};
crawler.start();
});
connection.query(sql, function(err,rows) {
if (err) throw err;
for (var i = rows.length - 1; i >= 0; i--) {
var path = rows[i].profile_uri;
path = path.replace('profil','korrektur');
crawler.queueURL(host + path);
};
connection.release();
crawler.start();
players_id = [];
});
$('a').each(function(index, a) {
var toQueueUrl = $(a).attr('href');
var linkContent = $(a).text();
if (!toQueueUrl) return;
if (toQueueUrl.indexOf('#') === 0 && toQueueUrl.length > 1) {
if ($(toQueueUrl).length === 0) {
console.error(chalk.red(util.format('ERROR: 404 relative link ON %s CONTENT %s LINKS TO %s', queueItem.url, linkContent, toQueueUrl)));
errors++;
}
}
if (!shouldCrawl(toQueueUrl)) {
return;
}
var absolutePath = url.resolve(queueItem.url, toQueueUrl);
crawler.queueURL(absolutePath, queueItem, { content: linkContent });
});
$('ul.devices').find('a').each(function(index, a) {
// if we come from a device-specific page, only choose that device link forward
if (selectDevice && $(a).attr('id') !== (selectDevice + '-link')) {
return;
}
var toQueueUrl = $(a).attr('href');
// include hash used to access redirector
var absolutePath = url.resolve(queueItem.url, toQueueUrl) + (parsedUrl.hash || '');
// preserve original fromUrl and content
// c.queue([{
// uri: absolutePath,
// callback: crawlCallback.bind(null, fromUrl, absolutePath, content)
// }]);
if (!queueItem.meta) {
console.log(queueItem);
}
crawler.queueURL(absolutePath, queueItem, { content: queueItem.meta.content });
});
transfers_id.each(function(index,el){
var $el = $(el),
id = $el.val(),
player_id = $('#submenue > li').eq(1).find('> a').attr('href').replace(/\S+?\/(\d{1,9})$/,'$1'),
season = $el.next().children().eq(0).find('select').val(),
transfer_date = $el.next().children().eq(3).find('input').val(),
month = $el.next().children().eq(4).find('select').val(),
loan = $el.next().children().eq(5).find('select').val(),
transfer_sum = $el.next().children().eq(7).find('input').val(),
contract_period = [$el.next().next().children().eq(0).find('input').eq(2).val(),$el.next().next().children().eq(0).find('input').eq(1).val(),$el.next().next().children().eq(0).find('input').eq(0).val()].join('-');
contract_period = /\d{4}\-\d{2}\-\d{2}/.test(contract_period) ? contract_period : '0000-00-00';
transfer_date = /\d{2}\.\d{2}\.\d{4}/.test(transfer_date) ? transfer_date.replace(/(\d{2})\.(\d{2})\.(\d{4})/,'$3-$2-$1') : moment(month + ' 1,' + season).format('YYYY-MM-DD');
transfer_sum = /\d/.test(transfer_sum) ? transfer_sum.replace(/\./g,'') : 0;
var sql = mysql.format("INSERT INTO transfermarket_transfer (id,season,transfer_date,transfer_sum,player_id,contract_period,loan) SELECT ? FROM dual WHERE NOT EXISTS(SELECT id FROM transfermarket_transfer WHERE id = ?)", [[id,season,transfer_date,transfer_sum,player_id,contract_period,loan],id]);
pool.getConnection(function(err, connection) {
connection.query(sql, function(err) {
if (err) throw err;
connection.release();
});
});
var path = queueItem.path.replace('korrektur','transfers');
crawler.queueURL(host + path);
})
crawler.queueURL(host + path);
};
connection.release();
crawler.start();
players_id = [];
});
});
}
}).on('fetcherror',function(queueItem, response){
crawler.queueURL(host + queueItem.path);
}).on('fetchtimeout',function(queueItem, response){
crawler.queueURL(host + queueItem.path);
}).on('fetchclienterror',function(queueItem, response){
crawler.queueURL(host + queueItem.path);
});
crawler.queueURL(host + '/statistik/letztetransfers');
crawler.queueURL(host + '/statistik/letztetransfers?page=2');
crawler.start();
/*pool.getConnection(function(err, connection) {
connection.query("SELECT profile_uri FROM transfermarket_player WHERE id NOT IN (SELECT DISTINCT player_id FROM `transfermarket_transfer`) ORDER BY id DESC LIMIT 100 OFFSET 0", function(err,rows) {
if (err) throw err;
for (var i = rows.length - 1; i >= 0; i--) {
var path = rows[i].profile_uri;
path = path.replace('profil','korrektur');
crawler.queueURL(host + path);
};
connection.release();
crawler.start();
});
}).addFetchCondition(function(parsedURL) {
if((/^\/\S+?\/startseite\/verein\/\d+?$/i.test(parsedURL.path) ||
/^\/\S+\/profil\/spieler\/\d{1,9}$/.test(parsedURL.path) ||
/^\/\S+\/korrektur\/spieler\/\d{1,6}$/.test(parsedURL.path) ||
/^\/\S+\/gesamtspielplan\/wettbewerb\/[\w|\d]+?(\/saison_id\/\d{4})?$/.test(parsedURL.path) ||
/^\/\S+\/startseite\/wettbewerb\/[\w|\d]+?(\/saison_id\/\d{4})?$/i.test(parsedURL.path)/* ||
/^\/\S+\/transfers\/spieler\/\d{1,6}$/.test(parsedURL.path)*/) && !/^\/end\-of\-career\/startseite\/verein\/\d+?$/.test(parsedURL.path) && !/^\/(unattached|unknown|\-tm)\/startseite\/verein\/\d+?$/.test(parsedURL.path)){
if(fetchedUrls.indexOf(parsedURL.path) == -1){//if url not in fetchedUrl
fetchedUrls.push(parsedURL.path)//push it into to avoid fetch twice
return true
}
return false;
}
return false;
///unknown/startseite/verein/75
});
//crawler.queueURL(host + '/1-bundesliga/gesamtspielplan/wettbewerb/L1/saison_id/2015');
//crawler.queueURL(host + '/championship/gesamtspielplan/wettbewerb/GB2/saison_id/2015');
//crawler.queueURL(host + '/pierre-emerick-aubameyang/profil/spieler/58864');
//crawler.queueURL(host + '/luis-henrique/profil/spieler/379877');
//crawler.queueURL(host + '/premier-league/startseite/wettbewerb/GB1');
//crawler.queueURL(host + '/championship/startseite/wettbewerb/GB2');
//crawler.queueURL(host + '//premier-league/gesamtspielplan/wettbewerb/GB1/saison_id/2015');
/*crawler.queueURL(host + '/jumplist/startseite/verein/11');
crawler.queueURL(host + '/jumplist/startseite/verein/148');*/
crawler.queueURL(host + '/premier-league/gesamtspielplan/wettbewerb/GB1');
crawler.queueURL(host + '/jumplist/startseite/wettbewerb/GB1');
crawler.start();
//crawler.queue.add('http', 'www.transfermarkt.co.uk', '20', '/');
//crawler.queueURL(host);
//crawler.start();
}).on('complete',function(){
console.log('complete');
process.exit();
}).on('fetcherror',function(queueItem, response){
console.log('fetcherror ' + queueItem.path)
crawler.queueURL(host + queueItem.path);
}).on('fetchtimeout',function(queueItem, response){
console.log('fetchtimeout ' + queueItem.path)
crawler.queueURL(host + queueItem.path);
}).on('fetchclienterror',function(queueItem, errorData){
console.log('fetchclienterror ' + queueItem.path)
console.log(errorData)
crawler.queueURL(host + queueItem.path);
}).addFetchCondition(function(parsedURL) {
if((/^\/\S+?\/startseite\/verein\/\d+?$/i.test(parsedURL.path) ||
/^\/\S+\/profil\/spieler\/\d{1,9}$/.test(parsedURL.path) ||
/^\/\S+\/korrektur\/spieler\/\d{1,6}$/.test(parsedURL.path) ||
/^\/\S+\/gesamtspielplan\/wettbewerb\/[\w|\d]+?(\/saison_id\/\d{4})?$/.test(parsedURL.path) ||
/^\/\S+\/startseite\/wettbewerb\/[\w|\d]+?(\/saison_id\/\d{4})?$/i.test(parsedURL.path)/* ||
/^\/\S+\/transfers\/spieler\/\d{1,6}$/.test(parsedURL.path)*/) && !/^\/end\-of\-career\/startseite\/verein\/\d+?$/.test(parsedURL.path) && !/^\/(unattached|unknown|\-tm)\/startseite\/verein\/\d+?$/.test(parsedURL.path)){
if(fetchedUrls.indexOf(parsedURL.path) == -1){//if url not in fetchedUrl
fetchedUrls.push(parsedURL.path)//push it into to avoid fetch twice
return true
}
return false;
}
return false;
///unknown/startseite/verein/75
});
crawler.queueURL(host + '/jumplist/startseite/verein/'+team_id);
crawler.start();
}).on('fetchclienterror',function(queueItem, errorData){
console.log('fetchclienterror ' + queueItem.path)
console.log(errorData)
crawler.queueURL(host + queueItem.path);
}).addFetchCondition(function(parsedURL) {
$('a.spielprofil_tooltip').each(function(i,el){
crawler.queueURL(host + $(el).attr('href'));
})
$('a[href*="#"]').each(function(i, anchor) {
crawler.queueURL($(anchor).attr('href'), queueItem);
});
}).on('fetcherror',function(queueItem, response){
crawler.queueURL(host + queueItem.path);
}).on('fetchtimeout',function(queueItem, response){
crawler.on("fetchcomplete",function(queueItem, responseBuffer, response){
var decoder = new StringDecoder('utf8'),sql,
$ = cheerio.load(decoder.write(responseBuffer));
if(/^\/\S+?\/startseite\/verein\/\d+?(\/saison_id\/\d{4})?$/i.test(queueItem.path)){
var team = new Team($);
//team.save_team_player(pool)
//team.update_team_player(pool)
team.get_player_url().forEach(function(url){
if(/^\/\S+\/nationalmannschaft\/spieler\/\d{1,9}$/.test(url)){
url = url.replace(/nationalmannschaft/,'profil');
}
var id = url.replace(/^\/\S+\/profil\/spieler\/(\d{1,9})$/,'$1'),
sql = mysql.format("SELECT 1 FROM transfermarket_player WHERE id = ? limit 1;", [id]);
pool.getConnection(function(err, connection) {
connection.query(sql, function(err,rows) {
if (err) throw err;
connection.release();
if(!rows.length){
console.log(id + ' is not in the database, it will first get the player');
crawler.queueURL(host + url);
} else {
console.log(id + ' is in the database, it will start to get the transfer');
crawler.queueURL(host + url.replace('profil','korrektur'));
}
});
});
});
//team.save(pool);
}
if(/^\/\S+\/profil\/spieler\/\d{1,9}$/.test(queueItem.path)){
var player = new Player($),path = queueItem.path;
player.save(pool);
console.log(player.player_id + ' get, start to get the transfer');
path = path.replace('profil','korrektur');
crawler.queueURL(host + path);
}
if(/^\/\S+\/korrektur\/spieler\/\d{1,9}$/.test(queueItem.path)){
var $ = cheerio.load(decoder.write(responseBuffer));
transfer_table = $('#transfers'),transfer_tbody = transfer_table.find('>tbody'),transfers_id = transfer_tbody.find(' > input[id$="trans_id"]');
transfers_id.each(function(index,el){
var $el = $(el),
id = $el.val(),
player_id = $('#submenue > li').eq(1).find('> a').attr('href').replace(/\S+?\/(\d{1,9})$/,'$1'),
season = $el.next().children().eq(0).find('select').val(),
transfer_date = $el.next().children().eq(3).find('input').val(),
month = $el.next().children().eq(4).find('select').val(),
loan = $el.next().children().eq(5).find('select').val(),
transfer_sum = $el.next().children().eq(7).find('input').val(),
contract_period = [$el.next().next().children().eq(0).find('input').eq(2).val(),$el.next().next().children().eq(0).find('input').eq(1).val(),$el.next().next().children().eq(0).find('input').eq(0).val()].join('-');
contract_period = /\d{4}\-\d{2}\-\d{2}/.test(contract_period) ? contract_period : '0000-00-00';
transfer_date = /\d{2}\.\d{2}\.\d{4}/.test(transfer_date) ? transfer_date.replace(/(\d{2})\.(\d{2})\.(\d{4})/,'$3-$2-$1') : moment(month + ' 1,' + season).format('YYYY-MM-DD');
transfer_sum = /\d/.test(transfer_sum) ? transfer_sum.replace(/\./g,'') : 0;
var sql = mysql.format("INSERT INTO transfermarket_transfer (id,season,transfer_date,transfer_sum,player_id,contract_period,loan) SELECT ? FROM dual WHERE NOT EXISTS(SELECT id FROM transfermarket_transfer WHERE id = ?)", [[id,season,transfer_date,transfer_sum,player_id,contract_period,loan],id]);
pool.getConnection(function(err, connection) {
connection.query(sql, function(err) {
if (err) throw err;
connection.release();
});
});
});
var path = queueItem.path.replace('korrektur','transfers');
console.log('Start to update the transfer of '+path.replace(/^\/\S+\/transfers\/spieler\/(\d{1,9})$/,'$1'));
crawler.queueURL(host + path);
};
if(/^\/\S+\/transfers\/spieler\/\d{1,9}$/.test(queueItem.path)){
var $ = cheerio.load(decoder.write(responseBuffer)),
transfer_table = $('.responsive-table > table'),transfer_tbody = transfer_table.find('>tbody'),transfers_tr = transfer_tbody.find(' > tr');
transfers_tr.each(function(index,el){
var $el = $(el);
if(typeof($el.children().last().find('a').attr('href')) !== 'undefined'){
var id = $el.children().last().find('a').attr('href').replace(/\S+?\/(\d+)$/,'$1'),
releasing_team_id = $el.children().eq(2).find('a').attr('href').replace(/^\/\S+?\/transfers\/verein\/(\d+?)(\/\S+)?$/,'$1'),
taking_team_id = $el.children().eq(5).find('a').attr('href').replace(/^\/\S+?\/transfers\/verein\/(\d+?)(\/\S+)?$/,'$1');
var sql = mysql.format("UPDATE transfermarket_transfer SET ? WHERE id = ?", [{releasing_team_id:releasing_team_id,taking_team_id:taking_team_id},id]);
pool.getConnection(function(err, connection) {
connection.query(sql, function(err) {
if (err) throw err;
connection.release();
});
});
};
})
console.log('Complete update the transfer of ' + queueItem.path.replace(/^\/\S+\/transfers\/spieler\/(\d{1,9})$/,'$1'));
}
}).on('complete',function(){
var Crawler = require("simplecrawler"),
host = 'http://www.whoscored.com',
crawler = new Crawler("www.whoscored.com", "/");
crawler.discoverResources = false;
crawler.userAgent = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36';
crawler.customHeaders = {
Host:'www.whoscored.com',
Referer:'http://www.whoscored.com/LiveScores',
'X-Requested-With':'XMLHttpRequest',
Cookie:'__gads=ID=e173268caa0f2b07:T=1432013869:S=ALNI_MaOSzNoD7wlFKgTdXpQP7oqPIlfag; OX_plg=swf|shk|pm; _gat=1; OX_sd=3; _ga=GA1.2.744658120.1432013868'
};
crawler.on("fetchcomplete",function(queueItem, responseBuffer, response){
console.log("Completed fetching resource:", queueItem.path);
})
crawler.queueURL(host + '/matchesfeed/?d=20150802');
crawler.start();
$('#yw1 td:first-child > [href*="startseite/verein"]').each(function(i,el){
crawler.queueURL(host + $(el).attr('href'));
})
}).on('fetchtimeout',function(queueItem, response){
console.log('fetchtimeout ' + queueItem.path)
crawler.queueURL(host + queueItem.path);
}).on('fetchclienterror',function(queueItem, errorData){
}).on('fetchclienterror',function(queueItem, response){
crawler.queueURL(host + queueItem.path);
});
/**
* @author nttdocomo
*/
var StringDecoder = require('string_decoder').StringDecoder,
Crawler = require("simplecrawler"),
host = 'http://www.transfermarkt.co.uk',
crawler = new Crawler('www.transfermarkt.co.uk');
crawler.maxConcurrency = 1;
crawler.interval = 600;
crawler.timeout = 5000;
crawler.discoverResources = false;
crawler.userAgent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36';
crawler.on("fetchcomplete",function(queueItem, responseBuffer, response){
console.log(queueItem.path);
console.log('fetchcomplete');
var decoder = new StringDecoder('utf8');
//console.log(decoder.write(responseBuffer));
}).on("fetcherror",function(queueItem, response){
console.log('fetcherror');
}).on("fetchstart",function(queueItem,requestOptions ){
console.log('fetchstart');
}).on("fetchtimeout",function(queueItem, crawlerTimeoutValue){
console.log(queueItem.path);
console.log('fetchtimeout');
}).on("fetchclienterror",function(queueItem, errorData){
console.log(queueItem.path);
console.log('fetchclienterror');
});
crawler.queueURL(host + '/johan-cruijff-schaal/startseite/wettbewerb/NLSC');
crawler.start();
result.forEach(function(url) {
crawler.queueURL(url);
});
crawler.on("fetchcomplete",function(queueItem, responseBuffer, response){
console.log("Completed fetching resource:", queueItem.path);
var decoder = new StringDecoder('utf8'),$,
next;
if(/^\/\S+?\/startseite\/verein\/\d+?\/\S*$/i.test(queueItem.path)){//competition
next = this.wait();
$ = cheerio.load(decoder.write(responseBuffer));
$('a.spielprofil_tooltip').each(function(i,el){
crawler.queueURL(host + $(el).attr('href'));
})
Nation.get_nation_by_team($).then(function(){
return Team.get_team($)
}).then(function(){
next();
})
};
if(/^\/\S+\/profil\/spieler\/\d{1,9}$/.test(queueItem.path)){//competition
next = this.wait();
$ = cheerio.load(decoder.write(responseBuffer));
crawler.queueURL(host + queueItem.path.replace(/^\/\S+\/profil\/spieler\/(\d{1,9})$/,'$1'));
Nation.get_nation_by_player_transfer($)/*.then(function(){
Nation.get_nation_by_player_transfer(cheerio.load(decoder.write(responseBuffer)));
})*/.then(function(){
return Player.get_player($)
}).then(function(){
return Team.get_team_by_transfers($)
}).then(function(){
return Transfer.get_trasfer_from_transfers($)
}).then(function(){
next();
}).catch(function(err){
console.log(err);
next();
})
};
//合同
if(/^\/\S+\/korrektur\/spieler\/\d{1,6}$/.test(queueItem.path)){
next = this.wait();
$ = cheerio.load(decoder.write(responseBuffer));
Transfer.get_trasfer_from_korrektur($).then(function(){
next();
})
};
if(/^\/\S+\/gesamtspielplan\/wettbewerb\/[\w|\d]+?(\/saison_id\/\d{4})?$/.test(queueItem.path)){
next = this.wait();
$ = cheerio.load(decoder.write(responseBuffer));
Nation.get_nation_by_competition($).then(function(){
return Team.get_team_by_match_plan($)
}).then(function(){
//console.log('team ok')
return Match.insert_match_by_competition($)
}).then(function(){
next()
})
};
if(/^\/\S+\/startseite\/wettbewerb\/[\w|\d]+?(\/saison_id\/\d{4})?$/.test(queueItem.path)){
next = this.wait();
$ = cheerio.load(decoder.write(responseBuffer));
$('#yw1 td:first-child > [href*="startseite/verein"]').each(function(i,el){
crawler.queueURL(host + $(el).attr('href'));
})
crawler.queueURL(host + queueItem.path.replace(/startseite/,'gesamtspielplan'));
Nation.get_nation_by_competition($).then(function(){
return Competition.get_competition($)
}).then(function(){
next()
})
};
/*if(/^\/\S+\/transfers\/spieler\/\d{1,6}$/.test(queueItem.path)){
next = this.wait();
Transfer.get_trasfer_from_transfers(cheerio.load(decoder.write(responseBuffer))).then(function(){
next();
})
};*/
}).on('complete',function(){