_combo: function(result, outputPath, outputCharset){
var self = this,
combinedComment = [
'/*',
'combined files : ',
'',
result.combined.join('\r\n'),
'',
'*/',
''
].join('\r\n');
// deal with output charset. if not specified, use charset in config.
outputCharset = (typeof outputCharset === 'undefined' || outputCharset === null) ? self._config.charset : outputCharset;
var combineFile = self._config.suffix ? outputPath.replace(/\.js$/i, self._config.suffix + '.js') : outputPath;
//prepare output dir
fileUtil.mkdirsSync(path.dirname(combineFile));
// if exists, unlink first, otherwise, there may be some problems with the file encoding.
if(fs.existsSync(combineFile)){
fs.unlinkSync(combineFile);
}
var fd = fs.openSync(combineFile, 'w');
// fs.writeSync(fd, combinedComment, 0, convertCharset(self._config.charset));
var combinedCommentBuffer = iconv.encode(combinedComment, outputCharset);
fs.writeSync(fd, combinedCommentBuffer, 0, combinedCommentBuffer.length);
fs.closeSync(fd);
fd = fs.openSync(combineFile, 'a');
for (var moduleName in result._moduleCache) {
if(result._moduleCache.hasOwnProperty(moduleName)){
var module = result._moduleCache[moduleName],
moduleContent = iconv.decode(fs.readFileSync(module.path), module.charset);
//add module path
var start = moduleContent.indexOf('KISSY.add(');
if(start == -1) {
start = moduleContent.indexOf('.add(');
if(start != -1) {
start = start + 5;
}
} else {
start = start + 10;
}
var end = moduleContent.indexOf('function', start);
//find it
if(start > -1 && end > start) {
//KISSY.add(/*xxx*/function(xxx)) or KISSY.add('xxx', function(xxx))
//remove user comments but preserve user defined module name.
var oldModuleName = moduleContent.substring(start, end),
moduleNameRegResult = oldModuleName.match(/^(\s*)\/\*(.*)\*\/(\s*)$/);
if(moduleNameRegResult != null){
// replace user comments with my module name.
// moduleContent = moduleContent.replace(oldModuleName, moduleNameRegResult[1]);
moduleContent = moduleContent.replace(oldModuleName, '\'' + module.name + '\', ');
}
} else if(start > -1 && end == start) {
//KISSY.add(function(xxx))
moduleContent = [moduleContent.slice(0, start), '\'' + module.name + '\',', moduleContent.slice(end)].join('');
}
// add a new line after.
moduleContent += '\r\n';
var buffer = iconv.encode(moduleContent, outputCharset);
fs.writeSync(fd, buffer, 0, buffer.length);
}
}
fs.closeSync(fd);
},
}, function (error, response, html) {
var bodyWithCorrectEncoding = iconv.decode(html, 'iso-8859-1');
if (!error) {
var $ = cheerio.load(bodyWithCorrectEncoding);
$('.boxEssenplanHolder').filter(function (elem) {
var data = $(this);
data.children('.boxEssenplanInfo').each(function (i) {
var plan = $(this);
plan.children().each(function (z) { // refactor!
var node = $(this);
if (z == 0) {
var h1 = node.children('h1');
if (h1.text().indexOf('Herwegh') !== -1) {
var dateCluster = h1.parent().parent().children('.boxGerichte').each(function () {
$(this).children('p').each(function () {
var month = moment().month() + 1;
var day = moment().format('DD');
var today = new Date();
// extract date...
var givenDateDay = $(this).text().match(/^\d+|\d+\b|\d+(?=\w)/g)[0];
var givenDateMonth = $(this).text().match(/^\d+|\d+\b|\d+(?=\w)/g)[1];
var currentYear = moment().format('YYYY');
var givenDate = new Date(currentYear + '-' + givenDateMonth + '-' + givenDateDay);
if (todayIsInSameWeekOf(givenDate)) {
var currentMenuLink = $(this).children('a').attr('href');
var week = moment().isoWeek();
var numOfDay = moment().day();
fetch({
uri: 'http://www.esko-catering.de/' + currentMenuLink,
encoding: null
}, function (error, response, html) {
var bodyWithCorrectEncoding = iconv.decode(html, 'iso-8859-1');
if (!error) {
var $ = cheerio.load(bodyWithCorrectEncoding);
$('.boxEssenplanHolder').filter(function () {
var data = $(this);
var resArray = [];
data.children('.boxEssenplanInfo').each(function (i) {
var plan = $(this);
var day = [];
var weekday = plan.children('.boxDatum').children('h1').text();
day.push({
'weekday': weekday
});
var mealNames = [];
var prices = [];
plan.children('.boxGerichte').each(function (i) {
// we want the content of the paragraphs but w/o the spans inside!!!
var mealName = $(this).children('p').clone().children().remove().end().text().trim();
mealNames.push(mealName);
});
plan.children('.boxPreis').each(function (i) {
var price = $(this).children('p').text();
prices.push(price);
});
var meals = [];
// re-arrange
for (var x = 0; x < mealNames.length; x++) {
meals.push({
'meal': mealNames[x],
'price': prices[x]
});
}
day.push({
'meals': meals
});
resArray.push(day);
});
var load = resArray[numOfDay - 1];
var sDay = load[0]['weekday'];
var aMeals = load[1]['meals'];
var sLines = "Hallo, hier ist esko-bot. Heute ist " + sDay + " und wir servieren:\n\n\n";
for (var zz = 0; zz < aMeals.length; zz++) {
sLines += aMeals[zz].meal + " für " + aMeals[zz].price + "\n";
}
sLines += "\nGuten Appetit!";
var payload = {
"channel": "rostock",
"username": "******",
"text": sLines,
"icon_emoji": ":hamburger:"
};
fetch({
url: "https://hooks.slack.com/services/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",
method: "POST",
json: true,
headers: {
"content-type": "application/json",
},
body: payload
}, function (error, resp, body) {
if (error) {
console.log(error);
} else {
if (resp.statusCode === 200) {
console.log("Sending to slack succeeded!");
res.setHeader('Content-Type', 'text/html');
res.send('<html><head><meta charset="utf-8"></head><body>Done!\n <pre>' + sLines + '</pre></body></html>');
} else {
console.log("Statuscode: " + resp.statusCode);
res.setHeader('Content-Type', 'text/html');
res.send('<html><head><meta charset="utf-8"></head><body>Error: ' + resp.statusCode + '\n <pre>' + sLines + '</pre></body></html>');
}
}
});
});
}
});
}
});
});
}
}
});
});
});
}
});
cmd.stdout.on("data", function(data) {
// args.callback(null, iconv.decode(data, 'gbk'));
resultData += iconv.decode(data, 'gbk');
});
res.on('end',function(){
var result = iconv.decode(bufferHelper.toBuffer(),'GBK');
calback(result);
});
* @param {Function} [Iconv] node-iconv function
* @return {String} Decded string
*/
decode(buf, fromCharset, Iconv) {
fromCharset = charset.normalizeCharset(fromCharset || 'UTF-8');
if (/^(us-)?ascii|utf-8|7bit$/i.test(fromCharset)) {
return buf.toString('utf-8');
}
try {
if (typeof Iconv === 'function') {
let decoder = new Iconv(fromCharset, 'UTF-8');
return decoder.convert(buf).toString();
}
return iconv.decode(buf, fromCharset);
} catch (err) {
// enforce utf-8, data loss might occur
return buf.toString();
}
},
/**
* Convert a string from specific encoding to UTF-8 Buffer
*
* @param {String|Buffer} str String to be encoded
* @param {String} [fromCharset='UTF-8'] Source encoding for the string
* @return {Buffer} UTF-8 encoded typed array
*/
convert(data, fromCharset, Iconv) {
fromCharset = charset.normalizeCharset(fromCharset || 'UTF-8');
return __awaiter(this, void 0, void 0, function* () {
let data = yield send("http://bcy.net/coser/");
let ddd = yield zibPromise(data);
data = iconv.decode(ddd, 'utf-8');
console.log(data);
});
const toUtf8 = function(contentType, body) {
return iconv.decode(body, getCharset(contentType, body));
};
res.on('end',function(){
html = icon.decode(bufferHelper.toBuffer(),'GB2312');
info.infomation(html,function(data){
callback(data);
});
});
return fs.readFileAsync(oldPath).then(function(buf) {
return iconv.decode(buf, jschardet.detect(buf).encoding);
}).then(function(content) {
cmd.stdout.on('data', function(data) {
resultData += iconv.decode(data, args.settings.encoding);
stream.emit('data', resultData);
});
command.stdout.on('data', function(data) {
var buffer = new Buffer(data);
var str = iconv.decode(buffer, 'gbk');;
console.log(str)
onOutput(str);
});
dita.stderr.on('data', function (data) {
console.log(iconv.decode(data, consoleEncoding));
});
}, function(error, response, body) {
jobCount--;
if (!error && response.statusCode == 200) {
var url = response.request.href;
var keyUrl = url;
if (keyUrl[keyUrl.length-1] === '/') {
keyUrl = keyUrl.substring(0, keyUrl.length - 1);
}
var $ = cheerio.load(body);
var title = $('title').text();
if ($('meta').filter(function() {
return (/big5$/i).test($(this).attr('content'));
}).length > 0) {
//$ = cheerio.load(iconv.decode(new Buffer(body), "big5"));
//console.log("detect big5");
title = iconv.decode(title, "big5");
}
else {
title = iconv.decode(title, "utf8");
}
//console.log("title: " + title);
//console.log("Url: " + url);
//console.log("Cut: " + cut);
var parent = sitemap.find(keyUrl);
if (parent === undefined) {
sitemap.put(keyUrl, new Sitemap.node({title: title, url: keyUrl}, sitemap));
parent = sitemap.get(keyUrl);
cache[keyUrl] = parent;
} else {
parent.data = {title: title, url: keyUrl};
}
//if (parent.parent !== undefined)
// console.log("Children: " + parent.parent.childrenSize());
//console.log("Level: " + level);
if (cut !== true && level <= maxLevel) {
$('a').each(function(index, a) {
//console.log("Href: " + $(a).attr('href'));
var href = $(a).attr('href');
var tmpUrl = url;
tmpUrl = tmpUrl.replace(/\?.*$/, "");
tmpUrl = tmpUrl.replace(/\#.*$/, "");
if (validUrl(href)) {
var match1 = href.match(/\.\.\//g);
tmpUrl = tmpUrl.substring(0, tmpUrl.lastIndexOf('/'));
//var prefix = tmpUrl;
var prefix = URL.parse(tmpUrl).pathname;
if (match1) {
for (var i = 0; i < match1.length; i++) {
tmpUrl = tmpUrl.substring(0, tmpUrl.lastIndexOf('/'));
}
tmpUrl = tmpUrl + href.substring(match1.length * 3 - 1);
} else if (href.match(/^\.\//)) {
tmpUrl = tmpUrl + href.substring(1);
} else if (href.match(/^\//)) {
var parsedUrl = URL.parse(tmpUrl);
tmpUrl = parsedUrl.protocol + '//' + parsedUrl.hostname + href;
} else if (href.match(/^\/\//)) {
tmpUrl = URL.parse(tmpUrl).protocol + href;
} else if (href.match(/http:\/\//)) {
tmpUrl = href;
} else {
tmpUrl = tmpUrl + '/' + href;
}
//var tmpPrefix = tmpUrl.substring(0, tmpUrl.lastIndexOf('/'));
var tmpPrefix = URL.parse(tmpUrl).pathname;
tmpPrefix = tmpPrefix.substring(0, tmpPrefix.lastIndexOf('/'));
if (tmpUrl[tmpUrl.length-1] === '/') {
tmpUrl = tmpUrl.substring(0, tmpUrl.length - 1);
}
if (URL.parse(url).host === URL.parse(tmpUrl).host) {
if (cache[tmpUrl] !== undefined) {
parent.put(tmpUrl, cache[tmpUrl]);
} else {
jobCount++;
var node = new Sitemap.node({title: "", url: tmpUrl}, parent);
parent.put(tmpUrl, node);
cache[tmpUrl] = node;
process.nextTick(function(){
//console.log("Prefix: " + prefix);
//console.log("tmpPrefix: " + tmpPrefix);
crawl(tmpUrl, tmpPrefix !== prefix, level + 1);
//crawl(tmpUrl, tmpPrefix.substring(0, prefix.length) !== prefix, level + 1);
});
}
}
}
});
}
}
if (jobCount === 0) {
callback(sitemap);
}
});
+ `file using a ${ENCODING} encoding`, async (t) => {
t.timeoutAfter(30000) // 30s
const expectedFileName1 = 'attributes.csv'
const expectedFileName2 = 'products-to-attributes.csv'
/* eslint-disable max-len */
const expectedResult1 =
'name,type,attributeConstraint,isRequired,isSearchable,label.en,label.de,textInputHint,displayGroup\n'
+ 'breite,number,None,false,false,EN:žluťoučký kůň úpěl ďábelské ódy,DE:ě=ášéýřéý=čáěéžěížěé,SingleLine,Other\n'
const expectedResult2 = 'name,key,description,breite\n'
+ 'custom-product-type,productTypeKey,Some description - žluťoučký kůň úpěl ďábelské ódy,X\n'
/*
* TODO
* use only one string when Node 10 LTS support kicks in October 2018
* remove support for Node v6
* remove the if(process.version[1])
*/
let expectedEncoded1 =
'name,type,attributeConstraint,isRequired,isSearchable,label.en,label.de,textInputHint,displayGroup\n'
+ 'breite,number,None,false,false,EN:�lu�ou�k� k�� �p�l ��belsk� �dy,DE:�=�������=����������,SingleLine,Other\n'
if (process.version[1] >= 8)
expectedEncoded1 = 'name,type,attributeConstraint,isRequired,isSearchable,label.en,label.de,textInputHint,displayGroup\n'
+ 'breite,number,None,false,false,EN:�lu�ou�k� k�� �p�l ��belsk� �dy,DE:�=������=��������,SingleLine,Other\n'
const expectedEncoded2 = 'name,key,description,breite\n'
+ 'custom-product-type,productTypeKey,Some description - �lu�ou�k� k�� �p�l ��belsk� �dy,X\n'
/* eslint-enable max-len */
try {
await before()
await productTypeExport.run()
await new Promise((resolve) => {
glob(path.join(OUTPUT_FOLDER, '*'), (err, files) => {
t.equal(files.length, 2, 'files length is 2')
t.equal(
files[0].split('/').pop(),
expectedFileName1,
`file name is ${expectedFileName1}`,
)
t.equal(
files[1].split('/').pop(),
expectedFileName2,
`file name is ${expectedFileName2}`,
)
resolve()
})
})
const fileContent1 =
fs.readFileSync(path.join(OUTPUT_FOLDER, expectedFileName1))
const fileContent2 =
fs.readFileSync(path.join(OUTPUT_FOLDER, expectedFileName2))
const decoded1 = iconv.decode(fileContent1, ENCODING)
const decoded2 = iconv.decode(fileContent2, ENCODING)
t.equal(
fileContent1.toString(),
expectedEncoded1,
`Attributes should be encoded in ${ENCODING}`,
)
t.equal(
fileContent2.toString(),
expectedEncoded2,
`ProductType should be encoded in ${ENCODING}`,
)
t.equal(
decoded1,
expectedResult1,
'Attributes should decode back to utf8',
)
t.equal(
decoded2,
expectedResult2,
'ProductType should decode back to utf8',
)
t.end()
} catch (e) {
t.end(e)
}
})
res.on('end',function(){
data = Buffer.concat(chunks,size);
data = iconv.decode(data,'GBK');
console.log('data2: '+data);
callback(null,data,line);
});
request(settings, function(err, response, body) {
// If an error occurred
if (err) {
if (err.message === 'ETIMEDOUT')
return callback(new Error('timeout'));
if (~err.message.search(/getaddrinfo/))
return callback(new Error('host-not-found'));
return callback(err);
}
// Overloading
job.res.url = response.request.href;
job.res.body = body;
job.res.status = response.statusCode;
job.res.headers = response.caseless.dict;
// Assessing some things
var json = /json/.test(job.res.headers['content-type']);
// Handling encoding
var sourceEncoding = job.req.encoding || spider.options.encoding;
if (sourceEncoding === false) {
// Applying some heuristics
//-- 1) Content type header
var contentType = job.res.headers['content-type'];
if (contentType)
sourceEncoding = contentType.split('charset=')[1] || false;
//-- 2) HTTP equivalent or meta charset
if (!sourceEncoding && !json) {
var m = body.match(/<meta.*?charset=([^"']+)/);
sourceEncoding = m && m[1];
}
// Fallback
if (!sourceEncoding)
sourceEncoding = 'utf-8';
}
if (sourceEncoding) {
try {
job.res.body = iconv.decode(new Buffer(body), sourceEncoding);
}
catch (e) {
return callback(new Error('encoding-error'));
}
}
// Status error
if (response.statusCode >= 400) {
var error = new Error('status-' + (response.statusCode || 'unknown'));
error.status = response.statusCode;
return callback(error);
}
// JSON?
if (json) {
try {
job.res.body = JSON.parse(job.res.body);
return callback(null, job);
}
catch (e) {}
}
// Parsing
if (spider.scraperScript) {
var $ = cheerio.load(
job.res.body,
job.cheerio || spider.options.cheerio || {}
);
if (spider.synchronousScraperScript) {
try {
job.res.data = spider.scraperScript.call(spider, $, job);
}
catch (e) {
return callback(e);
}
return callback(null, job);
}
else {
try {
spider.scraperScript.call(spider, $, function(err, data) {
job.res.data = data;
return callback(err, data);
}, job);
}
catch (e) {
return callback(e);
}
}
}
else {
return callback(null, job);
}
});
res.on('end', function () {
var data = Buffer.concat(chunks, size);
data = iconv.decode(data, 'GBK');
resolve(data);
});
res.on('end', () => {
var html = iconv.decode(bufferHelper.toBuffer(), 'gb2312');
$ = cheerio.load(html);
callback($);
});
}, ep.done('main', function (res, buffer) {
return iconv.decode(buffer, 'gbk')
}))
res.on('end', function () {
//注意，此编码必须与抓取页面的编码一致，否则会出现乱码，也可以动态去识别
var val = iconv.decode(bufferHelper.toBuffer(), 'utf-8');
//callback(val);
});
function(err, data, stderr) {
var datau = iconv.decode(data, 'GBK');
datau = datau.toString();
logger.info('编译执行返回结果： ' + datau);
// 验证编译是否成功
file = '/export/home/qarelease/antbuild/' + file;
data = datau;
if (data.indexOf('BUILD SUCCESSFUL') > -1) {
logger.info("编译成功");
// 提交文件个数
var len = 0, filetxtlist, goallist, nofile
fs.readFile(file, 'utf-8', function(err, datatxt) {
if (err) {
logger.error("fs.readFile" + err);
} else {
filetxt = datatxt;
logger.info(datatxt);
filetxtlist = datatxt.toString().split('\n')
/* len = datatxt.toString().split('\n').length - 1;
logger.info('提交文件个数: ' + len)
*/
var find = "/YP2G_";
var reg = new RegExp(find,"g");//建立了一个正则表达式，也可以写为var reg=/is/g;
var count = datatxt.match(reg); //match则匹配返回的字符串,这是很正规的做法
var nofind= "#/YP2G_";
var noreg = new RegExp(nofind,"g");//建立了一个正则表达式，也可以写为var reg=/is/g;
var count1 = datatxt.match(noreg); //match则匹配返回的字符串,这是很正规的做法
if(count){
if(count1){
len=count.length-count1.length;
}
else{
len=count.length;
}
}
else{
len=0;
}
logger.info('提交文件个数count.length-count1.length: ' + len);
// 编译文件返回
var begin = '[处理成功的文件]:', end = '[无变化的文件]:';
goal = data.split(begin)[1].split(end)[0];
// 更新文件个数number
number = goal.substring(1, 3);
logger.info('更新文件个数: ' + number);
// 更新文件清单
goal = goal.substring(3);
goallist = goal.split('\n')
// 失败文件：
var beginf = '[无变化的文件]:', endf = '[失败的文件]';
nofile = data.split(beginf)[1].split(endf)[0];
// 未更新文件
nofile = nofile.substring(3);
if (number == len) {
// 判断个数相等
if (!filetxt==goal) {// 判断内容是否一直
for (var i = 0; i < len; i++) {
// 判断是否存在，存在就修改
for (var j = 0; j < number; j++) {
if (goallist[j].indexOf(filetxtlist[i])) {
filetxtlist[i] = 'ok'
}
}
}
// 返回未编译的代码清单
for (var i = 0; i < len; i++) {
if (filetxtlist[i] = 'ok') {
continue;
}
nofile = nofile + filetxtlist[i]
}
return callback(null, start, '编译成功,文件可能不一致,提交：'+len+'个 ，更新：'+number+'个 ', nofile,
filetxt,goal)
}
return callback(null, start, '编译成功,全部一致', '无', filetxt,goal)
} else {
return callback(null, start, '编译完成,文件可能不一致,提交：'+len+'个 ，更新：'+number+'个', nofile, filetxt,goal)
}
}//else
}); //fs
}// if
else {
if (data.indexOf('[处理成功的文件]: 0') > -1) {
start = false;
logger.warn("无文件进行编译，编译失败！！！！！！！");
return callback(null, start, '编译失败！！！！！！！，无文件编译', nofile, filetxt,'无')
} else {
start = false;
logger.warn("编译失败！！！！！！！");
var beg = '[失败的文件]:', en = 'TaskEnd';
nofile = data.split(beg)[1].split(en)[0];
nofile = nofile.substring(3);
return callback(null, start, '编译失败！！！！！！！！'+nofile, nofile, filetxt,'无')
}
}
});//$p
request({url:'http://top.baidu.com/category?c=10&fr=topindex',encoding:null},function(err,response,body){
//console.log(body);
var result=iconv.decode(body,'gbk'); //拿到原来是gbk的buffer转成utf-8
console.log(result);
})
cmd.stderr.on("data", function(data) {
// Print error message
console.log(iconv.decode(data, settings.encoding));
});
function main() {
var INPUT_CSV_PATH = process.argv[2];
var fileContentBuf = fs.readFileSync(INPUT_CSV_PATH);
var fileContent = iconv.decode(fileContentBuf, 'cp1250');
var locations = readJsonFile('location-fuzzy-map.json');
var locationsFuse = new Fuse(locations, {threshold: 0.4, keys: ['locationName']});
var coverImages = readJsonFile('cover-image-fuzzy-map.json');
var coverImagesFuse = new Fuse(coverImages, {threshold: 0.2, keys: ['eventName']});
csv.parseAsync(fileContent, {
comment: '#',
delimiter: ';',
'auto_parse': false,
trim: true
})
.then(function(eventsData) {
var rows = _.filter(_.tail(eventsData), row => !_.isEmpty(row[0]));
var rowCount = rows.length;
rows = _.uniqBy(rows, row => row[0]);
var diff = rowCount - rows.length;
console.error('Removed', diff, 'duplicate events from the source data');
var events = _.map(rows, row => {
var startTime = moment.tz(row[2] + ' ' + row[4], DATE_FORMAT, 'Europe/Helsinki');
// Start date is same as in the end, the end time might go to the next
// day too, so we have to take that into account.
// e.g. start is 21:00 and end is 01:00
var endTime = moment.tz(row[2] + ' ' + row[5], DATE_FORMAT, 'Europe/Helsinki');
if (endTime.diff(startTime) < 0) {
console.error('endTime', endTime.toISOString(), '<', startTime.toISOString());
endTime = endTime.add(1, 'day');
console.error('New endTime:', endTime.toISOString());
}
var event = {
name: row[0],
locationName: row[1],
startTime: startTime,
endTime: endTime,
description: row[7],
organizer: row[8],
contactDetails: row[9],
teemu: row[10].indexOf('Yes') > 0,
};
var results = locationsFuse.search(event.locationName);
if (_.isEmpty(results)) {
console.error('Searched for:', event.locationName);
throw new Error('Could not match event to location: ' + JSON.stringify(event));
}
event.location = {
latitude: results[0].latitude,
longitude: results[0].longitude
};
if (event.locationName !== results[0].locationName) {
console.error(
'Match',
event.locationName,
'\n ->',
results[0].locationName,
JSON.stringify(event.location)
);
}
results = coverImagesFuse.search(event.name);
if (_.isEmpty(results)) {
console.error('Searched for:', event.name);
throw new Error('Could not match event name to cover image: ' + JSON.stringify(event));
}
event.coverImage = 'https://storage.googleapis.com/wappuapp/assets/' +
results[0].coverImage;
return event;
});
var sortedEvents = _.sortBy(events, event => {
return event.startTime.unix();
});
console.error('\n\n\n');
console.log(JSON.stringify(sortedEvents, null, 2));
});
}
const decodeXml = (str) => {
const match = str.toString().match(/<\?xml[^>]+encoding="([\w-]+)"\?>/)
return match
? iconv.decode(str, match[1])
: str
}
request(url, function (err, res, body) {
var html = iconv.decode(body, 'gb2312')
console.log(html)
})
}, function (error, response, html) {
var bodyWithCorrectEncoding = iconv.decode(html, 'iso-8859-1');
if (!error) {
var $ = cheerio.load(bodyWithCorrectEncoding);
$('.boxEssenplanHolder').filter(function () {
var data = $(this);
var resArray = [];
data.children('.boxEssenplanInfo').each(function (i) {
var plan = $(this);
var day = [];
var weekday = plan.children('.boxDatum').children('h1').text();
day.push({
'weekday': weekday
});
var mealNames = [];
var prices = [];
plan.children('.boxGerichte').each(function (i) {
// we want the content of the paragraphs but w/o the spans inside!!!
var mealName = $(this).children('p').clone().children().remove().end().text().trim();
mealNames.push(mealName);
});
plan.children('.boxPreis').each(function (i) {
var price = $(this).children('p').text();
prices.push(price);
});
var meals = [];
// re-arrange
for (var x = 0; x < mealNames.length; x++) {
meals.push({
'meal': mealNames[x],
'price': prices[x]
});
}
day.push({
'meals': meals
});
resArray.push(day);
});
var load = resArray[numOfDay - 1];
var sDay = load[0]['weekday'];
var aMeals = load[1]['meals'];
var sLines = "Hallo, hier ist esko-bot. Heute ist " + sDay + " und wir servieren:\n\n\n";
for (var zz = 0; zz < aMeals.length; zz++) {
sLines += aMeals[zz].meal + " für " + aMeals[zz].price + "\n";
}
sLines += "\nGuten Appetit!";
var payload = {
"channel": "rostock",
"username": "******",
"text": sLines,
"icon_emoji": ":hamburger:"
};
fetch({
url: "https://hooks.slack.com/services/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",
method: "POST",
json: true,
headers: {
"content-type": "application/json",
},
body: payload
}, function (error, resp, body) {
if (error) {
console.log(error);
} else {
if (resp.statusCode === 200) {
console.log("Sending to slack succeeded!");
res.setHeader('Content-Type', 'text/html');
res.send('<html><head><meta charset="utf-8"></head><body>Done!\n <pre>' + sLines + '</pre></body></html>');
} else {
console.log("Statuscode: " + resp.statusCode);
res.setHeader('Content-Type', 'text/html');
res.send('<html><head><meta charset="utf-8"></head><body>Error: ' + resp.statusCode + '\n <pre>' + sLines + '</pre></body></html>');
}
}
});
});
}
});
return parser.readBuffer(dataLength, (data) => {
callback(iconv.decode(data, codepage));
});
cmd.stderr.on("data", function(data) {
// Print error message
console.log(iconv.decode(data, 'gbk'));
});
}).then(function(response){
if (!response || response.status !== 200) {
if (!response){
logger.log('warn/scraper', "No response from resource server at " + url);
} else {
logger.log('warn/scraper', "Status from resource server at " + url +
": " + response.status);
}
return citationPromise.then(function(citation){
return build520(cr);
});
} else {
var str; // String from decoded Buffer object
var defaultCT = 'utf-8'; // Default content-type
var contentType = exports.contentTypeFromResponse(response);
// Load html into cheerio object; if neccesary, determine
// content type from html loaded with default content-type, and
// then reload again if non-default content-type is obtained.
if (contentType){
// Content Type detected in response
try {
str = iconv.decode(response.body, contentType);
chtml = cheerio.load(str);
} catch (e){
logger.log('warn/scraper', e);
}
} else {
str = iconv.decode(response.body, defaultCT);
try {
chtml = cheerio.load(str);
contentType = exports.contentTypeFromBody(chtml);
// If contentType is scraped from body and is NOT the default
// CT already loaded, re-decode and reload into cheerio.
if (contentType && contentType!== defaultCT){
try {
str = iconv.decode(response.body, contentType);
chtml = cheerio.load(str);
} catch(e){
// On failure, defaults to loaded body with default CT.
logger.log('warn/scraper', e);
}
}
} catch(e){
logger.log('warn/scraper', e);
}
}
// If the html has been successfully loaded into cheerio, proceed.
if (chtml){
// Create initial citation, which returns citation
return citationPromise.then(function(citation){
return scraper.parseHTML(cr, chtml).then(
// Success handler for parseHTML
function(citation){
logger.log('debug/scraper', "Sucessfully scraped resource at " + url);
cr.response.responseCode = 200;
return cr;
},
// Rejection handler for parseHTML
function(){
logger.log('debug/scraper', "Failed to parse HTML of resource at " + url);
return build520(cr);
}
);
});
} else {
logger.log('debug/scraper', "Failed to scrape resource at " + url);
return citationPromise.then(function(citation){
return build520(cr);
});
}
}
},