Client.fetch = function (url, options, callback) {
url = url.split("#")[0]; //Remove any anchor fragments
const http_options = {
timeout: 20000,
headers: {
accept: '*/*',
'user-agent': random_ua.generate()
},
followRedirects: false
};
var _options = {
title: true,
description: true,
type: true,
url: true,
siteName: true,
charset: true,
image: true,
meta: true,
images: true,
links: true
};
if (typeof options === 'function') {
callback = options;
} else if (typeof options === 'object') {
_.merge(http_options, options.http || {});
_.merge(_options, options.flags || {});
}
if (url === undefined || url === "") {
if (callback !== undefined) {
callback("Invalid URL", (url || ""));
}
return;
}
let redirectCount = 0;
const text = function () {
const headReq = rest.head(url);
const getReq = rest.get(url);
if (http_options.timeout) {
getReq.timeout(http_options.timeout);
headReq.timeout(http_options.timeout);
}
if (http_options.headers) {
getReq.set(http_options.headers);
headReq.set(http_options.headers);
}
headReq.end(function (err, headResult) {
const type = headResult.statusType;
if (err && err.timeout) {
return callback('Timeout');
}
if (!headResult) {
return callback(err);
}
if (type === 3) {
redirectCount++;
if (redirectCount > 5) {
return callback("Too many redirects");
}
url = URI.resolve(url, headResult.headers.location);
return text();
} else if (type > 5) {
return callback(headResult.statusCode);
}
let charSet = headResult.charset || 'utf8';
if (charSet === 'MS949') {
charSet = 'cp949';
}
return getReq
.charset(charSet)
.end((err, response) => {
if (err) {
return callback(err);
}
const result = response.text;
const meta = parseMeta(url, _options, result);
return callback(null, meta);
});
});
};
text();
};
Client.fetch = function(url, options, callback) {
if (url === undefined || url === "") {
if (callback !== undefined) {
callback("Invalid URL", (url || ""));
}
return;
}
var random_ua = require('modern-random-ua');
var http_options = {
timeout: 20000,
headers: {
'Accept': '*/*',
'User-Agent': random_ua.generate()
},
followRedirects: false
};
var _options = {
title: true,
description: true,
type: true,
url: true,
siteName: true,
charset: true,
image: true,
meta: true,
images: true,
links: true
};
if (typeof options === 'function') {
callback = options;
} else if (typeof options === 'object') {
_.merge(http_options, options.http || {});
_.merge(_options, options.flags || {});
}
var redirectCount = 0;
if (url.slice(-4) === ".pdf") {
var pdf = function() {
rest.head(url, http_options).on('complete', function(result, response) {
if (result instanceof Error) {
callback(result);
} else {
if (response.statusCode === 200) {
var meta = parseMeta(url, _options, result);
callback(null, meta);
}
}
}).on('3XX', function(data, res) {
redirectCount++;
if (redirectCount > 5) {
return callback("Too many redirects");
}
url = res.headers.location;
return pdf();
}).on('timeout', function() {
callback('Timeout');
});
};
pdf();
} else {
var text = function() {
rest.get(url, http_options).on('complete', function(result, response) {
if (result instanceof Error) {
callback(result);
} else {
if (response.statusCode === 200) {
var meta = parseMeta(url, _options, result);
callback(null, meta);
}
}
}).on('3XX', function(data, res) {
redirectCount++;
if (redirectCount > 5) {
return callback("Too many redirects");
}
url = res.headers.location;
return text();
}).on('timeout', function() {
callback('Timeout');
});
};
text();
}
};