UrlService.prototype.existsForType = function(params, cb) {
var url = params.url;
var type = params.type;
var id = params.id;
//validate required params
if (!url || !type) {
cb(new Error("The url and type parameters are required. URL=["+url+"] TYPE=["+type+"]"), false);
return;
}
//build pattern
if (url.charAt(0) === '/') {
url = url.substring(1);
}
if (url.charAt(url.length - 1) === '/') {
url = url.substring(0, url.length - 1);
}
var pattern = "^\\/{0,1}" + util.escapeRegExp(url) + "\\/{0,1}$";
//execute search
var where = {
url: new RegExp(pattern, 'g')
};
var dao = new pb.DAO();
dao.unique(type, where, id, function(err, isUnique) {
cb(err, !isUnique);
});
};
UrlService.prototype.existsForType = function(params, cb) {
var url = params.url;
var type = params.type;
var id = params.id;
var site = params.site;
//validate required params
if (!url || !type) {
return cb(new Error("The url and type parameters are required. URL=["+url+"] TYPE=["+type+"]"), false);
}
//build pattern
if (url.charAt(0) === '/') {
url = url.substring(1);
}
if (url.charAt(url.length - 1) === '/') {
url = url.substring(0, url.length - 1);
}
var pattern = "^\\/{0,1}" + util.escapeRegExp(url) + "\\/{0,1}$";
var where = {
url: new RegExp(pattern, "i")
};
if (site !== undefined) {
where[pb.SiteService.SITE_FIELD] = site;
}
this.siteQueryService.unique(type, where, id, function(err, isUnique) {
cb(err, !isUnique);
});
};
app.post('/blacklist', function(req, res) {
var url = req.body.site;
if(url[4] === 's'){
url = url.substring(8);
if (url.slice(-1) === '/'){
url = url.substring(0,url.length-1);
}
url += ':443';
}
blacklist.add(url);
res.json(Array.from(blacklist));
});
return function (params, callback) {
var url = buildurl(schema.rootUrl + schema.servicePath + method.path);
var parameters = {
options: {
url: url.substring(1, url.length - 1),
method: method.httpMethod
},
params: params,
requiredParams: method.parameterOrder || [],
pathParams: getPathParams(method.parameters),
context: context
};
if (method.mediaUpload && method.mediaUpload.protocols &&
method.mediaUpload.protocols.simple &&
method.mediaUpload.protocols.simple.path) {
var mediaUrl = buildurl(
schema.rootUrl +
method.mediaUpload.protocols.simple.path
);
parameters.mediaUrl = mediaUrl.substring(1, mediaUrl.length - 1);
}
return createAPIRequest(parameters, callback);
};
gpii.dataSource.URL.handle = function (options, directModel, callback, model) {
// Expand URL.
var url = options.urlResolver.resolve(directModel),
// Test whether the URL is to a local file.
isFileProtocol = gpii.dataSource.isFileProtocol(url),
func = gpii.dataSource.URL.handle.url,
oldCallback = callback,
parse = options.modelParser.parse;
if (isFileProtocol) {
url = url.substring(7);
func = gpii.dataSource.URL.handle.fs;
}
options.fireOnError = options.errback.events.onError.fire;
if (options.callbackWrapper) {
options.fireOnError = options.callbackWrapper.wrap(options.fireOnError);
parse = options.callbackWrapper.wrap(options.modelParser.parse);
}
callback = function (data) {
data = parse(data);
if (data && data.isError) {
options.fireOnError(data);
return;
}
oldCallback(data);
};
// If model exists and is an object - stringify it.
model = options.modelParser.stringify(model);
func.apply(null, [url, options, callback, model]);
};
app.use('*', function (req, res) {
var url = req.baseUrl;
console.log('baseUrl: %s', req.baseUrl);
var index = url.indexOf("?");
if (index > -1) {
url = url.substring(0, index);
}
if (url.indexOf(".png") > -1) {
res.setHeader('Content-Type', 'image/png');
} else if (url.indexOf(".jpg") > -1) {
res.setHeader('Content-Type', 'image/jpg');
} else if (url.indexOf(".json") > -1) {
res.setHeader('Content-Type', 'application/json');
} else {
res.setHeader('Content-Type', 'text/html');
}
fs.readFile(getFilePath(url), (err, data) => {
if (err) {
res.statusCode = 503;
res.end("<h1>503:" + err + "</h1>");
console.error(err);
} else {
res.end(data);
}
});
});
function Request( request ) {
var url = request.url;
var questionIndex = url.indexOf("?");
this.method = request.method;
this.nodeRequest = request;
this.headers = request.headers;
if(questionIndex > -1){
this.pathInfo = url.substring(0, questionIndex);
this.queryString = url.substring(questionIndex + 1);
}
else{
this.pathInfo = url;
this.queryString = "";
}
if(this.method != "GET"){ // optimize GET
this.body = new Input( request );
}
}
exports.proxy = function (req, res) {
var url = req.query.q || req.params.url;
if (url.substring(0, 4) !== 'http') {
url = 'http://' + url;
}
proxy.get(url, function (error, dom) {
if (error) {
res.writeHead(500, {'Content-Type': "application/json"});
res.end(JSON.stringify(error));
} else {
res.writeHead(200, {'Content-Type': "text/html"});
res.end(JSON.stringify(dom));
}
});
};
return function (params, callback) {
const url = generator_utils_1.default.buildurl(schema.rootUrl + schema.servicePath + method.path);
const parameters = {
options: {
url: url.substring(1, url.length - 1),
method: method.httpMethod
},
params: params,
requiredParams: method.parameterOrder || [],
pathParams: getPathParams(method.parameters),
context: context,
mediaUrl: null
};
if (method.mediaUpload && method.mediaUpload.protocols &&
method.mediaUpload.protocols.simple &&
method.mediaUpload.protocols.simple.path) {
const mediaUrl = generator_utils_1.default.buildurl(schema.rootUrl +
method.mediaUpload.protocols.simple.path);
parameters.mediaUrl = mediaUrl.substring(1, mediaUrl.length - 1);
}
return apirequest_1.default(parameters, callback);
};
app.get("/site/*", function(req, res) {
var url = req.url;
// cuts off the /s/
var filepath = url.substring("/site/".length);
// gets the sitename (at the beginning of the filepath)
var sitename = filepath.split("/")[0];
// gets the real filepath, after sitename
var realfilepath = filepath.substring(sitename.length);
if(realfilepath === "/" || realfilepath === "") {
res.redirect("./index.html");
return;
}
NameSchemaModel.findOne({name : sitename}, function(err, blob) {
var headpath = blob.filePath;
var access_token = blob.token;
//console.log(access_token);
var geturl = 'https://api-content.dropbox.com/1/files/dropbox' + headpath + "" + realfilepath;
ext_arr = realfilepath.split(".");
ext = ext_arr[ext_arr.length - 1];
if(ext === "html" || ext === "js") {
request.get(geturl, {
headers: { Authorization: 'Bearer ' + access_token}},
function(error, response, body) {
res.send(body);
});
} else {
// link 'em
geturl = 'https://api.dropbox.com/1/media/dropbox' + headpath + realfilepath;
request.post(geturl, {headers: { Authorization: 'Bearer ' + access_token}},
function(error, response, body) {
if(error){throw error}
res.redirect(JSON.parse(body).url);
});
}
});
});
return function(req, res, next) {
var reqHost = req.headers.host.split(':')[0];
//远程请求的域名不能和访问域名一致，否则会陷入请求循环。
if(reqHost === param.host){
return;
}
var url = path.resolve(req.url.replace(/http:\/\/.+?\//,'/'));//兼容windows,windows平台下取得的req.url带http://部分
var prefix = url.indexOf(param.servlet + '?');
//不包含combo的servlet，认为是单一文件
if(prefix === -1){
//combo不处理html文件，但是需要接管其他资源
if(!fileReg.test(url)) {
next();
return;
}
cosoleResp('Need', url);
var filteredUrl = filterUrl(url);
res.setHeader('Content-Type', mime.lookup(filteredUrl.split('?')[0])+';charset='+param.charset);
var singleFileContent = readFromLocal(filteredUrl);
if(singleFileContent){
res.end(singleFileContent);
return;
}
var fileName = crypto.createHash('md5').update(reqHost+url).digest('hex');
var cachedFile = readFromCache(filteredUrl, fileName);
if(cachedFile){
res.end(cachedFile);
return;
}
//本地没有，从服务器获取
var requestOption = buildRequestOption(url, req);
http.get(requestOption, function(resp) {
var buffs = [];
if(resp.statusCode !== 200){
var headerHost = '';
if(requestOption.headers && requestOption.headers.host)
{
headerHost = requestOption.headers.host;
}
cosoleResp('Not found', requestOption.host + requestOption.path + ' (host:'+ reset + yellow + headerHost + reset + ')');
res.writeHead(404);
res.end('File ' + requestOption.host + requestOption.path + ' not found.');
return;
}
resp.on('data', function(chunk) {
buffs.push(chunk);
});
resp.on('end', function() {
var buff = joinbuffers(buffs);
//fix 80% situation bom problem.quick and dirty
if(buff[0] === 239 && buff[1] === 187 && buff[2] === 191) {
buff = buff.slice(3, buff.length);
}
if(isBinFile(filteredUrl)){
var binfileName = crypto.createHash('md5').update(reqHost+requestOption.path).digest('hex');
cacheFile(binfileName, buff);
cosoleResp('Remote', requestOption.host + requestOption.path);
res.end(buff);
return;
}
cosoleResp('Remote', requestOption.host + requestOption.path);
var charset = isUtf8(buff) ? 'utf8' : 'gbk';
var longestMatchPos = longgestMatchedDir(filteredUrl);
//允许为某个url特别指定编码
var outputCharset = param.charset;
if(longestMatchPos){
if(param.urlBasedCharset && param.urlBasedCharset[longestMatchPos]){
outputCharset = param.urlBasedCharset[longestMatchPos];
}
}
var singleFileContent = adaptCharset(buff, outputCharset, charset);
var fileName = crypto.createHash('md5').update(reqHost+url).digest('hex');
cacheFile(fileName, buff, charset);
res.end(singleFileContent );
return;
});
}).on('error',function(e){
debugInfo('Networking error:' + e.message);
res.writeHead(404, { 'Content-Type': 'text/html;charset=utf-8'});
res.end('404 Error, File not found.');
return;
});
return;
}
cosoleResp('Need', url);
prefix = url.substring(0, prefix);
var files = url.substring(prefix.length + param.servlet.length + 1, url.length);
files = files.split(param.seperator, 1000);
var reqArray = [];
var prevNeedHttp = false ;//为循环做准备，用来判定上次循环的file是否需要通过http获取
var needHttpGet = '';
for(var i = 0, len = files.length; i < len; i++){
var file = files[i];
//combo URL有时候会多一个逗号
if(file === "") continue;
var fullPath = filterUrl(prefix + files[i]);
if(i === 0 ){
res.setHeader('Content-Type', mime.lookup(fullPath.split('?')[0])+';charset='+param.charset);
}
var fileContent = readFromLocal(fullPath);
if(!fileContent){
if(prevNeedHttp){
needHttpGet += ',' + file;
continue;
}
prevNeedHttp = true;
needHttpGet = file;
continue;
}
if(prevNeedHttp){
reqArray.push({file: needHttpGet, ready: false});
}
prevNeedHttp = false;
reqArray.push({file: file, content: fileContent, ready: true});
}
if(prevNeedHttp){
reqArray.push({file: needHttpGet, ready:false});
}
var reqPath = prefix + param.servlet + '?';
for(var i = 0, len = reqArray.length; i < len; i++){
if(reqArray[i].ready){
continue;
}
var cacheName = crypto.createHash('md5').update(reqHost+reqArray[i].file).digest('hex');
var cachedContent = readFromCache(reqArray[i].file, '/' + cacheName);
if(cachedContent){
reqArray[i].content = cachedContent;
reqArray[i].ready = true;
continue;
}
(function(id) {
var requestPath = reqPath + reqArray[id].file;
var requestOption = buildRequestOption(requestPath, req);
http.get(requestOption, function(resp) {
if(resp.statusCode !== 200){
var headerHost = '';
if(requestOption.headers && requestOption.headers.host)
{
headerHost = requestOption.headers.host;
}
cosoleResp('Not found', requestOption.host + reqPath + reqArray[id].file + '('+ yellow +'host:'+ headerHost + reset +')');
reqArray[id].ready = true;
reqArray[id].content = 'File '+ reqArray[id].file +' not found.';
sendData();
return;
}
var buffs = [];
resp.on('data', function(chunk) {
buffs.push(chunk);
});
resp.on('end', function() {
cosoleResp('Remote', requestOption.host + reqPath + reqArray[id].file);
reqArray[id].ready = true;
var buff = joinbuffers(buffs);
//fix 80% situation bom problem.quick and dirty
if(buff[0] === 239 && buff[1] === 187 && buff[2] === 191) {
buff = buff.slice(3, buff.length);
}
var fileName = crypto.createHash('md5').update(reqHost+reqArray[id].file).digest('hex');
var charset = isUtf8(buff) ? 'utf8' : 'gbk';
reqArray[id].content = adaptCharset(buff, param.charset, charset);
cacheFile('/' + fileName, buff, charset);
sendData();
});
}).on('error',function(e){
reqArray[id].ready = true;
debug('Networking error:' + e.message);
});
})(i);
}
var sendData = function(){
for(var j = 0, len = reqArray.length; j < len; j++){
if(reqArray[j].ready === false){
return;
}
}
reqArray.forEach(function(reqNode){
res.write(reqNode.content);
});
res.end();
}
//如果全部都在本地可以获取到，就立即返回内容给客户端
sendData();
}