exports.send = function(args) {
var client, options;
var uri, heirpart, authority, host, port, path, useProxy = false, proxyHost, proxyPort;
var isTls = args.uri.indexOf('https://') == 0;
uri = new URI(args.uri, false);
heirpart = uri.heirpart();
assert.ok(heirpart, 'URI [' + args.uri + '] is invalid');
authority = heirpart.authority();
assert.ok(authority, 'URI [' + args.uri + '] is invalid');
host = authority.host();
assert.ok(host, 'Host of URI [' + args.uri + '] is invalid');
port = authority.port() || (isTls ? 443 : 80);
assert.ok(port, 'Port of URI [' + args.uri + '] is invalid');
path = (heirpart.path().value || '') + (uri.querystring() || '');
if(args.config.proxy) {
var proxyConfig = args.config.proxy;
if (proxyConfig[host] && !proxyConfig[host].host) {
useProxy = false;
}
else if (proxyConfig[host] && proxyConfig[host].host) {
proxyHost = proxyConfig[host].host;
proxyPort = proxyConfig[host].port;
useProxy = true;
}
else if (proxyConfig['*']) {
proxyHost = proxyConfig['*'].host;
proxyPort = proxyConfig['*'].port;
useProxy = true;
}
}
options = {
host: useProxy ? proxyHost : host,
port: useProxy? proxyPort : port,
path: useProxy? uri.scheme() + '//' + host + path : path,
method: args.method,
headers: args.headers
};
client = isTls ? https : http;
// Send
args.httpReqTx = args.logEmitter.wrapEvent({
parent: args.parentEvent,
txType: 'QlIoHttpRequest',
txName: undefined,
message: JSON.stringify(options),
cb: args.cb
});
sendMessage(args, client, options, 0);
}
exports.send = function(args) {
var client, options;
var uri, heirpart, authority, host, port, path, useProxy = false, proxyHost, proxyPort;
var isTls = args.uri.indexOf('https://') == 0;
uri = new URI(args.uri, false);
heirpart = uri.heirpart();
assert.ok(heirpart, 'URI [' + args.uri + '] is invalid');
authority = heirpart.authority();
assert.ok(authority, 'URI [' + args.uri + '] is invalid');
host = authority.host();
assert.ok(host, 'Host of URI [' + args.uri + '] is invalid');
port = authority.port() || (isTls ? 443 : 80);
assert.ok(port, 'Port of URI [' + args.uri + '] is invalid');
path = (heirpart.path().value || '') + (uri.querystring() || '');
assert.ok(args.name, 'table name not specified');
if(args.config.proxy) {
var proxyConfig = args.config.proxy;
if (proxyConfig[host] && !proxyConfig[host].host) {
useProxy = false;
}
else if (proxyConfig[host] && proxyConfig[host].host) {
proxyHost = proxyConfig[host].host;
proxyPort = proxyConfig[host].port;
useProxy = true;
}
else if (proxyConfig['*']) {
proxyHost = proxyConfig['*'].host;
proxyPort = proxyConfig['*'].port;
useProxy = true;
}
}
options = {
host: useProxy ? proxyHost : host,
port: useProxy? proxyPort : port,
path: useProxy? uri.scheme() + '//' + host + path : path,
method: args.method,
headers: args.headers
};
client = isTls ? https : http;
// Avoid request backlog on any given socket.
client.globalAgent.maxSockets = 1000;
// Send
sendMessage(args, client, options, 0);
}
function sendOneRequest(args, resourceUri, params, holder, cb) {
var h, requestBody, mediaType = 'application/json', overrideStatus, client, isTls, options, auth,
clientRequest, template, start = Date.now();
var respData, respJson, uri, heirpart, authority, host, port, path, useProxy = false, proxyHost, proxyPort;
var httpReqTx = logUtil.wrapEvent(args.parentEvent, 'QlIoHttpRequest', null, cb);
var resource = args.resource;
var statement = args.statement;
var globalOpts = global.opts;
var emitter = args.emitter;
var conn = (globalOpts && globalOpts['connection']) ? globalOpts['connection'] : 'keep-alive';
h = {
'connection' : conn
};
var requestId = {
name : (globalOpts && globalOpts['request-id']) ? globalOpts['request-id'] : 'request-id',
value : params['request-id'] || resource.headers['request-id'] || httpReqTx.event.uuid
};
h[requestId.name] = requestId.value;
h['user-agent'] = 'ql.io/node.js';
// Clone headers - also replace any tokens
_.each(resource.headers, function(v, k) {
var compiled;
try {
compiled = strTemplate.parse(v);
v = compiled.format(params);
}
catch(e) {
// Ignore as we want to treat non-conformant strings as opaque
logUtil.emitWarning(httpReqTx.event, 'unable to parse header ' + v + ' error: ' + e.stack || e);
}
h[k.toLowerCase()] = v;
});
// appending Ip address to the request id header
h[requestId.name] += '!ql.io' + '!' + ip() + '[';
// Monkey patch headers
if(resource.monkeyPatch && resource.monkeyPatch['patch headers']) {
try {
h = patchHeaders(resourceUri, statement, params, h, resource.monkeyPatch['patch headers']);
}
catch(e) {
return cb(e);
}
}
// Perform authentication
if(resource.auth) {
resource.auth.auth(params, function(err, ack) {
if(err) {
return cb(err);
}
});
}
// For POST and PUT, if there is a template, process it
var body = {};
if(resource.monkeyPatch && resource.monkeyPatch['body template']) {
body = bodyTemplate(resourceUri, statement, params, h, resource.monkeyPatch['body template']);
}
if((resource.body || body) &&
(resource.method === 'post' || resource.method == 'put')) {
if(resource.body.type === 'application/x-www-form-urlencoded') {
try {
template = uriTemplate.parse(body.content || resource.body.content);
}
catch(err) {
global.opts.logger.warn(err);
return cb(err, null);
}
requestBody = formatUri(template, params, resource.defaults);
assert.ok(requestBody.length === 1, 'Body template processing resulted in an array. INTERNAL ERROR');
requestBody = requestBody[0];
}
else {
holder.statement = statement;
holder.params = params;
holder.body = args.request.body; // body received as part of the original request
requestBody = mustache.to_html(body.content || resource.body.content, holder);
}
if(resource.monkeyPatch && resource.monkeyPatch['patch body']) {
body = patchBody(resourceUri, statement, params, h, requestBody, resource.monkeyPatch['patch body']);
requestBody = body.content;
}
h['content-length'] = requestBody.length;
if(!h['content-type']) {
h['content-type'] = body.type || resource.body.type;
}
}
// TODO: Validate if everything required is set
// TODO: Remove unset params
// TODO: Local filtering
// Now try to get a representation of the resource
isTls = resourceUri.indexOf('https://') == 0;
uri = new URI(resourceUri, false);
heirpart = uri.heirpart();
assert.ok(heirpart, 'URI [' + resourceUri + '] is invalid')
authority = heirpart.authority();
assert.ok(authority, 'URI [' + resourceUri + '] is invalid')
host = authority.host();
assert.ok(host, 'Host of URI [' + resourceUri + '] is invalid')
port = authority.port() || (isTls ? 443 : 80);
assert.ok(port, 'Port of URI [' + resourceUri + '] is invalid')
path = (heirpart.path().value || '') + (uri.querystring() || '');
if (globalOpts && globalOpts.config && globalOpts.config.proxy) {
var proxyConfig = globalOpts.config.proxy;
if (proxyConfig[host] && !proxyConfig[host].host) {
useProxy = false;
}
else if (proxyConfig[host] && proxyConfig[host].host) {
proxyHost = proxyConfig[host].host;
proxyPort = proxyConfig[host].port;
useProxy = true;
}
else if (proxyConfig['*']) {
proxyHost = proxyConfig['*'].host;
proxyPort = proxyConfig['*'].port;
useProxy = true;
}
}
options = {
host: useProxy ? proxyHost : host,
port: useProxy? proxyPort : port,
path: useProxy? uri.scheme() + '//' + host + path : path,
method: resource.method || 'GET',
headers: h
};
client = isTls ? https : http;
// Emit
if(emitter) {
var uniqueId = uuid();
var packet = {
line: statement.line,
id: uniqueId,
uuid: httpReqTx.event.uuid,
method: options.method,
uri: resourceUri,
headers: [],
start: toISO(new Date()),
type: eventTypes.STATEMENT_REQUEST
};
if(requestBody) {
packet.body = requestBody;
}
_.each(h, function(v, n) {
packet.headers.push({
name: n,
value: v
});
});
emitter.emit(packet.type, packet);
}
clientRequest = client.request(options, function(res) {
res.setEncoding('utf8');
respData = '';
res.on('data', function (chunk) {
respData += chunk;
});
res.on('end', function() {
if(emitter) {
var packet = {
line: statement.line,
uuid: httpReqTx.event.uuid,
id: uniqueId,
status: res.statusCode,
headers: [],
time: new Date() - start,
body: respData,
type: eventTypes.STATEMENT_RESPONSE
};
_.each(res.headers, function(v, n) {
packet.headers.push({
name: n,
value: v
});
})
emitter.emit(eventTypes.STATEMENT_RESPONSE, packet);
if (res.headers[requestId.name]) {
emitter.emit(eventTypes.REQUEST_ID_RECEIVED, res.headers[requestId.name]);
}
else {
// Send back the uuid created in ql.io, if the underlying api
// doesn't support the request tracing or the table is not configured with
// the right name of the header.
emitter.emit(eventTypes.REQUEST_ID_RECEIVED, h[requestId.name]);
}
}
// TODO: Handle redirects
mediaType = sniffMediaType(mediaType, resource, statement, res, respData);
// TODO: Log level?
// TODO: For now, log verbose
logUtil.emitEvent(httpReqTx.event, resourceUri + ' ' +
sys.inspect(options) + ' ' +
res.statusCode + ' ' + mediaType.type + '/' + mediaType.subtype + ' ' +
sys.inspect(res.headers) + ' ' + (Date.now() - start) + 'msec');
// Parse
try {
if(!respData || /^\s*$/.test(respData)){
respJson = {};
}
else if(mediaType.subtype === 'xml') {
respJson = expat.toJson(respData, {object: true});
}
else if(mediaType.subtype === 'json') {
respJson = JSON.parse(respData);
}
else if(mediaType.type === 'text') {
// Try JSON
try {
respJson = JSON.parse(respData);
}
catch(e) {
try {
respJson = expat.toJson(respData, {object: true});
}
catch(e) {
e.body = respData;
return httpReqTx.cb(e);
}
}
}
}
catch(e) {
e.body = respData;
return httpReqTx.cb(e);
}
overrideStatus = res.statusCode;
if(resource.monkeyPatch && resource.monkeyPatch['patch status']) {
try {
overrideStatus = resource.monkeyPatch['patch status']({
status: res.statusCode,
headers: res.headers,
body: respJson || respData
})
}
catch(e) {
return httpReqTx.cb(e);
}
}
if(overrideStatus >= 200 && overrideStatus <= 300) {
if(respJson) {
if(resource.monkeyPatch && resource.monkeyPatch['patch response']) {
try {
respJson = resource.monkeyPatch['patch response']({
body: respJson
});
}
catch(e) {
return httpReqTx.cb(e);
}
}
// Projections
project.run(resource.resultSet, statement, respJson, function(filtered) {
return httpReqTx.cb(undefined, {
headers: {
'content-type': 'application/json'
},
body: filtered
});
});
}
else {
return httpReqTx.cb(undefined, {
headers: {
'content-type': mediaType
},
body: respData
});
}
}
else {
return httpReqTx.cb({
headers: {
'content-type': respJson ? 'application/json' : mediaType
},
body: respJson || respData
});
}
});
});
if(requestBody) {
clientRequest.write(requestBody);
}
clientRequest.on('error', function(err) {
logUtil.emitEvent(httpReqTx.event, 'error with uri - ' + resourceUri + ' - ' +
err.message + ' ' + sys.inspect(clientRequest, true, 10) + ' ' + (Date.now() - start) + 'msec');
err.uri = uri;
err.status = 502;
return httpReqTx.cb(err, undefined);
});
clientRequest.end();
}
function sendOneRequest(args, resourceUri, params, holder, cb) {
var h, requestBody, client, isTls, options, template;
var uri, heirpart, authority, host, port, path, useProxy = false, proxyHost, proxyPort;
var httpReqTx = logEmitter.wrapEvent(args.parentEvent, 'QlIoHttpRequest', null, cb);
var resource = args.resource;
var statement = args.statement;
var globalOpts = global.opts;
var emitter = args.emitter;
var conn = (globalOpts && globalOpts['connection']) ? globalOpts['connection'] : 'keep-alive';
h = {
'connection' : conn
};
var requestId = {
name : (globalOpts && globalOpts['request-id']) ? globalOpts['request-id'] : 'request-id',
value : params['request-id'] || resource.headers['request-id'] || httpReqTx.event.uuid
};
h[requestId.name] = requestId.value;
h['user-agent'] = 'ql.io/node.js';
// Clone headers - also replace any tokens
_.each(resource.headers, function(v, k) {
var compiled;
try {
compiled = strTemplate.parse(v);
v = compiled.format(params);
}
catch(e) {
// Ignore as we want to treat non-conformant strings as opaque
logEmitter.emitWarning(httpReqTx.event, 'unable to parse header ' + v + ' error: ' + e.stack || e);
}
h[k.toLowerCase()] = v;
});
// appending Ip address to the request id header
h[requestId.name] += '!ql.io' + '!' + ip() + '[';
// Monkey patch headers
if(resource.monkeyPatch && resource.monkeyPatch['patch headers']) {
try {
h = patchHeaders(resourceUri, statement, params, h, resource.monkeyPatch['patch headers']);
}
catch(e) {
return cb(e);
}
}
// Perform authentication
if(resource.auth) {
resource.auth.auth(params, function(err, ack) {
if(err) {
return cb(err);
}
});
}
// For POST and PUT, if there is a template, process it
var body = {};
if(resource.monkeyPatch && resource.monkeyPatch['body template']) {
body = bodyTemplate(resourceUri, statement, params, h, resource.monkeyPatch['body template']);
}
if((resource.body || body) &&
(resource.method === 'post' || resource.method == 'put')) {
if(resource.body.type === 'application/x-www-form-urlencoded') {
try {
template = uriTemplate.parse(body.content || resource.body.content);
}
catch(err) {
logEmitter.emitWarning(err);
return cb(err, null);
}
requestBody = formatUri(template, params, resource.defaults);
assert.ok(requestBody.length === 1, 'Body template processing resulted in an array. INTERNAL ERROR');
requestBody = requestBody[0];
}
else {
holder.statement = statement;
holder.params = params;
requestBody = mustache.to_html(body.content || resource.body.content, holder);
}
if(resource.monkeyPatch && resource.monkeyPatch['patch body']) {
body = patchBody(resourceUri, statement, params, h, requestBody, resource.monkeyPatch['patch body']);
requestBody = body.content;
}
h['content-length'] = requestBody.length;
if(!h['content-type']) {
h['content-type'] = body.type || resource.body.type;
}
}
// Now try to get a representation of the resource
isTls = resourceUri.indexOf('https://') == 0;
uri = new URI(resourceUri, false);
heirpart = uri.heirpart();
assert.ok(heirpart, 'URI [' + resourceUri + '] is invalid');
authority = heirpart.authority();
assert.ok(authority, 'URI [' + resourceUri + '] is invalid');
host = authority.host();
assert.ok(host, 'Host of URI [' + resourceUri + '] is invalid');
port = authority.port() || (isTls ? 443 : 80);
assert.ok(port, 'Port of URI [' + resourceUri + '] is invalid');
path = (heirpart.path().value || '') + (uri.querystring() || '');
if(globalOpts && globalOpts.config && globalOpts.config.proxy) {
var proxyConfig = globalOpts.config.proxy;
if (proxyConfig[host] && !proxyConfig[host].host) {
useProxy = false;
}
else if (proxyConfig[host] && proxyConfig[host].host) {
proxyHost = proxyConfig[host].host;
proxyPort = proxyConfig[host].port;
useProxy = true;
}
else if (proxyConfig['*']) {
proxyHost = proxyConfig['*'].host;
proxyPort = proxyConfig['*'].port;
useProxy = true;
}
}
options = {
host: useProxy ? proxyHost : host,
port: useProxy? proxyPort : port,
path: useProxy? uri.scheme() + '//' + host + path : path,
method: resource.method || 'GET',
headers: h
};
client = isTls ? https : http;
// Send
sendMessage(client, emitter, statement, httpReqTx, options, resourceUri, requestBody, h,
requestId, resource, args.xformers, 0);
}