AbstractZeroMQ.prototype.startAbstract = function(callback) {
logger.info('Start output to', this.to(), 'using ZeroMQ', zmq.version);
this.socket = zmq.socket('push');
if (this.zmq_high_watermark) {
logger.info('Setting high watermark on ZeroMQ socket', this.zmq_high_watermark);
if (zmq.version.match(/^2\./)) {
this.socket.setsockopt(zmq.ZMQ_HWM, parseInt(this.zmq_high_watermark, 10));
}
else {
this.socket.setsockopt(zmq.ZMQ_SNDHWM, parseInt(this.zmq_high_watermark, 10));
}
}
this.target.split(',').forEach(function(address) {
this.socket.connect(address);
}.bind(this));
if (this.zmq_check_interval) {
this.check_interval_id = setInterval(function() {
this.check();
}.bind(this), this.zmq_check_interval);
}
this.on_alarm = false;
callback();
};
this.sig_listener = function() {
if (this.closed_inputs) {
logger.info('Starting inputs plugins');
this.start_inputs();
}
else {
logger.info('Closing inputs plugins');
this.close_inputs();
}
}.bind(this);
fs.read(this.fdTailer.fd, buffer, 0, last_data.length, this.fdTailer.current_index - last_data.length, function(err, bytesRead, buffer) {
if (err) {
return this.emit('error', err);
}
if (bytesRead === last_data.length && last_data === buffer.toString(this.buffer_encoding, 0, last_data.length)) {
logger.info('Event changed received, but no data change and last data match', this.filename, 'fd', this.fdTailer.fd);
}
else {
logger.info('Event changed received, but no data change and last data does not match.', 'Restarting reading', this.filename, 'at 0 fd', this.fdTailer.fd);
this.restart(0);
}
}.bind(this));
fs.open(this.filename, 'r', function(err, fd) {
this.open_in_progress = false;
if (err) {
this.last_open_failed = true;
return callback(err);
}
this.to_be_processed = '';
// Some data about file in db_file ?
if (file_status[this.filename] && file_status[this.filename].last_data && file_status[this.filename].index) {
var last_data = file_status[this.filename].last_data;
// Enough data to check last_data ?
if (file_status[this.filename].index >= last_data.length && stats.size > last_data.length) {
var buffer = new Buffer(last_data.length);
fs.read(fd, buffer, 0, last_data.length, file_status[this.filename].index - last_data.length, function(err, bytesRead, buffer) {
if (err) {
return callback(err);
}
if (bytesRead === last_data.length && last_data === buffer.toString(this.buffer_encoding, 0, last_data.length)) {
logger.info('Start from last read index', this.filename, 'at', file_status[this.filename].index, 'fd', fd);
this.fdTailer = new FdTailer(fd, file_status[this.filename].index, this.buffer_encoding, this.buffer_size, stats.isFIFO(), this);
this.fdTailer.read();
}
else {
logger.info('Have last read index, but last data are not correct.', 'Start reading', this.filename, 'at end fd', fd);
this.fdTailer = new FdTailer(fd, stats.size, this.buffer_encoding, this.buffer_size, stats.isFIFO(), this);
this.fdTailer.read();
}
callback();
}.bind(this));
}
else {
logger.info('Have last read index, but file is too small.', 'Start reading', this.filename, 'at end fd', fd);
this.fdTailer = new FdTailer(fd, stats.size, this.buffer_encoding, this.buffer_size, stats.isFIFO(), this);
this.fdTailer.read();
callback();
}
}
// No data about file, starting normally
else {
if (start_index === undefined) {
logger.info('Start reading', this.filename, 'at end', 'fd', fd);
this.fdTailer = new FdTailer(fd, stats.size, this.buffer_encoding, this.buffer_size, stats.isFIFO(), this);
}
else {
logger.info('Start reading', this.filename, 'at', start_index, 'fd', fd);
this.fdTailer = new FdTailer(fd, start_index, this.buffer_encoding, this.buffer_size, stats.isFIFO(), this);
this.fdTailer.read();
}
callback();
}
}.bind(this));
exports.unregister = function(id) {
logger.info('Remove watcher on dir', id.dir);
current[id.dir].removeListener('change', id.callback);
if (current[id.dir].listeners('change').length === 0) {
logger.info('Removing empty listener on', id.dir);
if (platform === 'darwin') {
current[id.dir].stop();
}
else {
current[id.dir].close();
}
delete current[id.dir];
}
};
MonitoredFile.prototype.close = function(callback) {
var close_old_fd_tailers = function() {
if (this.oldFdTailers.length === 0) {
return callback();
}
clearTimeout(this.oldFdTailers[0].id);
this.oldFdTailers[0].f(close_old_fd_tailers);
}.bind(this);
logger.info('Closing monitoring for', this.filename);
if (this.dir_watcher) {
logger.debug('Closing directory monitoring for', this.dir);
this.dir_watcher.close();
this.dir_watcher = undefined;
}
if (this.fdTailer) {
this.fdTailer.close(function(last_data) {
file_status[this.filename] = {
last_data: last_data,
index: this.fdTailer.current_index,
};
close_old_fd_tailers();
}.bind(this));
}
else {
close_old_fd_tailers();
}
};
this.close_outputs(function() {
logger.info('Closing agent');
if (this.sig_listener) {
sig_listener.removeListener('SIGUSR1', this.sig_listener);
}
return callback();
}.bind(this));
InputSQS.prototype.start = function(callback) {
this.polling_delay = parseInt(this.polling_delay, 10);
this.closed_callback = undefined;
logger.info('Creating AWS SQS Input from', this.aws_host, 'polling delay', this.polling_delay);
this.waitMessage();
callback();
};
function loadPatterns(callback) {
var _leave = function(err) {
loaded = true;
onload_callbacks.forEach(function(cb) {
setImmediate(function() {
cb(err);
});
});
onload_callbacks = [];
};
if (loaded) {
return callback();
}
onload_callbacks.push(callback);
if (onload_callbacks.length > 1) {
return;
}
logger.info('Loading grok patterns');
patterns_loader.loadGrokPatterns(function(err, lines) {
if (err) {
return _leave(err);
}
var patterns = processLines(lines);
logger.info('Grok patterns loaded from patterns directories', Object.keys(patterns).length);
global_patterns = patterns;
_leave();
});
}
MonitoredFile.prototype.watch = function(callback) {
try {
this.dir = path.dirname(this.filename);
var basename = path.basename(this.filename);
logger.info('Watching dir', this.dir, 'for file', basename);
this.dir_watcher = directory_watcher.register(this.dir, function(event, filename) {
logger.debug('Event received for', this.filename, ':', event, filename);
if (event === 'rename' && basename === filename) {
this.emit('renamed');
}
else if (event === 'change' && basename === filename) {
this.emit('changed');
}
else if (event === 'change') {
this.emit('other_changed', filename);
}
else {
logger.debug('Unprocessed event while monitoring', this.filename, ':', event, filename);
}
}.bind(this));
callback();
}
catch (err) {
logger.error('Unable to monitor', this.dir, ':', err);
callback(err);
}
};
detector.on('exists', function(dir, newly_created) {
logger.info('Parent directory exists', dir, 'for reading', this.file);
fs.readdir(dir, function(err, l) {
if (err) {
return this.emit('error', new Error('Error while reading ' + dir + ' : ' + err));
}
l.forEach(function(x) {
this.monitorFile(dir, x, newly_created ? 0 : this.start_index);
}.bind(this));
try {
if (this.monitored_files[dir]) {
return;
}
var dir_watcher = directory_watcher.register(dir, function(event, filename) {
this.monitorFile(dir, filename, 0);
}.bind(this));
this.monitored_files[dir] = {
close: function(callback) {
directory_watcher.unregister(dir_watcher);
callback();
}
};
}
catch(err) {
return this.emit('error', new Error('Error while reading ' + dir + ' : ' + err));
}
}.bind(this));
}.bind(this));
InputUnix.prototype.start = function(callback) {
logger.info('Start listening on unix socket', this.socket);
this.server = net.createServer(function(c) {
c.on('data', function(data) {
this.unserialize_data(data, function(parsed) {
this.emit('data', parsed);
}.bind(this), function(data) {
this.emit('data', {
'message': data.toString().trim(),
'path': this.socket,
'type': this.type,
});
}.bind(this));
}.bind(this));
c.on('error', function(err) {
this.emit('error', err);
}.bind(this));
}.bind(this));
this.server.on('error', function(err) {
this.emit('error', err);
}.bind(this));
this.server.listen(this.socket, callback);
};
function RedisConnectionManager(host, port) {
events.EventEmitter.call(this);
this.host = host;
this.port = port;
logger.info('Connecting to Redis', this.host + ':' + this.port);
this.client = redis.createClient(this.port, this.host);
this.end_callback = function() {
logger.info('Redis connection lost to ' + this.host + ':' + this.port);
}.bind(this);
this.client.on('end', function() {
this.end_callback();
}.bind(this));
this.client.on('error', function(err) {
this.emit('error', err);
}.bind(this));
this.client.on('connect', function() {
logger.info('Connected to Redis', this.host + ':' + this.port);
this.emit('connect', this.client);
}.bind(this));
}
RedisConnectionManager.prototype.quit = function(callback) {
logger.info('Closing connection to Redis', this.host + ':' + this.port);
this.emit('before_quit', this.client);
this.end_callback = callback;
this.client.quit();
this.client = undefined;
};
var start_streaming = function() {
log.info(ctx.from + 'Starting streaming', ctx.directory);
copyHeadersIfExists(headers_to_copy_from_disk, ctx.result_headers, function(k, v) {response.setHeader(k, v)});
fs.open(ctx.filename, 'r', function(err, file_fd) {
if (err) {
response.statusCode = 500;
response.end();
log.error(ctx.from + 'Unable to open file for reading', ctx.filename, err);
}
fd = file_fd;
log.debug(ctx.from + 'File opened', ctx.filename);
ctx.events.once('data', read_data);
});
ctx.events.on('end', function(size, filename) {
fs.close(fd);
stop = true;
if (current != size) {
log.info(ctx.from + 'Have sent to client', current, 'expected', size, ', getting data from', filename, 'request', ctx.directory);
fs.open(filename, 'r', function(err, fd) {
if (err) {
log.error(ctx.from + 'Unable to open file', filename, err);
return;
}
var to_be_read = size - current;
var buffer_in = new Buffer(64 * 1024);
var r = function() {
fs.read(fd, buffer_in, 0, buffer_in.length, current, function(err, bytesRead, buffer_in) {
if (err) {
log.error(ctx.from + 'Error reading', filename, err);
return;
}
to_be_read -= bytesRead;
current += bytesRead;
log.debug(ctx.from + 'Sent', bytesRead, 'from real file, current', current, 'for', ctx.directory);
var buffer_out = new Buffer(bytesRead);
buffer_in.copy(buffer_out, 0, 0, bytesRead);
response.write(buffer_out);
if (to_be_read == 0) {
log.info(ctx.from + 'All data sent to client', ctx.directory);
response.end();
fs.close(fd);
return;
}
if (bytesRead != buffer_in.length) {
log.error(ctx.from + 'Wrong reading length', bytesRead, 'exptected', buffer_in.length, 'remaining', to_be_read, 'reading position', current - bytesRead);
return;
}
r();
})
};
r();
})
}
else {
log.info(ctx.from + 'All data has been sent to client from temp file', ctx.directory)
fs.close(fd);
response.end();
}
});
};
AbstractUdp.prototype.startAbstract = function(callback) {
logger.info('Start output to ' + this.to());
this.socket = dgram.createSocket('udp4');
callback();
};
InputSQS.prototype.close = function(callback) {
logger.info('Closing AWS SQS Input from', this.aws_host, 'waiting end of polling');
this.closed_callback = function() {
logger.info('Closed AWS SQS Input from', this.aws_host);
callback();
};
};
InputFile.prototype.start = function(callback) {
logger.info('Start input on file', this.file);
if (this.start_index) {
this.start_index = parseInt(this.start_index, 10);
}
this.filter = file_filter.create(path.basename(this.file));
var parent = path.dirname(this.file);
this.detector = new directory_detector.DirectoryDetector();
this.detector.on('exists', function() {
logger.info('Parent directory exists', parent, 'for reading', this.file);
fs.readdir(parent, function(err, l) {
if (err) {
return this.emit('error', err);
}
l.forEach(function(x) {
this.monitorFile(parent, x, this.start_index);
}.bind(this));
}.bind(this));
this.dir_watcher = directory_watcher.register(parent, function(event, filename) {
this.monitorFile(parent, filename, 0);
}.bind(this));
}.bind(this));
this.monitored_files = {};
this.detector.start(parent, callback);
};
InputUdp.prototype.afterLoadConfig = function(callback) {
logger.info('Start listening on udp', this.host + ':' + this.port);
this.configure_unserialize(this.unserializer);
this.server = dgram.createSocket('udp4');
this.server.on('message', function(data, remote) {
this.unserialize_data(data, function(parsed) {
this.emit('data', parsed);
}.bind(this), function(data) {
this.emit('data', {
'message': data.toString().trim(),
'host': remote.address,
'udp_port': this.port,
'type': this.type,
});
}.bind(this));
}.bind(this));
this.server.on('error', function(err) {
this.emit('init_error', err);
}.bind(this));
this.server.once('listening', callback);
this.server.bind(this.port, this.host);
}
InputZeroMQ.prototype.start = function(callback) {
logger.info('Start listening on zeromq', this.target, 'using ZeroMQ', zmq.version);
this.socket = zmq.socket('pull');
this.socket.bind(this.target, function(err) {
if (err) {
return callback(err);
}
logger.info('Zeromq ready on ' + this.target);
callback();
}.bind(this));
this.socket.on('message', function(data) {
this.unserialize_data(data, function(parsed) {
this.emit('data', parsed);
}.bind(this), function(data) {
var obj = {
'message': data.toString().trim(),
'zeromq_from': this.target,
'type': this.type,
};
this.emit('data', obj);
}.bind(this));
}.bind(this));
};
MonitoredFile.prototype.close = function(callback) {
logger.info('Closing monitoring for', this.filename);
if (this.dir_watcher) {
logger.debug('Closing directory monitoring for', this.dir);
directory_watcher.unregister(this.dir_watcher);
delete this.dir_watcher;
}
async.eachSeries([function(callback) {
if (this.fdTailer === undefined) {
return callback();
}
this.fdTailer.close(function(last_data) {
file_status[this.filename] = {
last_data: last_data,
index: this.fdTailer.current_index,
};
callback();
}.bind(this));
}.bind(this), function(callback) {
async.eachSeries(this.oldFdTailers, function(fd, callback) {
clearTimeout(fd.id);
fd.f(callback);
}, callback);
}.bind(this)], async_helper.call(), callback);
};
BaseFilter.prototype.init = function(url) {
logger.info('Initializing filter', this.config.name);
this.loadConfig(url, function(err) {
if (err) {
this.emit('init_error', err);
return;
}
this.on('input', function(data) {
if (this.processMessage(data)) {
var res = this.process(data);
if (res) {
if (res.length === undefined) {
res = [res];
}
for(var i = 0; i < res.length; i ++) {
this.emit('output', res[i]);
}
}
}
else {
this.emit('output', data);
}
}.bind(this));
this.emit('init_ok');
}.bind(this));
}
fs.read(fd, buffer, 0, last_data.length, file_status[this.filename].index - last_data.length, function(err, bytesRead, buffer) {
if (err) {
return callback(err);
}
if (bytesRead === last_data.length && last_data === buffer.toString(this.buffer_encoding, 0, last_data.length)) {
logger.info('Start from last read index', this.filename, 'at', file_status[this.filename].index, 'fd', fd);
this.fdTailer = new FdTailer(fd, file_status[this.filename].index, this.buffer_encoding, this.buffer_size, stats.isFIFO(), this);
this.fdTailer.read();
}
else {
logger.info('Have last read index, but last data are not correct.', 'Start reading', this.filename, 'at end fd', fd);
this.fdTailer = new FdTailer(fd, stats.size, this.buffer_encoding, this.buffer_size, stats.isFIFO(), this);
this.fdTailer.read();
}
callback();
}.bind(this));
FilterLateroomsSetPageType.prototype.afterLoadConfig = function(callback) {
this.pageRegexs = [{
regex: /\/((k[0-9]+_[a-z-]+)|(r[0-9]+_hotels-in-[a-z-]+)|((H|h)otels))\.aspx$/,
page: 'search'
},
{
regex: /\/$/,
page: 'home-page'
},
{
regex: /\/hotel-(reservations|directions|facilities|pictures|videos|reviews|special-offers)\//,
page: 'hotel-details'
},
{
regex: /\/Booking\/Online\//,
page: 'booking-form'
},
{
regex: /\/PhoneReservation\/Open\//,
page: 'telephone-booking'
}];
logger.info('Initializing laterooms set page type, source ' + this.source_field);
callback();
}
function load_ssl_files(obj, callback, list) {
if (list === undefined) {
return load_ssl_files(obj, callback, [
'ssl_key',
'ssl_cert',
'ssl_ca',
'ssl_crl',
'ssl_pfx'
]);
}
if (list.length === 0) {
return callback();
}
var to_check = list.shift();
if (obj[to_check]) {
logger.info('Load SSL file', to_check, obj[to_check]);
fs.readFile(obj[to_check], function(err, result) {
if (err) {
return callback(err);
}
obj[to_check] = result;
load_ssl_files(obj, callback, list);
});
}
else {
load_ssl_files(obj, callback, list);
}
}
OutputAmqp.prototype.start = function(callback) {
this.amqp_url = amqp_driver.buildUrl(this);
this.channel = undefined;
logger.info('Start AMQP output to', this.amqp_url, 'exchange', this.exchange_name, 'topic', this.topic);
this.connected_callback = function(channel) {
channel.assertExchange(this.exchange_name, this.topic ? 'topic' : 'fanout', {durable: this.durable}, function(err) {
if (err) {
logger.error('Unable to create exchange', err);
}
else {
this.channel = channel;
}
}.bind(this));
}.bind(this);
this.disconnected_callback = function() {
this.channel = undefined;
}.bind(this);
this.amqp_logger = logger;
this.driver = amqp_driver.createAmqpClient(this);
callback();
};
InputUdp.prototype.afterLoadConfig = function(callback) {
logger.info('Start listening on udp', this.host + ':' + this.port);
this.server = dgram.createSocket('udp4');
this.server.on('message', function(data, remote) {
try {
var parsed = JSON.parse(data);
this.emit('data', parsed);
}
catch(e) {
this.emit('data', {
'@message': data.toString().trim(),
'@source': this.source || 'udp_' + this.host + '_' + this.port,
'@source_host': remote.address,
'@type': this.type,
});
}
}.bind(this));
this.server.on('error', function(err) {
this.emit('init_error', err);
}.bind(this));
this.server.once('listening', callback);
this.server.bind(this.port, this.host);
}
OutputCube.prototype.close = function(callback) {
logger.info('Closing output to Cube collector', this.host + ':' + this.port);
this.socket.close();
callback();
}
OutputGelf.prototype.close = function(callback) {
logger.info('Closing GELF output to udp', this.host + ':' + this.port);
this.socket.close();
callback();
}
InputTcp.prototype.afterLoadConfig = function(callback) {
logger.info('Start listening on tcp', this.host + ':' + this.port);
this.server = net.createServer(function(c) {
c.on('data', function(data) {
this.extract_json_logstash_event(data, function(parsed) {
this.emit('data', parsed);
}.bind(this), function(data) {
this.emit('data', {
'@message': data.toString().trim(),
'@source': 'tcp_' + this.host + '_' + this.port,
'@type': this.type,
});
}.bind(this));
}.bind(this));
c.on('error', function(err) {
this.emit('error', err);
}.bind(this));
}.bind(this));
this.server.on('error', function(err) {
this.emit('init_error', err);
}.bind(this));
this.server.listen(this.port, this.host);
this.server.once('listening', callback);
}