AbstractZeroMQ.prototype.startAbstract = function(callback) {
  logger.info('Start output to', this.to(), 'using ZeroMQ', zmq.version);

  this.socket = zmq.socket('push');
  if (this.zmq_high_watermark) {
    logger.info('Setting high watermark on ZeroMQ socket', this.zmq_high_watermark);
    if (zmq.version.match(/^2\./)) {
      this.socket.setsockopt(zmq.ZMQ_HWM, parseInt(this.zmq_high_watermark, 10));
    }
    else {
      this.socket.setsockopt(zmq.ZMQ_SNDHWM, parseInt(this.zmq_high_watermark, 10));
    }
  }
  this.target.split(',').forEach(function(address) {
    this.socket.connect(address);
  }.bind(this));

  if (this.zmq_check_interval) {
    this.check_interval_id = setInterval(function() {
      this.check();
    }.bind(this), this.zmq_check_interval);
  }

  this.on_alarm = false;

  callback();
};
Example #2
0
 this.sig_listener = function() {
   if (this.closed_inputs) {
     logger.info('Starting inputs plugins');
     this.start_inputs();
   }
   else {
     logger.info('Closing inputs plugins');
     this.close_inputs();
   }
 }.bind(this);
 fs.read(this.fdTailer.fd, buffer, 0, last_data.length, this.fdTailer.current_index - last_data.length, function(err, bytesRead, buffer) {
   if (err) {
     return this.emit('error', err);
   }
   if (bytesRead === last_data.length && last_data === buffer.toString(this.buffer_encoding, 0, last_data.length)) {
     logger.info('Event changed received, but no data change and last data match', this.filename, 'fd', this.fdTailer.fd);
   }
   else {
     logger.info('Event changed received, but no data change and last data does not match.', 'Restarting reading', this.filename, 'at 0 fd', this.fdTailer.fd);
     this.restart(0);
   }
 }.bind(this));
Example #4
0
 fs.open(this.filename, 'r', function(err, fd) {
   this.open_in_progress = false;
   if (err) {
     this.last_open_failed = true;
     return callback(err);
   }
   this.to_be_processed = '';
   // Some data about file in db_file ?
   if (file_status[this.filename] && file_status[this.filename].last_data && file_status[this.filename].index) {
     var last_data = file_status[this.filename].last_data;
     // Enough data to check last_data ?
     if (file_status[this.filename].index >= last_data.length && stats.size > last_data.length) {
       var buffer = new Buffer(last_data.length);
       fs.read(fd, buffer, 0, last_data.length, file_status[this.filename].index - last_data.length, function(err, bytesRead, buffer) {
         if (err) {
           return callback(err);
         }
         if (bytesRead === last_data.length && last_data === buffer.toString(this.buffer_encoding, 0, last_data.length)) {
           logger.info('Start from last read index', this.filename, 'at', file_status[this.filename].index, 'fd', fd);
           this.fdTailer = new FdTailer(fd, file_status[this.filename].index, this.buffer_encoding, this.buffer_size, stats.isFIFO(), this);
           this.fdTailer.read();
         }
         else {
           logger.info('Have last read index, but last data are not correct.', 'Start reading', this.filename, 'at end fd', fd);
           this.fdTailer = new FdTailer(fd, stats.size, this.buffer_encoding, this.buffer_size, stats.isFIFO(), this);
           this.fdTailer.read();
         }
         callback();
       }.bind(this));
     }
     else {
       logger.info('Have last read index, but file is too small.', 'Start reading', this.filename, 'at end fd', fd);
       this.fdTailer = new FdTailer(fd, stats.size, this.buffer_encoding, this.buffer_size, stats.isFIFO(), this);
       this.fdTailer.read();
       callback();
     }
   }
   // No data about file, starting normally
   else {
     if (start_index === undefined) {
       logger.info('Start reading', this.filename, 'at end', 'fd', fd);
       this.fdTailer = new FdTailer(fd, stats.size, this.buffer_encoding, this.buffer_size, stats.isFIFO(), this);
     }
     else {
       logger.info('Start reading', this.filename, 'at', start_index, 'fd', fd);
       this.fdTailer = new FdTailer(fd, start_index, this.buffer_encoding, this.buffer_size, stats.isFIFO(), this);
       this.fdTailer.read();
     }
     callback();
   }
 }.bind(this));
exports.unregister = function(id) {
  logger.info('Remove watcher on dir', id.dir);
  current[id.dir].removeListener('change', id.callback);
  if (current[id.dir].listeners('change').length === 0) {
    logger.info('Removing empty listener on', id.dir);
    if (platform === 'darwin') {
      current[id.dir].stop();
    }
    else {
      current[id.dir].close();
    }
    delete current[id.dir];
  }
};
MonitoredFile.prototype.close = function(callback) {
  var close_old_fd_tailers = function() {
    if (this.oldFdTailers.length === 0) {
      return callback();
    }
    clearTimeout(this.oldFdTailers[0].id);
    this.oldFdTailers[0].f(close_old_fd_tailers);
  }.bind(this);
  logger.info('Closing monitoring for', this.filename);
  if (this.dir_watcher) {
    logger.debug('Closing directory monitoring for', this.dir);
    this.dir_watcher.close();
    this.dir_watcher = undefined;
  }
  if (this.fdTailer) {
    this.fdTailer.close(function(last_data) {
      file_status[this.filename] = {
        last_data: last_data,
        index: this.fdTailer.current_index,
      };
      close_old_fd_tailers();
    }.bind(this));
  }
  else {
    close_old_fd_tailers();
  }
};
Example #7
0
 this.close_outputs(function() {
   logger.info('Closing agent');
   if (this.sig_listener) {
     sig_listener.removeListener('SIGUSR1', this.sig_listener);
   }
   return callback();
 }.bind(this));
Example #8
0
InputSQS.prototype.start = function(callback) {
  this.polling_delay = parseInt(this.polling_delay, 10);
  this.closed_callback = undefined;
  logger.info('Creating AWS SQS Input from', this.aws_host, 'polling delay', this.polling_delay);
  this.waitMessage();
  callback();
};
Example #9
0
function loadPatterns(callback) {
  var _leave = function(err) {
    loaded = true;
    onload_callbacks.forEach(function(cb) {
      setImmediate(function() {
        cb(err);
      });
    });
    onload_callbacks = [];
  };
  if (loaded) {
    return callback();
  }
  onload_callbacks.push(callback);
  if (onload_callbacks.length > 1) {
    return;
  }
  logger.info('Loading grok patterns');
  patterns_loader.loadGrokPatterns(function(err, lines) {
    if (err) {
      return _leave(err);
    }
    var patterns = processLines(lines);
    logger.info('Grok patterns loaded from patterns directories', Object.keys(patterns).length);
    global_patterns = patterns;
    _leave();
  });
}
Example #10
0
MonitoredFile.prototype.watch = function(callback) {
  try {
    this.dir = path.dirname(this.filename);
    var basename = path.basename(this.filename);
    logger.info('Watching dir', this.dir, 'for file', basename);
    this.dir_watcher = directory_watcher.register(this.dir, function(event, filename) {
      logger.debug('Event received for', this.filename, ':', event, filename);
      if (event === 'rename' && basename === filename) {
        this.emit('renamed');
      }
      else if (event === 'change' && basename === filename) {
        this.emit('changed');
      }
      else if (event === 'change') {
        this.emit('other_changed', filename);
      }
      else {
        logger.debug('Unprocessed event while monitoring', this.filename, ':', event, filename);
      }
    }.bind(this));
    callback();
  }
  catch (err) {
    logger.error('Unable to monitor', this.dir, ':', err);
    callback(err);
  }
};
Example #11
0
  detector.on('exists', function(dir, newly_created) {
    logger.info('Parent directory exists', dir, 'for reading', this.file);

    fs.readdir(dir, function(err, l) {
      if (err) {
        return this.emit('error', new Error('Error while reading ' + dir + ' : ' + err));
      }
      l.forEach(function(x) {
        this.monitorFile(dir, x, newly_created ? 0 : this.start_index);
      }.bind(this));
      try {
        if (this.monitored_files[dir]) {
          return;
        }
        var dir_watcher = directory_watcher.register(dir, function(event, filename) {
          this.monitorFile(dir, filename, 0);
        }.bind(this));
        this.monitored_files[dir] = {
          close: function(callback) {
            directory_watcher.unregister(dir_watcher);
            callback();
          }
        };
      }
      catch(err) {
        return this.emit('error', new Error('Error while reading ' + dir + ' : ' + err));
      }
    }.bind(this));

  }.bind(this));
Example #12
0
InputUnix.prototype.start = function(callback) {
  logger.info('Start listening on unix socket', this.socket);

  this.server = net.createServer(function(c) {
    c.on('data', function(data) {
      this.unserialize_data(data, function(parsed) {
        this.emit('data', parsed);
      }.bind(this), function(data) {
        this.emit('data', {
          'message': data.toString().trim(),
          'path': this.socket,
          'type': this.type,
        });
      }.bind(this));
    }.bind(this));
    c.on('error', function(err) {
      this.emit('error', err);
    }.bind(this));
  }.bind(this));

  this.server.on('error', function(err) {
    this.emit('error', err);
  }.bind(this));

  this.server.listen(this.socket, callback);
};
function RedisConnectionManager(host, port) {
  events.EventEmitter.call(this);
  this.host = host;
  this.port = port;

  logger.info('Connecting to Redis', this.host + ':' + this.port);

  this.client = redis.createClient(this.port, this.host);
  this.end_callback = function() {
    logger.info('Redis connection lost to ' + this.host + ':' + this.port);
  }.bind(this);

  this.client.on('end', function() {
    this.end_callback();
  }.bind(this));

  this.client.on('error', function(err) {
    this.emit('error', err);
  }.bind(this));

  this.client.on('connect', function() {
    logger.info('Connected to Redis', this.host + ':' + this.port);
    this.emit('connect', this.client);
  }.bind(this));
}
RedisConnectionManager.prototype.quit = function(callback) {
  logger.info('Closing connection to Redis', this.host + ':' + this.port);
  this.emit('before_quit', this.client);
  this.end_callback = callback;
  this.client.quit();
  this.client = undefined;
};
Example #15
0
 var start_streaming = function() {
   log.info(ctx.from + 'Starting streaming', ctx.directory);
   copyHeadersIfExists(headers_to_copy_from_disk, ctx.result_headers, function(k, v) {response.setHeader(k, v)});
   fs.open(ctx.filename, 'r', function(err, file_fd) {
     if (err) {
       response.statusCode = 500;
       response.end();
       log.error(ctx.from + 'Unable to open file for reading', ctx.filename, err);
     }
     fd = file_fd;
     log.debug(ctx.from + 'File opened', ctx.filename);
     ctx.events.once('data', read_data);
   });
   ctx.events.on('end', function(size, filename) {
     fs.close(fd);
     stop = true;
     if (current != size) {
       log.info(ctx.from + 'Have sent to client', current, 'expected', size, ', getting data from', filename, 'request', ctx.directory);
       fs.open(filename, 'r', function(err, fd) {
         if (err) {
           log.error(ctx.from + 'Unable to open file', filename, err);
           return;
         }
         var to_be_read = size - current;
         var buffer_in = new Buffer(64 * 1024);
         var r = function() {
           fs.read(fd, buffer_in, 0, buffer_in.length, current, function(err, bytesRead, buffer_in) {
             if (err) {
               log.error(ctx.from + 'Error reading', filename, err);
               return;
             }
             to_be_read -= bytesRead;
             current += bytesRead;
             log.debug(ctx.from + 'Sent', bytesRead, 'from real file, current', current, 'for', ctx.directory);
             var buffer_out = new Buffer(bytesRead);
             buffer_in.copy(buffer_out, 0, 0, bytesRead);
             response.write(buffer_out);
             if (to_be_read == 0) {
               log.info(ctx.from + 'All data sent to client', ctx.directory);
               response.end();
               fs.close(fd);
               return;
             }
             if (bytesRead != buffer_in.length) {
               log.error(ctx.from + 'Wrong reading length', bytesRead, 'exptected', buffer_in.length, 'remaining', to_be_read, 'reading position', current - bytesRead);
               return;
             }
             r();
           })
         };
         r();
       })
     }
     else {
       log.info(ctx.from + 'All data has been sent to client from temp file', ctx.directory)
       fs.close(fd);
       response.end();
     }
   });
 };
AbstractUdp.prototype.startAbstract = function(callback) {
  logger.info('Start output to ' + this.to());

  this.socket = dgram.createSocket('udp4');

  callback();
};
Example #17
0
InputSQS.prototype.close = function(callback) {
  logger.info('Closing AWS SQS Input from', this.aws_host, 'waiting end of polling');
  this.closed_callback = function() {
    logger.info('Closed AWS SQS Input from', this.aws_host);
    callback();
  };
};
Example #18
0
InputFile.prototype.start = function(callback) {
  logger.info('Start input on file', this.file);

  if (this.start_index) {
    this.start_index = parseInt(this.start_index, 10);
  }

  this.filter = file_filter.create(path.basename(this.file));

  var parent = path.dirname(this.file);
  this.detector = new directory_detector.DirectoryDetector();

  this.detector.on('exists', function() {
    logger.info('Parent directory exists', parent, 'for reading', this.file);

    fs.readdir(parent, function(err, l) {
      if (err) {
        return this.emit('error', err);
      }
      l.forEach(function(x) {
        this.monitorFile(parent, x, this.start_index);
      }.bind(this));
    }.bind(this));

    this.dir_watcher = directory_watcher.register(parent, function(event, filename) {
      this.monitorFile(parent, filename, 0);
    }.bind(this));

  }.bind(this));

  this.monitored_files = {};

  this.detector.start(parent, callback);
};
Example #19
0
InputUdp.prototype.afterLoadConfig = function(callback) {
  logger.info('Start listening on udp', this.host + ':' + this.port);

  this.configure_unserialize(this.unserializer);

  this.server = dgram.createSocket('udp4');

  this.server.on('message', function(data, remote) {
    this.unserialize_data(data, function(parsed) {
      this.emit('data', parsed);
    }.bind(this), function(data) {
      this.emit('data', {
        'message': data.toString().trim(),
        'host': remote.address,
        'udp_port': this.port,
        'type': this.type,
      });
    }.bind(this));
  }.bind(this));

  this.server.on('error', function(err) {
    this.emit('init_error', err);
  }.bind(this));

  this.server.once('listening', callback);

  this.server.bind(this.port, this.host);
}
InputZeroMQ.prototype.start = function(callback) {
  logger.info('Start listening on zeromq', this.target, 'using ZeroMQ', zmq.version);

  this.socket = zmq.socket('pull');
  this.socket.bind(this.target, function(err) {
    if (err) {
      return callback(err);
    }
    logger.info('Zeromq ready on ' + this.target);

    callback();
  }.bind(this));

  this.socket.on('message', function(data) {
    this.unserialize_data(data, function(parsed) {
      this.emit('data', parsed);
    }.bind(this), function(data) {
      var obj = {
        'message': data.toString().trim(),
        'zeromq_from': this.target,
        'type': this.type,
      };
      this.emit('data', obj);
    }.bind(this));
  }.bind(this));
};
Example #21
0
MonitoredFile.prototype.close = function(callback) {
  logger.info('Closing monitoring for', this.filename);
  if (this.dir_watcher) {
    logger.debug('Closing directory monitoring for', this.dir);
    directory_watcher.unregister(this.dir_watcher);
    delete this.dir_watcher;
  }
  async.eachSeries([function(callback) {
    if (this.fdTailer === undefined) {
      return callback();
    }
    this.fdTailer.close(function(last_data) {
      file_status[this.filename] = {
        last_data: last_data,
        index: this.fdTailer.current_index,
      };
      callback();
    }.bind(this));
  }.bind(this), function(callback) {
    async.eachSeries(this.oldFdTailers, function(fd, callback) {
      clearTimeout(fd.id);
      fd.f(callback);
    }, callback);
  }.bind(this)], async_helper.call(), callback);
};
Example #22
0
BaseFilter.prototype.init = function(url) {
  logger.info('Initializing filter', this.config.name);

  this.loadConfig(url, function(err) {
    if (err) {
      this.emit('init_error', err);
      return;
    }
    this.on('input', function(data) {
      if (this.processMessage(data)) {
        var res = this.process(data);
        if (res) {
          if (res.length === undefined) {
            res = [res];
          }
          for(var i = 0; i < res.length; i ++) {
            this.emit('output', res[i]);
          }
        }
      }
      else {
        this.emit('output', data);
      }
    }.bind(this));

    this.emit('init_ok');
  }.bind(this));
}
Example #23
0
 fs.read(fd, buffer, 0, last_data.length, file_status[this.filename].index - last_data.length, function(err, bytesRead, buffer) {
   if (err) {
     return callback(err);
   }
   if (bytesRead === last_data.length && last_data === buffer.toString(this.buffer_encoding, 0, last_data.length)) {
     logger.info('Start from last read index', this.filename, 'at', file_status[this.filename].index, 'fd', fd);
     this.fdTailer = new FdTailer(fd, file_status[this.filename].index, this.buffer_encoding, this.buffer_size, stats.isFIFO(), this);
     this.fdTailer.read();
   }
   else {
     logger.info('Have last read index, but last data are not correct.', 'Start reading', this.filename, 'at end fd', fd);
     this.fdTailer = new FdTailer(fd, stats.size, this.buffer_encoding, this.buffer_size, stats.isFIFO(), this);
     this.fdTailer.read();
   }
   callback();
 }.bind(this));
FilterLateroomsSetPageType.prototype.afterLoadConfig = function(callback) {
  this.pageRegexs = [{
	regex: /\/((k[0-9]+_[a-z-]+)|(r[0-9]+_hotels-in-[a-z-]+)|((H|h)otels))\.aspx$/,
	page: 'search'
  },
  {
	regex: /\/$/,
	page: 'home-page'
  },
  {
	regex: /\/hotel-(reservations|directions|facilities|pictures|videos|reviews|special-offers)\//,
	page: 'hotel-details'
  },
  {
	regex: /\/Booking\/Online\//,
	page: 'booking-form'
  },
  {
    regex: /\/PhoneReservation\/Open\//,
	page: 'telephone-booking'
  }];

  logger.info('Initializing laterooms set page type, source ' + this.source_field);

  callback();
}
Example #25
0
function load_ssl_files(obj, callback, list) {
  if (list === undefined) {
    return load_ssl_files(obj, callback, [
      'ssl_key',
      'ssl_cert',
      'ssl_ca',
      'ssl_crl',
      'ssl_pfx'
    ]);
  }
  if (list.length === 0) {
    return callback();
  }
  var to_check = list.shift();
  if (obj[to_check]) {
    logger.info('Load SSL file', to_check, obj[to_check]);
    fs.readFile(obj[to_check], function(err, result) {
      if (err) {
        return callback(err);
      }
      obj[to_check] = result;
      load_ssl_files(obj, callback, list);
    });
  }
  else {
    load_ssl_files(obj, callback, list);
  }
}
Example #26
0
OutputAmqp.prototype.start = function(callback) {
  this.amqp_url = amqp_driver.buildUrl(this);
  this.channel = undefined;
  logger.info('Start AMQP output to', this.amqp_url, 'exchange', this.exchange_name, 'topic', this.topic);


  this.connected_callback = function(channel) {
    channel.assertExchange(this.exchange_name, this.topic ? 'topic' : 'fanout', {durable: this.durable}, function(err) {
      if (err) {
        logger.error('Unable to create exchange', err);
      }
      else {
        this.channel = channel;
      }
    }.bind(this));
  }.bind(this);
  this.disconnected_callback = function() {
    this.channel = undefined;
  }.bind(this);
  this.amqp_logger = logger;

  this.driver = amqp_driver.createAmqpClient(this);

  callback();
};
Example #27
0
InputUdp.prototype.afterLoadConfig = function(callback) {
  logger.info('Start listening on udp', this.host + ':' + this.port);

  this.server = dgram.createSocket('udp4');

  this.server.on('message', function(data, remote) {
    try {
      var parsed = JSON.parse(data);
      this.emit('data', parsed);
    }
    catch(e) {
      this.emit('data', {
        '@message': data.toString().trim(),
        '@source': this.source || 'udp_' + this.host + '_' + this.port,
        '@source_host': remote.address,
        '@type': this.type,
      });
    }
  }.bind(this));

  this.server.on('error', function(err) {
    this.emit('init_error', err);
  }.bind(this));

  this.server.once('listening', callback);

  this.server.bind(this.port, this.host);
}
Example #28
0
OutputCube.prototype.close = function(callback) {
  logger.info('Closing output to Cube collector', this.host + ':' + this.port);

  this.socket.close();

  callback();
}
Example #29
0
OutputGelf.prototype.close = function(callback) {
  logger.info('Closing GELF output to udp', this.host + ':' + this.port);

  this.socket.close();

  callback();
}
Example #30
0
InputTcp.prototype.afterLoadConfig = function(callback) {
  logger.info('Start listening on tcp', this.host + ':' + this.port);

  this.server = net.createServer(function(c) {
    c.on('data', function(data) {
      this.extract_json_logstash_event(data, function(parsed) {
        this.emit('data', parsed);
      }.bind(this), function(data) {
        this.emit('data', {
          '@message': data.toString().trim(),
          '@source': 'tcp_' + this.host + '_' + this.port,
          '@type': this.type,
        });
      }.bind(this));
    }.bind(this));
    c.on('error', function(err) {
      this.emit('error', err);
    }.bind(this));
  }.bind(this));

  this.server.on('error', function(err) {
    this.emit('init_error', err);
  }.bind(this));

  this.server.listen(this.port, this.host);

  this.server.once('listening', callback);
}