MysqlClientPool.createMysqlPool = function ( config ) {
  return genericPool.Pool({
    name: config.name,
    create: function ( callback ) {
      var connection = mysql.createConnection({
        host: config.host,
        port: config.port,
        user: config.user,
        debug: config.debug || false,
        password: config.password,
        database: config.database,
        charset: config.charset,
        typeCast: function ( field, next ) {
          if ( field.type == 'TINY' && field.length == 1 ) {
            return (field.string() == '1'); // 1 = true, 0 = false
          }
          return next();
        }
      });
      callback(null, connection);
    },
    destroy: function ( client ) {
      client.end();
    },
    min: 2,
    max: config.maxConnection || 10,
    idleTimeoutMillis: config.idleTimeoutMillis || 30000,
    log: config.log || false
  });
};
 initialize : function(options) {
     this.close = this.close.bind(this);
     this.setOptions(options);
     this._tmpDir = this.options.tmpDir || './tmp';
     this._mercator = new SphericalMercator({
         size : this.getTileSize()
     });
     if (!fs.existsSync(this._tmpDir)) {
         fs.mkdirSync(this._tmpDir);
     }
     var poolSize = this.options.poolSize || process.env.UV_THREADPOOL_SIZE
             || require('os').cpus().length;
     var that = this;
     this._pool = poolModule.Pool({
         name : 'mapnik',
         create : function(callback) {
             var tileSize = that.getTileSize();
             that._newMapnikInstance({
                 tileSize : tileSize,
                 width : tileSize,
                 height : tileSize
             }).then(function(result) {
                 callback(null, result);
             }, function(err) {
                 callback(err);
             })
         },
         destroy : function(map) {
         },
         max : poolSize,
         idleTimeoutMillis : that.options.poolTimeout || 30000,
         log : this.options.log
     });
 },
Example #3
0
exports.init = function (callback) {

    conn_props = local.config.db_config;

    mysql_pool = pool.Pool({
        name     : 'mysql',
        create   : function (callback) {
            var c = mysql.createConnection({
                host:     conn_props.host,
                user:     conn_props.user,
                password: conn_props.password,
                database: conn_props.database
            });
            callback(null, c);
        },
        destroy           : function(client) { client.end(); },
        max               : conn_props.pooled_connections,
        idleTimeoutMillis : conn_props.idle_timeout_millis,
        log               : false
    });

    // run a test query to make sure it's working.
    exports.run_mysql_query("SELECT 1", [], function (err, results) {
        if (err != null) {
            callback(err);
            console.error("Unable to connect to database server. Aborting.");
        } else {
            console.log("Database initialised and connected.");
            callback(null);
        }
    });

};
Example #4
0
Connector.prototype.createPool = function (factory) {
  var self = this;
  var pool;
  factory.create = function (callback) {
    if (self.settings.trace) {
      console.log('pooling create');
    }
    return self.driver.connect(self.settings, function (err, conn) {
      if (err) {
        return callback(err);
      }
      conn.getPool = function () {
        return pool;
      };
      conn.oldClose = conn.close;
      conn.close = function () {
        if (self.settings.trace) {
          console.log('pooling release');
        }
        return pool.release(conn);
      };
      return callback(null, conn);
    });
  };
  factory.destroy = function (conn) {
    if (self.settings.trace) {
      console.log('pooling destroy');
    }
    conn.oldClose();
  };
  pool = genericPool.Pool(factory);
  return pool;
};
Example #5
0
  return function* (next) {
    if (!this.app._mongoPool) {
      debug('Connect: ' + mongoUrl);
      this.app._mongoPool = poolModule.Pool({
        name     : 'koa-mongo',
        create   : function(callback) {
          MongoClient.connect(mongoUrl, {
            server: {poolSize: 1},
            native_parser: true
          }, callback);
        },
        destroy  : function(client) {client.close();},
        max      : max,
        min      : min, 
        idleTimeoutMillis : timeout,
        log : log 
      });
    }
    this.mongo = yield this.app._mongoPool.acquire;
    if (!this.mongo) this.throw('Fail to acquire one mongo connection')
    debug('Acquire one connection');

    try {
      yield* next;
    } catch (e) {
      throw e;
    } finally {
      this.app._mongoPool.release(this.mongo);
      debug('Release one connection');
    }
  }
function Wrapper(options) {
    var delay = options.conf.retry_delay || 100;

    this.conf = options.conf;
    this.log = options.log;
    this.retryLimit = options.conf.retry_limit || 5;
    this.randomDelay = function() {
        return Math.ceil(Math.random() * delay);
    };

    this.connectionPool = poolModule.Pool({
        name: 'sqlite',
        create: function(callback) {
            var client = new sqlite3.Database(expandDBName(options));
            callback(null, client);
        },
        destroy: function(client) {
            client.close();
        },
        max: 1,
        idleTimeoutMillis: options.conf.pool_idle_timeout || 10000,
        log: options.log
    });
    P.promisifyAll(this.connectionPool, { suffix: '_p' });
    this.readerConnection = new sqlite3.Database(expandDBName(options));
}
 init: function (url, dbName, opts) {
     url = url + "/" + dbName;
     opts = opts || {};
     return GenericPool.Pool({
         name: "mongoPool",
         create: function (callBack) {
             MongoClient.connect(url, {}, function (err, dbConn) {
                 if (err) {
                     logger.error(err);
                     if (dbConn) {
                         dbConn.__ended = true;
                     }
                     return callBack(err, dbConn);
                 }
                 logger.debug("Connected to mongo db.");
                 return callBack(null, dbConn);
             });
         },
         destroy: function (dbConn) {
             return dbConn.close();
         },
         validate: function (dbConn) {
             return !dbConn.__ended;
         },
         max: opts.max || 5,
         min: opts.min || 0,
         refreshIdle: opts.refreshIdle || true,
         reapIntervalMillis: opts.reapIntervalMillis || 1000,
         idleTimeoutMillis: opts.idleTimeoutMillis || 30000
     });
 }
Example #8
0
  pool: property(function() {
    if (this._pool) { return this._pool; }

    var self = this;
    var create = function(callback) {
      // an arbitrary value, usually a _client_, (and will be referred to as
      // _client_ throughout this code) will be passed through from the result of
      // the adapter's `_connect` to the success function. this will result in
      // the pool resource vending that _client_ when new acquisitions are made.
      var success = _.partial(callback, null);
      var failure = callback;
      self._connect().then(success, failure);
    };

    var destroy = function(client) {
      // it is expected that when resources are released from the pool the
      // `release` call will be provided with the _client_, the same object that
      // was returned from the `_connect` method. so here, it gets passed along
      // to `_disconnect` so the `_disconnect` gets the same _client_ object that
      // `_connect` returned.
      self._disconnect(client);
    };

    self._pool = BluebirdPromise.promisifyAll(pool.Pool({
      name: 'connection',
      create: create,
      destroy: destroy,
      max: 10,
      min: 2,
      idleTimeoutMillis: 30000
    }));

    return self._pool;
  })
Example #9
0
function CarbonClient(properties) {
  properties = properties || {};
  var self = this;
  this._dsn    = properties.dsn;
  var dsn  = url.parse(this._dsn);
  this._pool   = poolModule.Pool({
      name     : 'carbon-cache',
      create   : function(callback) {
          var c = net.createConnection(parseInt(dsn.port, 10) || 2003, dsn.hostname)
          c.on('error', function(err) {
            console.log('destroying carbon-cache client.');
            self._pool.destroy(c);
          });

          // parameter order: err, resource
          // new in 1.0.6
          callback(null, c);
      },
      destroy  : function(client) { console.log('destroying connection to carbon-cache'); client.end(); },
      max      : 10,
      // specifies how long a resource can stay idle in pool before being removed
      idleTimeoutMillis : 30000,
      reapIntervalMillis : 10000,
       // if true, logs via console.log - can also be a function
      log : false 
  });
}
  //--- constructor
  function Provider(table, dbconfig, debug) {
    var self = this;

    if(!dbconfig) {
      dbconfig = {
        host     : 'localhost',
        user     : '******',
        password : '',
        database : 'test'
      };
    }

    // define connection to mysql
    this.pool = generic_pool.Pool({
      name: 'mysql',
      max: 10,
      idleTimeoutMillis: 30000,
      log: false,
      create: function(callback) {
        console.log('pool create connection');
        var client = mysql.createConnection(self.dbconfig);
        client.connect();
        callback(null, client);
      },
      destroy: function(client) {
        client.end(function(err) {
          if(!err) client.destroy();
        });
      }
    });

    this.debug = debug ? true : false;
    this.dbconfig = dbconfig;
    this.table = table;
  };
Example #11
0
function createWorkersPool() {
    if (!childMMLBuilderPool) {
        childMMLBuilderPool = pool.Pool({
            name: 'mml-builder',
            create: function(callback) {
                return callback(null, fork(__dirname + '/mml-builder-child.js'));
            },
            destroy: function(child) {
                child.kill();
                child.disconnect();
            },
            validate: function(child) {
                return child && child.connected;
            },
            max: 8,
            min: 2,
            idleTimeoutMillis: 60000,
            reapIntervalMillis: 5000,
            log: false
        });

        process.on('exit', function (code) {
            childMMLBuilderPool.destroyAllNow();
        });
    }
}
Example #12
0
var createMysqlPool = function(app) {
	var mysqlConfig = app.get('mysql');
	return _poolModule.Pool({
		name: 'mysql',
		create: function(callback) {
			var mysql = require('mysql');
			var client = mysql.createConnection({
				host: mysqlConfig.host,
				user: mysqlConfig.user,
				password: mysqlConfig.password,
				database: mysqlConfig.database,
				port: 3306
			});
 
			client.connect(function(err) {
  			// connected! (unless `err` is set)
			});
			callback(null, client);
			

        // parameter order: err, resource
        // new in 1.0.6
        	//callback(null, c);
		},
		destroy: function(client) {
			client.end();
		},
		max: 10,
		idleTimeoutMillis : 30000,
		log : false
	});
};
	_connectToCubrid : function(){
		var self = this;
		this._bIsCubridConnected = false;

		this._oCubrid = GenericPool.Pool({
			name     : 'CUBRID',
		    // you can limit this pool to create maximum 10 connections
		    max      : 10,
		    min 	 : 1,
		    // destroy the connection if it's idle for 30 seconds
		    idleTimeoutMillis : 30000,
		    log : self._htCubrid.bDebug ,
		    create   : function(callback) {
		        var oCubrid = Cubrid.createCUBRIDConnection(
							self._htCubrid.sHostname, 
							self._htCubrid.nPort, 
							self._htCubrid.sUser,
							self._htCubrid.sPassword,
							self._htCubrid.sDatabase);
		        oCubrid.connect(function (err) {
		        	console.log('connected in _connectToCubrid, Migrator.js');
			  		self._bIsCubridConnected = true;
			  		self._checkConnection();
		        	callback(err, oCubrid);
		        });
		    },
		    destroy  : function(oCubrid) {
		      oCubrid.close();
		    }
		});


	},
Example #14
0
PDFConverter.prototype.initQueue = function() {
	if (USE_QUEUE) {
		convLogger.debug("Using queueing of conversion tasks");

		/**
		 * Pool will be used for queueing only
		 *
		 * @type {Object}
		 */
		this.pool = poolModule.Pool({
			name: "pdf-converter",
			min: POOL_MIN_NODES,
			max: POOL_MAX_NODES,
			idleTimeoutMillis: POOL_MAX_IDLE,

			/* place our own logger */
			log: function() { poolLogger.debug.apply(poolLogger, arguments); },

			/* do nothing */
			create: function(callback) {
				callback(null, new Boolean(true));
			},

			/* do nothing */
			destroy: function() {
				callback();
			}
		});
	} else {
		convLogger.debug("No queueing used - conversions run simultaneously");
	}
};
Example #15
0
module.exports.startPool = function() {
    var generic_pool = require('generic-pool');
    var mysql = require('mysql');
    var local = require('../local.config.js');
    conn_props = local.config.db_config;
    var pool = generic_pool.Pool({
    name: 'mysql',
    max: conn_props.max_poolsize,
    min: conn_props.min_poolsize,
    create: function(callback) {
        var c = mysql.createConnection({
            host:     conn_props.host,
            user:     conn_props.user,
            password: conn_props.password,
            database: conn_props.database
            });
        c.connect(function(err, server) {
            callback(err, c);
            });
        },
        destroy: function(c) {
            c.end();
        },
        idleTimeoutMillis : conn_props.idle_timeout_ms,
        log : conn_props.log_pool
    });
    if(pool){
   return pool;
    }
    else {
        throw new Error('unable to create sql pool!');
        } 


}
Example #16
0
var createMysqlPool = function(mysqlConfig) {
    return _poolModule.Pool({
        name: 'mysql',
        create: function(callback) {
            var mysql = require('mysql');
            var client = mysql.createConnection({
                host: mysqlConfig.host,
                user: mysqlConfig.user,
                password: mysqlConfig.password,
                database: mysqlConfig.database,
                multipleStatements: true
            });
            client.insert = client.query;
            client.update = client.query;
            client.delete = client.query;
            callback(null, client);
        },
        destroy: function(client) {
            client.end();
        },
        max: 10,
        idleTimeoutMillis : 30000,
        log : false
    });
};
Example #17
0
exports.create = function (database) {
  var ret = { database: database };

  ret.pool = genericPool.Pool({
    name: "db",
    create: function (callback) {
      backendModule.create(database, callback);
    },
    destroy: function (client) {
      logger.debug("Cleaning up a DB connection");

      if (backendModule.destroy) backendModule.destroy(client);
    },
    min: database.minConnections || 0,
    max: database.maxConnections || 20,
    idleTimeoutMillis: database.maxTimeout || 10000
  });

  // use our pool
  ret.query = function (sql, binds, cbDone) {
    exports.query(sql, binds, cbDone, ret.pool);
  };

  return ret;
};
Example #18
0
 var createConnectionPool = function(collection){
   pool = generic_pool.Pool({
     name: "azure",
     log: collection.log || false,
     max: collection.maxServices || 9,
     idleTimeoutMillis : 900000, // 15 minutes
     create: function(callback){
       var ts = null;
       try{
           if(collection.development){
             ts = azure.createTableService("UseDevelopmentStorage=true");
           }else{
             var retryOperations = collection.retryOperations || "ExponentialRetryPolicyFilter";
             retryOperations = new azure[retryOperations]();
             ts = azure.createTableService(collection.account, collection.secret).withFilter(retryOperations);
           }
       }catch(e){
         callback(e);
       }
       callback(null, ts);
     },
     destroy: function(conn){
       // Azure services communicate over OData and don't need to be explicitly closed
       conn = null;
     }
   });
 };
Example #19
0
function MongoObj(config){
    var pool = poolModule.Pool({
	name	 : config.name || 'mysql',
	create   : function(callback) {
	    // callback parameter order: err, resource
	    // new in 1.0.6
	    mongo.MongoClient.connect(["mongodb://",
				       config.mongodb.host, ':',
				       config.mongodb.port, '/',
				       config.mongodb.database].join(''), callback);
	},
	destroy  : function(client) { client.close(); },
	max	  : config.maxConnection || 10,
	idleTimeoutMillis : config.idleTimeoutMillis || 30000,
	log : config.log || false
    });
    function slowLog(req, info, caller, time){
	var logmsg = [info, ' query by: ', caller.func, ':', caller.line].join('');
	logmsg = ['Slow query, takes', time, 'ms', logmsg].join(' ');
	log.log(req, logmsg);
	log.console(req, 'WARNING', log.level.ASSERTION, logmsg);
    }
    var self = {
	acquire: function(req, collectionName, next){
	    if (!pool){
		next('failed');
		return;
	    }
	    var caller = exception.caller();
	    var acquireTimeBegin = new Date().getTime();
	    //console.log('pool.acquire');
	    // acquire connection - callback function is called
	    // once a resource becomes available
	    pool.acquire(function(err, db) {
		var acquireTimeEnd = new Date().getTime();
		if (acquireTimeEnd - acquireTimeBegin > queryTimeLimit){
		    slowLog(req, 'acquire time', caller, acquireTimeEnd - acquireTimeBegin);
		}
		if (err){
                    next(err);
                }else if(!db){
		    pool.release(db);
		    next("mongo db not connected");
		}else{
		    var collection = db.collection(collectionName);
		    var beginTime = new Date().getTime();
		    next(null, collection, function(){
			var endTime = new Date().getTime();
			if (endTime - beginTime > queryTimeLimit){
			    slowLog(req, 'query time', caller, endTime - beginTime);
			}					
			pool.release(db);
		    });
		}
	    });
	}
    };
    return self;
}
Example #20
0
exports.connect = function() {
  if (!databaseJsonLoaded) {
    tryLoadDatabaseJson(path.join(process.cwd(), "database.json"));
  }

  var callback = arguments[arguments.length - 1];
  var opts;
  if (arguments.length === 1) {
    opts = getDefaultConnectOptions();
  } else {
    opts = arguments[0];
  }

  if (!opts || !opts.driver) {
    throw new Error("Invalid options. Have you configured your database.json file?");
  }
  var driverName = opts.driver;
  var Driver = require('./drivers/' + driverName + '.js');
  var driver = new Driver();

  if (opts.pooling && opts.pooling.name) {
    var pool;
    opts.pooling.create = function(callback) {
      if (opts.trace) {
        console.log('pooling create');
      }
      return driver.connect(opts, function(err, conn) {
        if (err) {
          return callback(err);
        }
        conn.getPool = function() {
          return pools[opts.pooling.name];
        };
        conn.oldClose = conn.close;
        conn.close = function() {
          if (opts.trace) {
            console.log('pooling release');
          }
          return pool.release(conn);
        };
        return callback(null, conn);
      });
    };
    opts.pooling.destroy = function(conn) {
      if (opts.trace) {
        console.log('pooling destroy');
      }
      conn.oldClose();
    };
    if (pools[opts.pooling.name]) {
      pool = pools[opts.pooling.name];
    } else {
      pool = pools[opts.pooling.name] = genericPool.Pool(opts.pooling);
    }
    return pool.acquire(connectAfterAutoDefinesComplete.bind(this, callback));
  } else {
    return driver.connect(opts, connectAfterAutoDefinesComplete.bind(this, callback));
  }
};
Example #21
0
 this.add = function(create) {
   var pool_factory = {};
   for (i in factory) {
     pool_factory[i] = factory[i];
   }
   pool_factory.create = create;
   cluster.push(PoolModule.Pool(pool_factory));
 }
Example #22
0
  var makePool = function(config) {
    config = config || {};
    config.host = config.host || 'localhost';
    config.port = config.port || 6379;
    config.db = config.db || 0;
    config.options = config.options || {};
    config.options.detect_buffers = config.options.detect_buffers || true;
    config.pool = config.pool || {};

    var pool = Pool({
      name: poolId(config),
      create: function(callback) {
        var client = redis.createClient(config.port, config.host, config.options);     

        replaceClientAsNecessary(client);

        client.on('connect', function() {
          client.send_anyway = true;
          client.select(config.db, function(err) {
            if (err) throw err;
          });
          client.send_anyway = false;
        });

        client.on('error', function(err) {
          //if (err) throw err;
          console.error(err);
        });

        return callback(null, client);
      },
      destroy: function(client) {
        //return client.quit();
        return client._destroy();
      },
      // Maximum number of concurrent clients
      max: config.pool.max || 3,
      // Minimum number of connections ready in the pool
      // If set, then make sure to drain() on process shutdown
      min: config.pool.min || 0,
      // How long a resource can stay idle before being removed
      idleTimeoutMillis: config.pool.idleTimeoutMillis || 30*1000,
      reapIntervalMillis: config.pool.reapIntervalMillis || 1*1000,
      // Use console.log if true, but it can also be function (message, level, debug)
      log: config.pool.log || debug
    });

    // If a minimum number of clients is set, then process.exit() can hang
    // unless the following listener is set.
    process.on('exit', function() {
      pool.drain(function() {
        pool.destroyAllNow();
      });
    });

    return pool;
  };
Example #23
0
var FTPPool = function (name, connectionData, maxConnections, idleTimeoutMillis, logger) {

    if (typeof connectionData === 'undefined') {
        throw new Error('No FTP (%s)parameters defined.', name);
    }

    var LOG = helper.logger(logger);

    if (typeof connectionData.host === 'undefined' || connectionData.host === '') {
        throw new Error('No host defined in parameters.');
    }

    function createFtpConnection(callback) {
        LOG.debug('Create FTP connection...');
        var ftp, err;
        try {
            ftp = new JsFtp({
                host: connectionData.host,
                port: connectionData.port
            });
            ftp.on('error', callback);
            LOG.info('FTP (%s) > Connect to : %s:%d', name, connectionData.host, connectionData.port);
        } catch (e) {
            err = e;
        }
        if (err) {
            callback(err, ftp);
        } else {
            LOG.info('FTP (%s) > Connect with username : %s', name, connectionData.username);
            ftp.auth(connectionData.username, connectionData.password, function (err) {
                callback(err, ftp);
            });
        }
    }

    function destroyFtpConnection(ftp) {
        LOG.debug('Destroy FTP connection...');
        if (typeof ftp !== 'undefined') {
            ftp.raw.quit(function (err) {
                if (err) {
                    LOG.error(err);
                }
                LOG.info('FTP (%s) > Disconnected', name);
            });
        }
    }

    return poolModule.Pool({
        name: name,
        create: createFtpConnection,
        destroy: destroyFtpConnection,
        max: maxConnections,
        idleTimeoutMillis: idleTimeoutMillis,
        log: false
    });

};
/**
 * Function: RedisConnectionPool
 *
 * A high-level redis management object. It manages a number of connections in
 * a pool, using them as needed and keeping all aspects of releasing active
 * connections internal to the object, so the user does not need to worry about
 * forgotten connections leaking memory and building up over time.
 *
 * Parameters:
 *
 *   uid - (string) - Unique identifer to retreive an existing instance from
 *                    elsewhere in an application. If left undefined, one will
 *                    be generate automatically and avaialble via the `uid`
 *                    property of the returned object.
 *
 *   cfg - (object) - A series of configuration parameters to be optionally
 *                    passed in and used during initialization of the object.
 *
 *
 *   cfg.host - (string) - Redis host (default: "127.0.0.1")
 *
 *   cfg.port - (number) - Redis port (default: 6379)
 *
 *   cfg.max_clients - (number) - Max clients alive in the connection pool at
 *                                once. (default: 30)
 *
 *   cfg.perform_checks - (boolean) - Perform a series of redis checks,
 *                                    currently this checks to to see if
 *                                    blocking push/pops can be used.
 *                                    (default: false)
 *
 *   cfg.database - (number) - if you prefer a specific database number for this
 *                             pool, you can specify that here (default: 0)
 *
 * Returns:
 *
 *   A RedisConnectionPool object
 */
function RedisConnectionPool(uid, cfg) {
  this.uid            = (typeof uid ==='string') ? uid : 'redis-connection-pool-' + Math.floor((Math.random() * 99999) + 10000);
  this.host           = (typeof cfg.host === 'string') ? cfg.host : '127.0.0.1';
  this.port           = (typeof cfg.port === 'number') ? cfg.port : 6379;
  this.max_clients    = (typeof cfg.max_clients === 'number') ? cfg.max_clients : 30;
  this.perform_checks = (typeof cfg.perform_checks === 'boolean') ? cfg.perform_checks : false;
  this.options        = (typeof cfg.options === 'object') ? cfg.options : null;
  this.database       = (typeof cfg.database === 'number') ? cfg.database : 0;

  this.blocking_support = true;
  this.version_array    = undefined;
  this.version_string   = undefined;

  var self = this;

  var i = 0;
  this.pool = Pool({
    name: self.uid,
    create: function (callback) {
      var client = redis.createClient(self.port, self.host, self.options);
      client.__name = "client" + i;
      i = i + 1;

      self.database = self.database || 0;

      debug('selecting database ' + self.database);
      client.on('error', function (err) {
        debug(err);
      });
      
      client.on('ready', function () {
        client.select(self.database, function (err) {
          debug('2. selected database: ' + client.selected_db);
          callback(null, client);
        });
      });
    },
    destroy: function (client) {
      return client.quit();
    },
    max: self.max_clients,
    log: false
  });

  redisCheck.apply(this, []);

  setTimeout(function poolStats(pool) {
    // periodically report pool statistics
    debug('REDIS POOL: [size: ' + pool.getPoolSize() +
                ' avail:' + pool.availableObjectsCount() +
                ' waiting:' + pool.waitingClientsCount() + ']');
    setTimeout(poolStats, 300000, pool);
  }, 300000, this.pool);

  return this;
}
Example #25
0
  getOrCreate: function(clientConfig) {
    clientConfig = clientConfig || {};
    var name = JSON.stringify(clientConfig);
    var pool = pools.all[name];
    if(pool) {
      return pool;
    }
    pool = genericPool.Pool({
      name: name,
      max: defaults.poolSize,
      idleTimeoutMillis: defaults.poolIdleTimeout,
      reapIntervalMillis: defaults.reapIntervalMillis,
      log: defaults.poolLog,
      create: function(cb) {
        var client = new pools.Client(clientConfig);
        client.connect(function(err) {
          if(err) return cb(err, null);

          //handle connected client background errors by emitting event
          //via the pg object and then removing errored client from the pool
          client.on('error', function(e) {
            pool.emit('error', e, client);
            pool.destroy(client);
          });

          return cb(null, client);
        });
      },
      destroy: function(client) {
        client.end();
      }
    });
    pools.all[name] = pool;
    //mixin EventEmitter to pool
    EventEmitter.call(pool);
    for(var key in EventEmitter.prototype) {
      if(EventEmitter.prototype.hasOwnProperty(key)) {
        pool[key] = EventEmitter.prototype[key];
      }
    }
    //monkey-patch with connect method
    pool.connect = function(cb) {
      pool.acquire(function(err, client) {
        if(err)  return cb(err, null, function() {/*NOOP*/});
        cb(null, client, function(err) {
          if(err) {
            pool.destroy(client);
          } else {
            pool.release(client);
          }
        });
      });
    };
    return pool;
  }
Example #26
0
Crawler.prototype.init = function (options) {
    var self = this;

    // you can use jquery or jQuery
    options = checkJQueryNaming(options);

    // if using rateLimits we want to use only one connection with delay in between requests
    if (options.rateLimits !== 0) {
        options.maxConnections = 1;
    }

    //Default options
    self.options = _.extend({
        autoWindowClose:    true,
        cache:              false, //false,true, [ttl?]
        forceUTF8:          false,
        incomingEncoding:   null, //TODO remove or optimize
        jQuery:             true,
        jQueryUrl:          path.resolve(__dirname,"../vendor/jquery-1.8.3.min.js"),
        maxConnections:     10,
        method:             "GET",
        onDrain:            false,
        priority:           5,
        priorityRange:      10,
        referer:            false,
        retries:            3,
        retryTimeout:       10000,
        skipDuplicates:     false,
        onDrain:            false,
        rateLimits:         0,
        referer:            false,
        incomingEncoding:   null
    },options);

    // Don't make these options persist to individual queries
    var globalOnlyOptions = ["maxConnections", "priorityRange", "onDrain"];

    //Setup a worker pool w/ https://github.com/coopernurse/node-pool
    self.pool = Pool({
        name         : 'crawler',
        //log        : self.options.debug,
        max          : self.options.maxConnections,
        priorityRange: self.options.priorityRange,
        create       : function(callback) {
            callback(1);
        },
        destroy      : function(client) {}
    });

    self.plannedQueueCallsCount = 0;
    self.queueItemSize = 0;

    self.onDrain = function() {};
    self.cache = {};
};
Example #27
0
function mongo(options) {
  options = options || {};
  var host = options.host || 'localhost';
  var port = options.port || 27017;
  var max = options.max || 100;
  var min = options.min || 1;
  var timeout = options.timeout || 30000;
  var log = options.log || false;
  var db = options.db;
  var mongoUrl = options.uri || options.url;
  if(!mongoUrl) {
    if (options.user && options.pass) {
      mongoUrl = 'mongodb://' + options.user + ':' + options.pass + '@' + host + ':' + port;
    } else {
      mongoUrl = 'mongodb://' + host + ':' + port;
    }
    if (db) {
      mongoUrl = mongoUrl + '/' + db;
    }
  }

  var mongoPool = poolModule.Pool({
    name     : 'koa-mongo',
    create   : function(callback) {
      MongoClient.connect(mongoUrl, {
        server: {poolSize: 1},
        native_parser: true,
        uri_decode_auth: true
      }, function (err, client) {
        if (err) throw err;
        callback(err, client);
      });
    },
    destroy  : function(client) {client.close();},
    max      : max,
    min      : min, 
    idleTimeoutMillis : timeout,
    log : log 
  });

  return function* mongo(next) {
    this.mongo = yield mongoPool.acquire.bind(mongoPool);
    if (!this.mongo) this.throw('Fail to acquire one mongo connection')
    debug('Acquire one connection (min: %s, max: %s, poolSize: %s)', min, max, mongoPool.getPoolSize());

    try {
      yield* next;
    } catch (e) {
      throw e;
    } finally {
      mongoPool.release(this.mongo);
      debug('Release one connection (min: %s, max: %s, poolSize: %s)', min, max, mongoPool.getPoolSize());
    }
  };
} 
Example #28
0
exports.get_pool = function (server, port, host, connect_timeout, pool_timeout, max) {
    var port = port || 25;
    var host = host || 'localhost';
    var connect_timeout = (connect_timeout === undefined) ? 30 : connect_timeout;
    var pool_timeout = (pool_timeout === undefined) ? 300 : pool_timeout;
    var name = port + ':' + host + ':' + pool_timeout;
    if (!server.notes.pool) {
        server.notes.pool = {};
    }
    if (!server.notes.pool[name]) {
        var pool = generic_pool.Pool({
            name: name,
            create: function (callback) {
                var smtp_client = new SMTPClient(port, host, connect_timeout);
                logger.logdebug('[smtp_client_pool] uuid=' + smtp_client.uuid + ' host=' + host 
                    + ' port=' + port + ' pool_timeout=' + pool_timeout + ' created');
                callback(null, smtp_client);
            },
            destroy: function(smtp_client) {
                logger.logdebug('[smtp_client_pool] ' + smtp_client.uuid + ' destroyed, state=' + smtp_client.state);
                smtp_client.state = STATE_DESTROYED;
                smtp_client.socket.destroy();
                // Remove pool object from server notes once empty
                var size = pool.getPoolSize();
                if (size === 0) {
                    delete server.notes.pool[name];
                }
            },
            max: max || 1000,
            idleTimeoutMillis: pool_timeout * 1000,
            log: function (str, level) {
                level = (level === 'verbose') ? 'debug' : level;
                logger['log' + level]('[smtp_client_pool] [' + name + '] ' + str);
            }
        });

        var acquire = pool.acquire;
        pool.acquire = function (callback, priority) {
            var callback_wrapper = function (err, smtp_client) {
                smtp_client.pool = pool;
                if (smtp_client.state === STATE_DEAD) {
                    smtp_client.destroy();
                    pool.acquire(callback, priority);
                    return;
                }
                smtp_client.state = STATE_ACTIVE;
                callback(err, smtp_client);
            };
            acquire.call(pool, callback_wrapper, priority);
        };
        server.notes.pool[name] = pool;
    }
    return server.notes.pool[name];
};
Example #29
0
PG.prototype.connect = function(config, callback) {
  var self = this;
  var c = config;
  var cb = callback;
  //allow for no config to be passed
  if(typeof c === 'function') {
    cb = c;
    c = defaults;
  }

  //get unique pool name even if object was used as config
  var poolName = typeof(c) === 'string' ? c : c.user+c.host+c.port+c.database;
  var pool = pools[poolName];

  if(pool) return pool.acquire(cb);

  var pool = pools[poolName] = genericPool.Pool({
    name: poolName,
    create: function(callback) {
      var client = new self.Client(c);
      client.connect();

      var connectError = function(err) {
        client.removeListener('connect', connectSuccess);
        callback(err, null);
      };

      var connectSuccess = function() {
        client.removeListener('error', connectError);

        //handle connected client background errors by emitting event
        //via the pg object and then removing errored client from the pool
        client.on('error', function(e) {
          self.emit('error', e, client);
          pool.destroy(client);
        });
        callback(null, client);
      };

      client.once('connect', connectSuccess);
      client.once('error', connectError);
      client.on('drain', function() {
        pool.release(client);
      });
    },
    destroy: function(client) {
      client.end();
    },
    max: defaults.poolSize,
    idleTimeoutMillis: defaults.poolIdleTimeout
  });
  return pool.acquire(cb);
}
Example #30
0
function VerticaConnectionPool(user, password, database, host, port){
    var pool = poolModule.Pool({
        name     : 'vertica',
        create   : function(callback) {
            var vertica = require('vertica');
            var client = vertica.connect({user: user, password: password, database: database, host: host, port: port}, function(err) {
                if(err){
                    callback(err, null);
                    return;
                }
                else{
                    callback(null, client);
                }
            });
            
            client.connection.on('error', function (err) {
                callback(err, null);
                return;
            });
            
            client.on('error', function(error){
                callback(error, null);
                return;
            });
            // parameter order: err, resource
            // new in 1.0.6
        },
        destroy  : function(client,callback) { 
            try {
              client.disconnect();
          } catch (err) {
              logger.error('client.disconnect error');
              logger.error(err.stack);
              if(callback) callback(err);
          }  
          client.connection.on('error', function (err) {
              callback(err, null);
              return;
          });
          
          client.on('error', function(error){
              callback(error, null);
              return;
          });
        },
        max      : 1,
        idleTimeoutMillis : 30000, // need to decide on an appropriate value
        log : false
    });
    logger.info("VERTICA POOOOL CREATED with " + user + " " + password + " " + database + " " + host + " " + port);
    return pool;
}