var mover = function(file, done){
var dest = !sep_reg.test(that.filename) ?
path.join(config.components, that.filename, basename(file)) :
path.join(config.views, that.filename + '-' + path.basename(file));
var loader = config.viewMini ?
async.compose(htmlcompressor, async.apply(fs.readFile, file, 'utf8')) :
async.apply(fs.readFile, file);
var run = async.compose( async.apply(fs.writeFile, dest), loader, mkdirp);
run(path.dirname(dest), done);
};
describe('#urlrefactor()', function () {
var worker = async.compose(shasum, urllist.urlrefactor, fs.readFile.partial(undefined, 'utf8', undefined));
var logger = utils.message;
var warning = config.warning;
beforeEach(function () {
utils.message = mock_logger();
config.warning = true;
});
afterEach(function () {
utils.message = logger;
config.warning = warning;
});
it('should refactor urls', function (done) {
// put path pairs into urllist
path_pairs.forEach(function (pair) {
urllist.pair(pair);
});
// transform the content
worker(viewfile, function (err, hash) {
if (err) console.error(prettyjson.render(err));
expect(utils.message._message).to.be('might be broken link: \u001b[35m/asset/vendor/backbone.localStorage-fake.js\u001b[39m');
expect(hash).to.be(correct_hash);
done();
})
});
});
BuilderDatabase.prototype.saveHash = function(project, packages, callback){
if (!packages || !packages.length){
if (callback) callback(null);
return;
}
var packageString = typeof packages == 'string' ? packages : packages.join(';');
var hash = md5.digest_s(packageString);
function hashCount(db, cb){
db.get('SELECT COUNT(*) AS count FROM hashes WHERE md5 = ?', {1: hash}, function(err, row){
cb(err, row && {count: row.count, db: db, hash: hash});
});
}
function insertIfNotExisting(res, cb){
if (res.hash && res.count > 0){
cb(null, {hash: hash, packages: packages});
} else {
var values = {1: hash, 2: packageString, 3: Math.round(Date.now() / 1000)};
res.db.run('INSERT INTO hashes (md5, packages, date) VALUES (?, ?, ?)', values, function(error){
if (error) cb(error);
else cb(null, {hash: hash, packages: packages});
});
}
}
async.compose(
insertIfNotExisting,
hashCount,
this.getDatabase.bind(this, project)
)(callback);
};
exports.clean = function (command, callback) {
// variables declare
var folders = [], eraser;
if (typeof command == 'function') {
callback = command;
command = 'build';
}
command = (command && typeof command == 'string' && command == 'stack') ? 'stack' : 'build';
callback = typeof callback == 'function' ? callback : function () {
};
// set folders to clean
switch (command) {
case 'stack':
folders.push(config.js, config.css, config.components, config.src);
break;
case 'build':
folders.push(config.distPublic, config.distViews);
break;
}
// compose our own eraser
eraser = async.compose(async.each.partial(undefined, rimraf, undefined), glob, dirfiles);
// looping the folders to empty
if (config.warning) exports.message('Start to clean folers:\n - ' + folders.join('\n - '));
async.each(folders, eraser, callback);
// build the path to ensure we only empty the files under the directory
function dirfiles(dir_path, done) {
done(null, path.join(dir_path, '*'));
}
};
function S3GzipPutObject(s3, s3params, body, callback) {
var put = async.compose(function (zdata, callback) {
s3.putObject(combineObjects(s3params, {'Body': zdata}), callback)
},
zlib.gzip
);
put(body, callback);
}
BuilderDatabase.prototype.loadHash = function(project, hash, callback){
function getHash(db, cb){
db.get('SELECT * FROM hashes WHERE md5 = ?', {1: hash}, function(err, row){
cb(err, row ? {hash: row.md5, packages: row.packages.split(';')} : null);
});
}
async.compose(
getHash,
this.getDatabase.bind(this, project)
)(callback);
};
exports.copyFile = function (source, destination, next) {
var write = function (data, next) {
fs.writeFile(destination, data, next)
}
var done = function (next) {
console.log("Copied %s to %s", source, destination)
next(null)
}
async.compose(done, write, fs.readFile)(source, next)
}
Task.prototype.run = function(callback){
var filename = this.filename;
var nature = this.nature;
// decide the files loader
utils.message('start to stack %s:%s', filename.magenta, nature.magenta);
//this.concat ? concat_runner.call(this, logging) : non_concat_htmlrunner.call(this, logging);
// update the mapping object list
try {
this._mappings = filesUtils.getMappingByPatterns(this._patterns, config.moveExtnames);
} catch (err) {
this._mappings = [];
utils.warn(err);
}
var run = this.concat ? async.compose(files_copy.bind(this), concat_runner.bind(this))
: async.compose(files_copy.bind(this), non_concat_htmlrunner.bind(this));
run(logging);
function logging(err, bytes) {
if (err) {
utils.warn(err);
utils.warn('stacking %s:%s failed.', filename.magenta, nature.magenta);
return callback();
}
if (!bytes) {
utils.message('stack %s:%s written', filename.magenta, nature.magenta);
return callback();
}
utils.message('stack %s:%s written %s', filename.magenta, nature.magenta, pretty_bytes(bytes));
callback();
}
};
loadUrlAsync( articleUrl, function( html, articleId, revId ) {
if ( html ) {
var prepareAndSaveArticle = async.compose( writeArticle, applyOtherTreatments, rewriteUrls, parseHtml );
prepareAndSaveArticle(html, articleId, function ( error, result ) {
if ( error ) {
console.error( "Error by preparing and saving file " + error );
process.exit( 1 );
} else {
setTimeout( finished, 0 );
}
});
} else {
delete articleIds[ articleId ];
setTimeout( finished, 0 );
}
}, articleId);
module.exports = function(project, options){
if (!project){
throw new Error("'project' argument is required");
}
if (!options) options = {};
if (!options.title) options.title = project;
var dir = path.join(__dirname, '../', pkg._buildOutput, project, 'docs');
var getVersions = async.apply(loadJSON, dir + '/versions.json');
var getDocsContent = async.apply(loadDocsVersions, dir);
var loadDocs = waitForIt(async.compose(getDocsContent, getVersions));
fs.watch(dir, debounce(function(){
console.log('resetting ' + dir + ' docs data');
loadDocs.reset();
}));
return function(req, res, next){
loadDocs.get(function(err, data){
if (err) return next(err);
var docs = data.docs;
var versions = data.versions;
var latest = data.latest;
var version = req.params.version || latest;
if (!docs[version]) version = latest;
res.render(project + '/docs', {
page: "/" + project + "/docs",
title: options.title,
content: docs[version].content,
toc: docs[version].toc,
version: version,
versions: versions
});
});
};
};
exports.index = function(req, res, next){
var viewsPath = path.resolve(__dirname, "../views") + "/";
var docDir = "docSections/";
var docPath = viewsPath + docDir;
console.log("docPath: " + docPath);
function getSectionData(filenamesString, cb) {
function capitaliseFirstLetter(string) {
return string.charAt(0).toUpperCase() + string.slice(1);
}
function getSectionName (filename, cb) {
var basename = path.basename(filename, ".html");
var sectionName = capitaliseFirstLetter(basename.replace(/-/g, " "));
cb(null, {
filename: docDir + filename,
sectionName: sectionName,
anchorId: basename
});
}
console.log("FilenamesString: " + filenamesString);
var filenames = filenamesString.trim().split("\n");
async.map(filenames, getSectionName, function(err, sections) {
cb(err, sections);
});
}
var listSections = async.compose(getSectionData, fs.readFile);
listSections(docPath+"sectionList.txt", "utf8", function(err, sections) {
if (err)
return next(new Error("Something borked"));
var includeSections = sections.map(function(section) {
return "<% include " + section.filename + " %>";
});
console.log(sections);
res.render('playground', {sections: sections});
});
};
Middleware.render = function (view, req, res, options, params) {
var viewParams = params,
httpResponse = {
req: req,
res: res,
view: view,
params: viewParams
},
call = async.compose(this.session, this.meta);
if (viewParams === undefined) {
viewParams = {};
}
if (httpResponse.params === undefined) {
httpResponse.params = {};
}
if (httpResponse.params.session === undefined) {
httpResponse.params.session = {};
}
var rendering = function (){
call(httpResponse, function (err, httpResponse) {
logger.debug(httpResponse.params.session);
res.render(view, httpResponse.params);
});
};
if (options && options.authenticate){
this.authenticate(req, res, rendering);
} else {
rendering();
}
};
'use strict'
const Qool = require('./lib/Qool')
const level = require('level-bytewise')
const rimraf = require('rimraf')
const path = require('path')
const async = require('async')
const _ = require('lodash')
let iterator = async.compose(
all,
(index, cb) => {
console.log('starting cycle %d', index)
console.log('deleting database')
rimraf(path.join(__dirname, 'db'), cb)
}
)
const ITERATIONS = 100
async.timesSeries(ITERATIONS, iterator, (err, results) => {
if (err) return console.error(err)
console.log('avg: %d', results.reduce((sum, member) => { return sum + member }) / ITERATIONS)
})
function all(cb) {
const db = level('db')
const queue = new Qool(db)
const SIZE = process.argv[2] || 100000
const ENQUEUE_SIZE = SIZE / 4
'file':'m3u8',
'type':'mp4',
'url':json.norVid
}
]
};
return callback(null,download_urls);
}catch(err){
return callback(new Error(err));
}
}else{
return callback(new Error(err));
}
});
};
var getter = async.compose(getUrls,getRequestUrl);
var sohu = function(req,res){
var url = req.query.url;
getter(url,function(err,result){
if(!err){
res.send(handler.success('sohu',url,result));
}else{
res.send(handler.fail('sohu',url,err));
}
});
}
module.exports = sohu;
function index(posts, callback){
var total = {
posts: posts,
urls: {},
tags: {}
};
posts.forEach(function(post, i){
post.date = new Date(post.date);
total.urls[post.permalink] = i;
if (post.tags && Array.isArray(post.tags)) post.tags.forEach(function(tag){
tag = tag.toLowerCase();
(total.tags[tag] || (total.tags[tag] = [])).push(i);
});
});
callback(null, total);
}
var load = async.compose(index, loadContent, loadJSONPosts);
module.exports = waitForIt(load);
fs.watch(dir, debounce(function(){
console.log('reloading blog data');
module.exports.reset();
}));
var concat_runner = function(callback){
var loader = /^(css|chtml)$/.test(this.nature)
? ( this.autoprefixer ? css.partial(undefined, this.autoprefixer, undefined) : css )
: ( !this.commonjs ? concat : browserify.partial(undefined, { browserify: this.browserify, autoprefixer: this.autoprefixer}, undefined) );
//( this.commonjs == 'webmake' ? webmake : browserify )),
var runner;
var dest;
var filename = this.filename;
var nature = this.nature;
switch (this.nature){
case 'css':
dest = path.join( config.css, filename + '.min.css');
runner = async.compose(cleanCSS, loader);
break;
case 'chtml':
dest = !sep_reg.test(filename)
? path.join( config.components, filename + '.min.css')
: path.join( config.views, filename + '.min.css');
runner = async.compose(cleanCSS, loader);
break;
case 'js':
dest = path.join( config.js, filename + '.min.js');
runner = async.compose( uglify, loader);
break;
case 'jhtml':
dest = !sep_reg.test(filename)
? path.join( config.components, filename + '.min.js')
: path.join( config.views, filename + '.min.js');
runner = async.compose( uglify, loader);
break;
case 'html':
case 'tmpl':
case 'tpl':
dest = !sep_reg.test(filename)
? path.join( config.components, filename + (this.ext ? this.ext : '.html'))
: path.join( config.views, filename + (this.ext ? this.ext : '.html'));
runner = config.viewMini ? async.compose(htmlcompressor, loader) : loader;
break;
default:
return callback(new RangeError('unrecognized nature:' + this.nature));
}
runner(this.files, function(err, codes){
if (err) return callback(err);
async.series([
mkdirp.partial(path.dirname(dest), undefined),
fs.writeFile.partial(dest, codes, undefined)
],
function(err) {
if (err) return callback(err);
if (typeof codes == 'string') {
return callback(null, Buffer.byteLength(codes, 'utf8'));
}
if (Buffer.isBuffer(codes)) {
return callback(null, codes.length);
}
callback(new TypeError(format('%s:%s codes was written successful, however codes type should be string or buffer but got %s.', filename.magenta, nature.magenta, typeof codes)))
}
);
});
};
commands.push(
JSON.stringify({
index : {
_index: 'page',
_type: 'blogging',
_id: post.permalink
}
}),
JSON.stringify({
title: post.title,
date: post.date,
author: post.author,
tags: post.tags.slice(),
url: "/blog/" + post.permalink,
content: post.content.toString()
})
);
});
var _commands = commands.join("\n") + "\n";
request.post("http://localhost:9200/_bulk/", {
body: _commands
}, function(err, res, body){
callback(err, body);
});
}
module.exports = async.compose(insertBlog, data.get);
var async = require('async');
/**
* 通过compose组合，f(g(h()))的形式，从内层到外层的执行的顺序。
*/
function f(n,callback){
console.log('f:'+n);
setTimeout(function(){
callback(null,n+1);
},1000);
}
function g(n,callback){
console.log('g:'+n);
setTimeout(function(){
callback(null,n*2);
},1000);
}
function h(n,callback){
console.log('h:'+n);
setTimeout(function(){
callback(null,n-10);
},1000);
}
var fgh = async.compose(f,g,h);
fgh(4,function(err,result){
console.log('result:'+result);
})
function Broker(config, components) {
var self = this;
var vhosts = {};
var publications = {};
var subscriptions = {};
var sessions = [];
var init = async.compose(tasks.initShovels, tasks.initSubscriptions, tasks.initPublications, tasks.initCounters, tasks.initVhosts);
var nuke = async.compose(tasks.deleteVhost, tasks.disconnectVhost, tasks.nukeVhost);
var purge = tasks.purgeVhost;
var disconnect = tasks.disconnectVhost;
var bounce = tasks.bounceVhost;
var counters = _.defaults({}, components.counters, { stub: stub, inMemory: inMemory, inMemoryCluster: inMemoryCluster });
this.config = config;
this._init = function(next) {
debug('Initialising broker');
init(config, { broker: self, components: { counters: counters } }, function(err, config, ctx) {
vhosts = ctx.vhosts;
publications = ctx.publications;
subscriptions = ctx.subscriptions;
sessions = [];
setImmediate(function() {
next(err, self);
});
});
return this;
};
this.nuke = function(next) {
debug('Nuking broker');
self.unsubscribeAll(function(err) {
if (err) return next(err);
async.eachSeries(_.values(vhosts), function(vhost, callback) {
nuke(config, { vhost: vhost }, callback);
}, function(err) {
if (err) return next(err);
vhosts = publications = subscriptions = {};
debug('Finished nuking broker');
next();
});
});
return this;
};
this.purge = function(next) {
debug('Purging all queues in all vhosts');
async.eachSeries(_.values(vhosts), function(vhost, callback) {
purge(config, { vhost: vhost }, callback);
}, function(err) {
if (err) return next(err);
debug('Finished purging all queues in all vhosts');
next();
});
return this;
};
this.shutdown = function(next) {
debug('Shutting down broker');
self.unsubscribeAll(function(err) {
if (err) return next(err);
async.eachSeries(_.values(vhosts), function(vhost, callback) {
disconnect(config, { vhost: vhost }, callback);
}, function(err) {
if (err) return next(err);
debug('Finished shutting down broker');
next();
});
});
return this;
};
this.bounce = function(next) {
debug('Bouncing broker');
self.unsubscribeAll(function(err) {
if (err) return next(err);
async.eachSeries(_.values(vhosts), function(vhost, callback) {
bounce(config, { vhost: vhost }, callback);
}, function(err) {
if (err) return next(err);
debug('Finished bouncing broker');
next();
});
});
return this;
};
this.publish = function(name, message, overrides, next) {
if (arguments.length === 3) return self.publish(name, message, {}, arguments[2]);
if (_.isString(overrides)) return self.publish(name, message, { routingKey: overrides }, next);
if (!publications[name]) return next(new Error(format('Unknown publication: %s', name)));
publications[name].publish(message, overrides, next);
return this;
};
this.forward = function(name, message, overrides, next) {
if (arguments.length === 3) return self.forward(name, message, {}, arguments[2]);
if (_.isString(overrides)) return self.forward(name, message, { routingKey: overrides }, next);
if (!config.publications[name]) return next(new Error(format('Unknown publication: %s', name)));
publications[name].forward(message, overrides, next);
return this;
};
this.subscribe = function(name, overrides, next) {
if (arguments.length === 2) return self.subscribe(name, {}, arguments[1]);
if (!subscriptions[name]) return next(new Error(format('Unknown subscription: %s', name)));
subscriptions[name].subscribe(overrides, function(err, session) {
if (err) return next(err);
sessions.push(session);
next(null, session);
});
return this;
};
this.unsubscribeAll = function(next) {
async.eachSeries(sessions.slice(), function(session, cb) {
sessions.shift();
session.cancel(cb);
}, next);
};
this.getFullyQualifiedName = this.qualify = function(vhost, name) {
return fqn.qualify(name, config.vhosts[vhost].namespace);
};
}
var debug = require('debug')('rascal:Broker');
var format = require('util').format;
var inherits = require('util').inherits;
var EventEmitter = require('events').EventEmitter;
var _ = require('lodash');
var async = require('async');
var tasks = require('./tasks');
var configure = require('../config/configure');
var validate = require('../config/validate');
var fqn = require('../config/fqn');
var preflight = async.compose(validate, configure);
var stub = require('../counters/stub');
var inMemory = require('../counters/inMemory');
var inMemoryCluster = require('../counters/inMemoryCluster').worker;
module.exports = {
create: function(config, components, next) {
if (arguments.length === 2) return this.create(config, {}, arguments[1]);
preflight(_.cloneDeep(config), function(err, config) {
if (err) return next(err);
new Broker(config, components)._init(next);
});
},
};
inherits(Broker, EventEmitter);
function Broker(config, components) {
var self = this;
var vhosts = {};
exec('git describe --abbrev=0', {
cwd: vers.dir
}, (error, stdout, stderr) => {
var tag = stdout.replace('\n', '');
if (tag == vers.version) callback(error, vers);
else callback(vers.version + ' tag mismatch');
});
}
function mkdirs(vers, callback){
fs.mkdirs(vers.dir, function(err){
callback(err, vers);
});
}
var cloneAndCheckout = async.compose(verifyTag, checkout, clone, mkdirs);
function cloneAndCheckoutAll(versions, callback){
async.map(versions, cloneAndCheckout, callback);
}
var cloneNonExisting = async.compose(cloneAndCheckoutAll, findNonExisting);
var pkg = require('../package.json');
var projects = pkg._projects;
var versions = [];
prime.each(projects, function(project, name){
project.versions.forEach(function(version){
versions.push({
var deleteTableIfExists = function (client, tableName, callback) {
var exists = function (name, cbExists) {
client.describeTable({ TableName: name }, function (err, data) {
if (err) {
if (err.code === "ResourceNotFoundException") {
cbExists(null, { exists: false, definition: { TableName: name } });
} else {
error("deleteTableIfExists - describeTable error: " + JSON.stringify(err, null, 2));
cbExists(err);
}
} else {
cbExists(null, { exists: true, description: { TableName: data.Table.TableName } });
}
});
};
var deleteTable = function (r, cbDelete) {
if (r.exists) {
client.deleteTable(r.description, function (err, data) {
if (err) {
error("Error deleting '" + r.description.TableName + "': " + JSON.stringify(err, null, 2));
cbDelete(err);
} else {
cbDelete(null, { Table: { TableName: data.TableDescription.TableName, TableStatus: data.TableDescription.TableStatus } });
}
});
} else {
cbDelete(null, r.description);
}
};
var active = function (d, cbActive) {
var status = d.Table.TableStatus;
async.until(
function () { return status === "DELETED" },
function (cbUntil) {
client.describeTable({ TableName: d.Table.TableName }, function (err, data) {
if (err) {
if (err.code === "ResourceNotFoundException") {
status = "DELETED";
return cbUntil(null, d.Table.TableName);
}
error("Error calling describeTable for '" + d.Table.TableName + "'");
cbUntil(err);
} else {
setTimeout(cbUntil, 1000);
}
});
},
function (err, r) {
if (err) {
error("connect delete table error: " + err);
return cbActive(err);
}
cbActive(null, r);
});
};
async.compose(active, deleteTable, exists)(tableName, function (err, result) {
if (err) callback(err);
else callback(null, result);
});
};
var createTableIfNotExists = function (client, params, callback) {
var exists = function (p, cbExists) {
client.describeTable({ TableName: p.TableName }, function (err, data) {
if (err) {
if (err.code === "ResourceNotFoundException") {
debug("Table " + p.TableName + " doesn't exist yet: " + JSON.stringify(p, null, 2));
cbExists(null, { exists: false, definition: p });
} else {
error("Table " + p.TableName + " doesn't exist yet but describeTable: " + JSON.stringify(err, null, 2));
cbExists(err);
}
} else {
debug("Table " + p.TableName + " already exists.");
cbExists(null, { exists: true, description: data });
}
});
};
var create = function (r, cbCreate) {
if (!r.exists) {
debug("Creating " + r.definition.TableName);
client.createTable(r.definition, function (err, data) {
if (err && !isTableAlreadyExistsError(err)) {
error("Error while creating " + r.definition.TableName + ": " + JSON.stringify(err, null, 2));
cbCreate(err);
} else {
debug(params.TableName + " created. Waiting for activiation.");
cbCreate(null, { Table: { TableName: params.TableName, TableStatus: data ? data.TableDescription.TableStatus : "UNKNOWN"} });
}
});
} else {
cbCreate(null, r.description);
}
};
var active = function (d, cbActive) {
var status = d.Table.TableStatus;
async.until(
function () { return status === "ACTIVE" },
function (cbUntil) {
debug("checking " + d.Table.TableName + " status.");
client.describeTable({ TableName: d.Table.TableName }, function (err, data) {
if (err) {
error("There was an error checking " + d.Table.TableName + " status: " + JSON.stringify(err, null, 2));
cbUntil(err);
} else {
status = data.Table.TableStatus;
setTimeout(cbUntil, 1000);
}
});
},
function (err, r) {
if (err) {
error("connect create table error: " + err);
return cbActive(err);
}
debug("Table " + d.Table.TableName + " is active.");
cbActive(null, r);
});
};
async.compose(active, create, exists)(params, function (err, result) {
if (err) callback(err);
else callback(null, result);
});
};
async.doWhilst(function (next) {
async.compose(drop, read)({}, function (err, result) {
if (err) next(err);
else next(null, result);
});
}, function () {
}
function getReal(results,callback){
async.map(results,getLocation,function(err,results){
if(!err){
var urls = {
urls:results
}
return callback(null,urls);
}
else{
return callback(err);
}
})
}
var async = require('async');
var getter = async.compose(getReal,getUrl,getMMSID);
var letv = function(req,res){
var url = req.query.url;
getter(url,function(err,result){
if(!err)
res.send(handler.success('letv',url,result,''));
else{
res.send(handler.fail('letv',url,err));
}
});
}
module.exports = letv;
);
function add1(n, callback) {
setTimeout(function () {
callback(null, n + 1);
}, 10);
}
function mul3(n, callback) {
setTimeout(function () {
callback(null, n * 3);
}, 10);
}
var add1mul3 = async.compose(mul3, add1);
add1mul3(4, function (err, result) {
console.log(result)
});
// create a queue object with concurrency 2
var q = async.queue(function (task, callback) {
console.log('hello ' + task.name);
callback();
}, 2);
function Vhost(config) {
var self = this;
var connection;
var connectionConfig;
var channelPool = createChannelPool({ confirm: false, size: config.publicationChannelPools.regularPoolSize });
var confirmChannelPool = createChannelPool({ confirm: true, size: config.publicationChannelPools.confirmPoolSize });
var channelCreator = async.queue(createChannel, 1);
var init = async.compose(tasks.closeChannel, tasks.applyBindings, tasks.purgeQueues, tasks.checkQueues, tasks.assertQueues, tasks.checkExchanges, tasks.assertExchanges, tasks.createChannel, tasks.createConnection, tasks.checkVhost, tasks.assertVhost);
var purge = async.compose(tasks.closeConnection, tasks.closeChannel, tasks.purgeQueues, tasks.createChannel, tasks.createConnection);
var nuke = async.compose(tasks.closeConnection, tasks.closeChannel, tasks.deleteQueues, tasks.deleteExchanges, tasks.createChannel, tasks.createConnection);
var timer = backoff({});
pauseChannelAllocation();
this.name = config.name;
this.connectionIndex = 0;
this.init = function(next) {
debug('Initialising vhost: %s', config.name);
pauseChannelAllocation();
init(config, { connectionIndex: this.connectionIndex }, function(err, config, ctx) {
if (err) return next(err);
self.emit('connect');
attachErrorHandlers(ctx.connection, config);
forwardEvents(ctx.connection, self, function(eventName) {
return eventName === 'blocked' || eventName === 'unblocked';
});
debug('vhost: %s was initialised with connection: %s', config.name, ctx.connection._rascal_id);
connection = ctx.connection;
this.connectionIndex = ctx.connectionIndex;
connectionConfig = ctx.connectionConfig;
timer = backoff(ctx.connectionConfig.retry);
resumeChannelAllocation();
return next(null, self);
});
return self;
};
this.nuke = function(next) {
debug('Nuking vhost: %s', config.name);
pauseChannelAllocation();
nuke(config, { connectionIndex: this.connectionIndex }, function(err, config, ctx) {
if (err) return next(err);
connection = undefined;
debug('Finished nuking vhost: %s', config.name);
setImmediate(next);
});
};
this.purge = function(next) {
debug('Purging vhost: %s', config.name);
purge(config, { purge: true, connectionIndex: this.connectionIndex }, function(err, config, ctx) {
if (err) return next(err);
debug('Finished purging vhost: %s', config.name);
setImmediate(next);
});
};
this.bounce = function(next) {
async.series([
self.disconnect,
self.init,
], next);
};
this.disconnect = function(next) {
debug('Disconnecting vhost: %s', config.name);
pauseChannelAllocation();
if (!connection) return next();
connection.removeAllListeners();
connection.on('error', function(err) {
debug('Error disconnecting from %s. Original error was: %s', connectionConfig.loggableUrl, err.message);
});
connection.close(next);
};
this.getChannel = function(next) {
channelCreator.push({ confirm: false }, next);
debug('Requested channel. Outstanding channel requests: %d', channelCreator.length());
};
this.getConfirmChannel = function(next) {
channelCreator.push({ confirm: true }, next);
debug('Requested confirm channel. Outstanding channel requests: %d', channelCreator.length());
};
this.borrowChannel = channelPool.borrow;
this.returnChannel = channelPool.return;
this.borrowConfirmChannel = confirmChannelPool.borrow;
this.returnConfirmChannel = confirmChannelPool.return;
function createChannelPool(options) {
var displayType = options.confirm ? ' confirm' : '';
var pool = new Pool({
max: options.size,
create: function(next) {
channelCreator.push(options, function(err, channel) {
if (err) return next(err);
var releaseChannel = _.once(function() {
channel._rascal_closed = true;
pool.release(channel);
});
channel.once('error', releaseChannel);
channel.once('close', releaseChannel);
next(null, channel);
});
},
destroy: function(channel) {
if (!channel._rascal_closed) channel.close();
},
refreshIdle: false,
validate: function(channel) {
return !channel._rascal_closed && connection && connection.connection === channel.connection;
},
});
var poolQueue = async.queue(function(__, next) {
pool.acquire(next);
}, 1);
function stats() {
return format('Queue size: %d, pool size: %d, available: %d, taken: %d',
poolQueue.length(), pool.getPoolSize(), pool.availableObjectsCount(), pool.inUseObjectsCount());
}
function borrow(next) {
debug('Requested%s channel. %s', displayType, stats());
poolQueue.push(null, function (err, channel) {
if (err) return next(err);
debug('Borrowed%s channel: %s. %s', displayType, channel._rascal_id, stats());
next(null, channel);
});
}
function release(channel) {
debug('Returning%s channel: %s. %s', displayType, channel._rascal_id, stats());
pool.release(channel);
}
return {
borrow: borrow,
return: release,
pause: poolQueue.pause.bind(poolQueue),
resume: poolQueue.resume.bind(poolQueue),
};
}
function createChannel(options, next) {
// Same problem as https://github.com/guidesmiths/rascal/issues/17
var once = _.once(next);
var invocations = 0;
var channelId = uuid();
options.confirm ? connection.createConfirmChannel(callback) : connection.createChannel(callback);
function callback(err, channel) {
invocations++;
if (err) {
debug('Error creating channel: %s from %s: %s', channelId, connectionConfig.loggableUrl, err.message);
return once(err);
}
channel._rascal_id = channelId;
channel.connection._rascal_id = connection._rascal_id;
channel.connection.setMaxListeners(0);
debug('Created channel %s from connection: %s', channel._rascal_id, connection._rascal_id);
// See https://github.com/squaremo/amqp.node/issues/388
if (invocations > 1) {
debug('Closing superfluous channel: %s previously reported as errored', channel._rascal_id);
return channel.close();
}
once(null, channel);
}
}
function pauseChannelAllocation() {
channelCreator.pause();
channelPool.pause();
confirmChannelPool.pause();
}
function resumeChannelAllocation() {
channelCreator.resume();
channelPool.resume();
confirmChannelPool.resume();
}
function attachErrorHandlers(connection, config) {
connection.removeAllListeners('error');
var errorHandler = _.once(handleConnectionError.bind(null, connection, config));
connection.once('error', errorHandler);
connection.once('close', errorHandler);
}
function handleConnectionError(borked, config, err) {
debug('Handling connection error: %s initially from connection: %s, vhost:%s', err.message, borked._rascal_id, connectionConfig.loggableUrl);
self.emit('disconnect');
pauseChannelAllocation();
connection = undefined;
self.emit('error', err);
connectionConfig.retry && self.init(function(err) {
if (!err) return;
var delay = timer.next();
debug('Will attempt reconnection in in %dms', delay);
return setTimeout(handleConnectionError.bind(null, borked, config, err), delay).unref();
});
}
}
var lastKey = data.Contents[data.Contents.length - 1].Key
// override Marker, don't disturb original options
options = !options ? {} : combineObjects(options, {})
options['Marker'] = lastKey
s3.listObjects(options, lister)
} else {
callback(null, retls);
}
}
var req = s3.listObjects(options, lister)
}
var S3GetObjectGunzip = async.compose(function (data, callback) {
zlib.gunzip(data.Body, callback);
},
function (params, callback) {
params.s3.getObject(params.params, callback);
});
/**
* applies function to every ungzipped file in bucket returned by an s3 list operation
* mapper - function(fileName, fileContents, callback) that is applied...must call callback with error or result
*/
function S3MapBucket(s3, s3params, limit, mapper, callback) {
async.waterfall([ function (callback) {
S3ListObjects(s3, s3params,
function(err, ls) {
if (err)
return callback([err, s3params]);
var files = ls.filter(function (x) {return x.Size > 0})
var info = {};
info.source = 'qiyi';
info.urls = urls;
info.name = name;
callback(null,info);
}catch(err){
callback(new Error('JSON解析错误'));
}
}
}else{
callback(new Error('获取数据出错'));
}
});
}
var getVideo = async.compose(getM3U8, getVID);
var qiyi = function(req,res){
var handler = require('../handlers');
getVideo(req.query.url,function(err,result){
if(!err){
res.send(handler.success(type,req.query.url,result,result.name));
}else{
res.send(handler.fail(type,req.query.url,err));
}
});
}
module.exports = qiyi;
var notUndefined = function(xx) {return !uu.isUndefined(xx);};
var agen_urls = uu.filter(uu.pluck(register_data, 'json_url'), notUndefined);
cb(null, agen_urls);
}
/* 5. Pull down the body and parse JSON into a data structure (register_data) */
function register_url2register_data(register_url, cb){
log(arguments.callee.name);
var err_resp_body2register_data = function(error, response, body) {
if (!error && response.statusCode == 200) {
var register_data = JSON.parse(body);
cb(null, register_data);
}
};
request(register_url, err_resp_body2register_data);
}
/*bringing it together w/ async compose*/
function parentId_to_names2console(parentId_to_names) {
log(arguments.callee.name);
console.log(JSON.stringify(parentId_to_names, null));
}
//forces all request to complete before subsequence code was executed
var register_url2console = async.compose(agen_datas2parentId_to_names,
agen_urls2agen_datas,
register_data2agen_urls,
register_url2register_data);
var register_url = "https://www.federalregister.gov/api/v1/agencies.json";
register_url2console(register_url,parentId_to_names2console);