function promiseCallback(resolve, reject) {
cli.main(function(args, options) {
if (1 === args.length) {
return resolve(args[0]);
} else if (args.length > 1) {
return reject(new Error('only one input file can be specified'));
} else {
return resolve();
}
});
}
/*
* The purpose of this file is to run from the command line.
* This is great for integration testing without having to run
* from a build.
*/
var cli = require('cli');
var task = require('./task.js');
var util = require('./utility.js');
cli.parse({
cwd: ['c', 'Working directory', 'string', null],
baseDir: ['b', 'Folder to search', 'string', null],
warFilePath: ['w', 'Path to war file to use', 'string', null]
});
cli.main(function(args, options) {
if(options.warFilepath === null || options.cwd === null)
{
cli.getUsage();
return;
}
// Call the task
task.run(options.warFilePath, options.cwd, options.baseDir);
});
cli.parse({
connections: ['c', 'Single test for specified count of connections', 'int', '100'],
connectionsList: ['l', 'Multiple tests for specified list count of connections (-l 1,10,100,1000)', 'string'],
output: ['o', 'File to save JSON result', 'file']
});
cli.main(function (args, options) {
if (args.length !== 2) {
cli.fatal('Wrong number of arguments. Must be exactly 2 arguments! See `' + cli.app + ' -h` for details');
}
var connections;
if (options.connectionsList) {
connections = options.connectionsList.split(',');
multipleTest(args[0], args[1], connections, cli, function (result) {
if (options.output) {
writeJson(result);
}
});
} else {
test(args[0], args[1], options.connections, cli, function (result) {
if (options.output) {
writeJson([result]);
}
});
}
});
/*db.get(doc.id, {rev: doc.rev}, function(e, doc){
//res.send(doc);
})*/
})
logger.printData(name, data);
});
app.get('/logDemo', function (req, res) {
var name = 'a';
res.send(homepage_graph.replace('{{{{name}}}}', name));
});
app.get('/:name', function (req, res) {
var name = req.params.name || 'unknown';
res.send(homepage.replace('{{{{name}}}}', name));
});
cli.main(function (args, options) {
app.listen(options.port);
this.ok('Listening on port ' + options.port);
});
// option flags
cli.parse({
cwd: [ 'Current Working Directory', 'string' ],
requestProxy: [ 'rp', 'Request Proxy', 'string' ]
});
cli.main(function ( args, opts ) {
// consider making this dynamic off of file structure reads
// but for now just keep it simple in an array.
let definedModes = [ 'init', 'server' ];
cli.ok( 'Welcome to PatternGuide\'s Front-end Devlopment Workflow' );
// every mode class should have a `start` method
args.map(function ( mode ) {
if ( definedModes.indexOf( mode ) != -1 ) {
let Module = require( path.join( __dirname, 'modes', mode ) );
new Module().start();
}
});
// if no mode is passed, let's run the init mode
if ( !args.length ) {
let Init = require( path.join( __dirname, 'modes', 'init' ) );
cli.info( 'No mode passed, starting PatternGuide Init' );
new Init().start();
}
});
});
};
const ACTION_MAP = {
authenticate: createConfig,
create: createGitRepo,
open: openAppInBrowser
};
cli.parse({
authenticate: ['a', 'Authenticate to rnplay.org with a token'],
create: ['c', 'Create a git remote for this application'],
open: ['o', 'Opens the last created application in rnplay.org']
});
cli.main((args, options) => {
const action = ACTION_MAP[getFirstTrueOption(options)];
if (!action) {
cli.getUsage();
return;
}
action(cli)
.catch((e) => {
cli.error('Ooops, there has been an error: \n' + e.message);
cli.info('If you are sure that you did nothing wrong, please file an issue at the rnplay-cli repo!');
})
.finally(() => process.exit());
});
/*
* Describe the required options
*/
cli.parse(options, commands);
cli.main(function() {
if (!this.options.path) {
cli.fatal(msgs.argsError.noPath);
return;
}
var path = wpa.formatPath(this.options.path);
switch (this.command) {
case 'install':
wpa.install(path);
break;
case 'update':
wpa.update(path);
break;
default:
break;
}
});
#!/usr/bin/env node
const cli = require('cli');
const app = require('./package.json');
const ironclad = require('./ironclad');
cli.enable('version').enable('status');
cli.setApp(app.name, app.version);
cli.parse(ironclad.opts);
cli.main((args, opts) => {
ironclad.load(args, opts);
});
cli.main(function(args, options) {
function parseJSONOption(value) {
if (value !== null) {
var jsonArray;
try {
jsonArray = JSON.parse(value);
}
catch (e) {}
if (jsonArray instanceof Array) {
return jsonArray;
}
else {
return [value];
}
}
}
function runMinify(original) {
var status = 0;
var minified = null;
try {
minified = minify(original, minifyOptions);
}
catch (e) {
status = 3;
process.stderr.write('Error: Minification error');
}
if (minifyOptions.lint) {
minifyOptions.lint.populate();
}
if (minified !== null) {
// Write the output
try {
if (output !== null) {
fs.writeFileSync(path.resolve(output), minified);
}
else {
process.stdout.write(minified);
}
}
catch (e) {
status = 4;
process.stderr.write('Error: Cannot write to output');
}
}
cli.exit(status);
}
if (options.version) {
process.stderr.write(appName + ' v' + appVersion);
cli.exit(0);
}
if (options['config-file']) {
try {
var fileOptions = JSON.parse(fs.readFileSync(path.resolve(options['config-file']), 'utf8'));
if ((fileOptions !== null) && (typeof fileOptions === 'object')) {
minifyOptions = fileOptions;
}
}
catch (e) {
process.stderr.write('Error: Cannot read the specified config file');
cli.exit(1);
}
}
mainOptionKeys.forEach(function(key) {
var paramKey = changeCase.paramCase(key);
var value = options[paramKey];
if (options[paramKey] !== null) {
switch (mainOptions[key][1]) {
case 'json':
minifyOptions[key] = parseJSONOption(value);
break;
case true:
minifyOptions[key] = value;
break;
default:
minifyOptions[key] = true;
}
}
});
if (minifyOptions.lint === true) {
minifyOptions.lint = new HTMLLint();
}
if (args.length) {
input = args;
}
if (options.output) {
output = options.output;
}
if (input !== null) { // Minifying one or more files specified on the CMD line
var original = '';
input.forEach(function(afile) {
try {
original += fs.readFileSync(afile, 'utf8');
}
catch (e) {
process.stderr.write('Error: Cannot read file ' + afile);
cli.exit(2);
}
});
runMinify(original);
}
else { // Minifying input coming from STDIN
process.stdin.pipe(concat({ encoding: 'string' }, runMinify));
}
});
CLI.main(function (args, options) {
log.enableDebug(options.debug);
var project = args[0];
var releaseRequested = args[1] || 'latest';
if (project === 'all') {
releaseRequested = 'all';
project = null;
}
if (project === 'latest') {
releaseRequested = 'latest';
project = null;
}
if (!isNaN(project)) {
releaseRequested = parseInt(project, 10);
project = null;
log.debug('project: this one');
log.debug('releaseRequested: ' + releaseRequested);
}
if (!isNaN(releaseRequested, 10)) {
releaseRequested = parseInt(releaseRequested, 10);
log.debug('releaseRequested: ' + releaseRequested);
}
if (!project) {
try {
log.debug('Project not specified. Looking for a package.json in ' + process.cwd() + ' instead.');
project = JSON.parse(FS.readFileSync(process.cwd() + '/package.json').toString()).name;
} catch (e) {
log.debug('Package.json not found');
}
}
function generateOutput(data) {
var fn = options.json ? JSON.stringify
: options.markdown ? require('./output/markdown')
: hasColor ? require('./output/terminal')
: require('./output/markdown');
return fn(data);
}
changelog.generate(project, releaseRequested)
.then(generateOutput)
.then(console.info.bind(console))
.catch(function(err) {
if (typeof err === 'string') {
log.error(err);
} else {
throw err;
}
})
.done();
});
cli.main(function (args, options) {
var conf
try {
conf = require(options.config_path)[options.env]
if (!conf) {
throw 'Config file not found'
}
} catch (ex) {
console.log(ex);
cli.fatal('Config file not found. Using: ' + options.config_path)
}
var ordrinMongoConnectionString = 'mongodb://' + conf.mongo.user + ':' + conf.mongo.password + '@' + conf.mongo.host + ':' + conf.mongo.port + '/' + conf.mongo.db
var ordrinDb = mongoose.createConnection(ordrinMongoConnectionString);
var mongooseLayer = new VenueUtil.mongo(ordrinDb, {modelCollectionMapping:{
Scrape:options.scrapeSourceCollection,
VenueMeta:options.metaTargetCollection,
RestaurantMerged:options.restaurantSourceCollection
}});
var elasticSearchLayer = new VenueUtil.elasticsearch(conf.elastic_search);
var elasticSearchClient = elasticSearchLayer.createElasticSearchClient();
function getMoreRestaurants(start, num, cb) {
mongooseLayer.models.RestaurantMerged.find({'network_ids.name':'skyfetch_id'}, {}, {skip:start, limit:num, sort:{_id:-1}}, function (err, venues) {
cb(err, venues);
})
}
function getVenueScrapes(skyfetchId, cb) {
var query = {locationId:skyfetchId};
mongooseLayer.models.Scrape.find(query, {}, {sort:{_id:-1}}, function (err, metas) {
cb(err, metas);
})
}
async.waterfall([
function getSourceVenuesCount(cb) {
mongooseLayer.models.Venue.count(function (error, venueCount) {
cb(error, venueCount);
})
},
function processVenues(venueCount, cb) {
var start = 0;
var pageCount = Math.ceil(venueCount / options.pageSize);
var pages = [];
for (var i = 0; i < pageCount; i++) {
pages.push(i);
}
async.forEachLimit(pages, 1, function (pageNum, forEachPageCb) {
var start = pageNum * options.pageSize;
getMoreRestaurants(start, options.pageSize, function (err, venues) {
if (err || (!venues || venues.length == 0)) {
err = err ? err : 'No more venues.';
forEachPageCb(err);
} else {
async.forEach(venues, function (venue, forEachVenueCb) {
var skyfetchId = false;
async.waterfall([
function getScrapes(cb) {
if (!venue.network_ids || venue.network_ids.length == 0) {
cb(undefined, []);
} else {
for (var i = 0; i < venue.network_ids.length; i++) {
if (venue.network_ids[i].name && venue.network_ids[i].name == 'skyfetch_id') {
skyfetchId = venue.network_ids[i].id
break;
}
}
}
if (skyfetchId) {
getVenueScrapes(skyfetchId, function (scrapeError, scrapes) {
cb(scrapeError, venue, scrapes);
})
} else {
cb(scrapeError, venue, []);
}
},
function processScrapes(venue, scrapes, cb) {
var networkScrapes = {}
var ratings = [];
var totalRating = 0;
var reviews = [];
var menu = false;
if (scrapes && scrapes.length > 0) {
async.forEach(scrapes, function (scrape, forEachScrapeCb) {
//we want the most recent scrape for the given network.
if (!networkScrapes[scrape.network]) {
var sourceUrl = false;
if (scrape.data && scrape.data.url) {
sourceUrl = scrape.data.url;
} else if (scrape.params && scrape.params.url) {
sourceUrl = scrape.params.url;
}
var attribution = new Attribution(scrape.network, sourceUrl, scrape.updatedAt, scrape._id.toString());
networkScrapes[scrape.network] = scrape;
if (scrape.data) {
if (scrape.data.reviews) {
for (var i = 0; i < scrape.data.reviews.length; i++) {
var review = scrape.data.reviews[i];
var reviewObject = {
attribution:attribution
}
reviewObject.content = review.content ? review.content : null;
reviewObject.dtreviewed = review.dtreviewed ? review.dtreviewed : null;
reviewObject.author = review.author ? review.author : null;
if (review.rating) {
try {
reviewObject.rating = parseFloat(review.rating);
} catch (ex) {
}
}
reviews.push(reviewObject)
}
}
if (scrape.data.rating) {
try {
var ratingNumber = parseFloat(scrape.data.rating)
var rating = {
attribution:attribution,
rating:ratingNumber
}
ratings.push(rating);
totalRating += ratingNumber;
} catch (ex) {
}
}
if (scrape.data.ordrinMenu) {
menu = scrape.data.ordrinMenu;
menu.attribution = attribution;
}
}
}
forEachScrapeCb();
}, function (forEachError) {
venue.ratings = ratings;
venue.ratings_count = ratings.length;
venue.source_count = Object.keys(networkScrapes).length;
if (venue.ratings_count > 0) {
console.log(venue._id.toString());
try {
var average_rating = parseFloat(totalRating / venue.ratings_count).toFixed(2);
venue.average_rating = average_rating;
} catch (ex) {
}
}
venue.reviews = reviews;
venue.reviews_count = reviews.length;
if (menu) {
venue.menu = menu;
}
cb(forEachError, venue, networkScrapes);
});
} else {
cb(undefined, venue, networkScrapes)
}
},
function saveVenue(venue, networkScrapes, cb) {
venue.save(function (saveError, saveResult) {
cb(saveError, saveResult, networkScrapes);
})
},
function indexVenue(venue, networkScrapes, cb) {
var doc = venue._doc;
doc._id=doc._id.toString();
doc.id=doc._id;
elasticSearchClient.index(conf.ordrin_index_name, conf.venue_type_name,doc)
.on('data', function (data) {
console.log(data);
})
.on('done', function (done) {
console.log(done);
cb(undefined,venue, networkScrapes)
})
.on('error', function (error) {
cb(error,venue, networkScrapes)
})
.exec();
},
function saveMetas(venue, networkScrapes, cb) {
async.forEach(Object.keys(networkScrapes), function (network, forEachNetworkCb) {
var scrape = networkScrapes[network];
var venueMeta = {data:scrape.data || {}};
//venueMeta._id = scrape._id.toString();
venueMeta.venue_id = venue._id.toString();
venueMeta.pair_check_quality = scrape.quality || null;
venueMeta.last_pair_check = scrape.lastChecked || null;
venueMeta.network = scrape.network;
venueMeta.skyfetch_location_id = skyfetchId;
mongooseLayer.models.VenueMeta.findById(scrape._id, function (scrapeError, existingScrape) {
if (!existingScrape) {
venueMeta._id = scrape._id.toString();
new mongooseLayer.models.VenueMeta(venueMeta).save(function (saveError, saveResult) {
forEachNetworkCb(undefined, saveResult);
})
} else {
mongooseLayer.models.VenueMeta.update({_id:scrape._id}, venueMeta, function (updateError, updateResult) {
forEachNetworkCb(undefined, updateResult);
})
}
})
}, function (forEachError) {
cb(forEachError);
});
}
],
function (waterfallError) {
forEachPageCb(waterfallError);
}
)
}
,
function (forEachError) {
start += venues.length;
forEachPageCb(forEachError)
}
)
;
}
})
}
)
}
],
function (waterfallError, results) {
if (waterfallError) {
console.log(waterfallError);
}
process.exit(1);
}
)
})
var cli = require('cli'),
Agent = require('./agent');
cli.parse({
test: ['t', 'Check if the address of a server is in the TwemProxy config', 'bool', false],
host: ['h', 'Redis sentinel hostname', 'string', '127.0.0.1'],
port: ['p', 'Redis sentinel port number', 'number', 26379],
config: ['f', 'Path to twemproxy config', 'path', '/etc/redis/twemproxy.yml'],
command: ['c', 'Command to restart twemproxy', 'string', '/etc/init.d/nutcracker restart'],
log: ['l', 'The log file location', 'string', '/var/log/twemproxy_sentinel.log'],
warnsms: ['w', 'Command to send warn sms', 'string', '/data/redis-twemproxy-agent/bin/warnsms '],
});
cli.main(function (args, options) {
var config = { nutcracker_config_file: options.config,
redis_sentinel_ip: options.host,
redis_sentinel_port: options.port,
restart_command: options.command,
log_file: options.log,
warnsms_command: options.warnsms,
test: options.test };
Agent.bootstrap(config);
});
var fs = require('fs');
var watson = require('watson-developer-cloud');
var cli = require('cli');
var conversion = require('./docconversion');
var outputFile = 'solrdocs.json';
var options = cli.parse({
dir: [ 'i', 'directory to process', 'dir', null]
});
cli.main(function(args,options) {
if(options.dir == null) {
cli.getUsage();
exit();
}
});
conversion.init(outputFile);
var inputDirectory = __dirname + '/' + options.dir;
fs.readdir(inputDirectory,
function(err, list) {
if (err)
throw err;
list.forEach(function(currentValue, index, array){
var closeCb = function(o){
var conv = o;
if (authenticated) {
checkIp(proxy, function(err, ip) {
if (err) return util.log(err);
util.log("Your new identity is " + ip);
});
} else {
util.log('Successful authenticated');
authenticated = true;
util.log('Requesting new identity' + (!once ? ' each ' + timeout + 's' : ''));
}
if (once) return client.end('SIGNAL NEWNYM\r\n');
timer = setTimeout(function() {
client.write('SIGNAL NEWNYM\r\n');
}, timeout * 1000);
});
client.on('end', function() {
if (timer) clearTimeout(timer);
});
}
cli.parse({
proxy: ['x', 'Proxy address', 'string', 'localhost:9050'],
port: ['p', 'Control port', 'number', 9051],
timeout: ['t', 'Identity change interval in seconds', 'number', 10],
once: ['o', 'Change the identity only once', 'boolean', false]
});
cli.main(function(args, options) {
var password = args.pop();
run(parseInt(options.port), options.proxy, password, parseInt(options.timeout), options.once);
});
cli.main(function(_, options) {
console.log('');
var args = {};
Object.keys(options).forEach(function (key) {
var val = options[key];
if ('string' === typeof val) {
val = val.replace(/^~/, homedir);
}
key = key.replace(/\-([a-z0-9A-Z])/g, function (c) { return c[1].toUpperCase(); });
args[key] = val;
});
if (args.domains) {
args.domains = args.domains.split(',');
}
makeDirectories();
function makeDirectories() {
mkdirp(configDir, function (err) {
if (err) {
console.error("Could not create config directory '" + configDir + "':", err.code);
console.error(err.stack);
return;
}
mkdirp(vhostDir, function (err) {
if (err) {
console.error("Could not create vhost directory '" + vhostDir + "':", err.code);
console.error(err.stack);
return;
}
startServers();
});
});
}
function configure(le, args, cb) {
var vhost;
var pubDir;
var index;
if (!(args.email && args.agreeTos && args.server && args.domains)) {
cb({ error : { message: "missing one or more of agreeTos,domains,email,server" } });
return;
}
vhost = args.domains[0];
pubDir = path.join(vhostDir, vhost);
index = path.join(pubDir, 'index.html');
makeLandingPage();
function makeLandingPage() {
mkdirp(pubDir, function (err) {
if (err) {
cb(err);
return;
}
fs.exists(index, function (exists) {
if (exists) {
configureForHttps();
return;
}
fs.writeFile(path.join(pubDir, 'index.html'), welcomeHtml.replace(/:hostname/g, vhost), 'utf8', function (err) {
if (err) {
cb(err);
return;
}
configureForHttps();
});
});
});
}
function configureForHttps() {
if (args.debug) {
console.log('[LEX] configureForHttps');
console.log(args);
}
le.setConfig(args, cb);
}
}
function createConfigurator(le) {
var app = express();
app.use('/', express.static(path.join(__dirname, '..', 'lib', 'configurator')));
app.use(require('body-parser').json());
app.get('/api/com.daplie.lex/sites', function (req, res, next) {
le.getConfigs({ configDir: configDir }, function (err, configs) {
if (err) {
next(err);
return;
}
res.send(configs);
});
});
app.post('/api/com.daplie.lex/sites', function (req, res, next) {
var data = req.body;
configure(le, data, function (err, configs) {
if (err) {
console.error("[LEX/bin] configure");
console.error(err.stack);
next(err);
return;
}
res.send(configs);
});
});
return app;
}
function startServers() {
// Note: using staging server url, remove .testing() for production
var LE = require('letsencrypt');
var LEX = require('../');
var le = LE.create({
configDir: configDir
, manual: true
, privkeyPath: LE.privkeyPath
, fullchainPath: LE.fullchainPath
, certPath: LE.certPath
, chainPath: LE.chainPath
, renewalPath: LE.renewalPath
, accountsDir: LE.accountsDir
}, {
setChallenge: LEX.setChallenge
, removeChallenge: LEX.removeChallenge
});
var app = express();
var vhosts = {};
vhosts['localhost.daplie.com'] = createConfigurator(le, vhosts);
app.use('/', function (req, res, next) {
var hostname = (req.hostname||req.headers.host||'').replace(/^www\./, '');
var pubDir = path.join(vhostDir, hostname);
if (vhosts[hostname]) {
vhosts[hostname](req, res, next);
return;
}
fs.exists(pubDir, function (exists) {
if (exists) {
vhosts[hostname] = express().use('/', express.static(pubDir));
vhosts[hostname](req, res, next);
} else {
vhosts['localhost.daplie.com'](req, res, next);
}
});
});
app.use('/', express.static(path.join(__dirname, '..', 'lib', 'public')));
LEX.create({
onRequest: app
, configDir: configDir
, letsencrypt: le
, approveRegistration: function (domain, cb) {
le.getConfig({ domains: [domain] }, function (err, config) {
if (!(config && config.checkpoints >= 0)) {
cb(null, null);
return;
}
cb(null, {
email: config.email
// can't remember which it is, but the pyconf is different that the regular variable
, agreeTos: config.tos || config.agree || config.agreeTos
, server: config.server || LE.productionServerUrl
, domains: config.domains || [domain]
});
});
}
}).listen([80], [443, 5001]);
}
/*
// should get back account, path to certs, pems, etc?
console.log('\nCertificates installed at:');
console.log(Object.keys(results).filter(function (key) {
return /Path/.test(key);
}).map(function (key) {
return results[key];
}).join('\n'));
*/
});
cli.main(function (args, options) {
var conf
try {
conf = require(options.config_path)[options.env]
if (!conf) {
throw new Exception('Config file not found')
}
} catch (ex) {
cli.fatal('Config file not found. Using: ' + options.config_path)
}
var network = 'foursquare';
var redisClient = redis.createClient(conf.redis.port, conf.redis.host);
var ordrinMongoConnectionString = 'mongodb://' + conf.mongo.user + ':' + conf.mongo.password + '@' + conf.mongo.host + ':' + conf.mongo.port + '/' + conf.mongo.db
var dbConn = mongoose.createConnection(ordrinMongoConnectionString);
var mongooseLayer = new VenueUtil.mongo(dbConn, {modelCollectionMapping:{
Venue:'clean_venues',
"RestaurantMerged":"amex_venues"
}});
var foursquare = new FsqClient('TYR00ZTQ44Q5E0NOWHBJSZLXFRKK1MZUCF1NJ0WWPTK1IG01', 'SNT2F4NQC0QKP0DSRXFR1ZUV1PPA0LIUAGBNWO2S3N2XDLC3')
function updateScrape(scrape, res, updateScrapeCb) {
if (res && res.response && res.response.tips) {
var fsqVenue = res.response.venue;
scrape.data.fsqtips=res.response.tips;
scrape.markModified('data');
scrape.save(function(err,saved){
updateScrapeCb(err,saved);
})
//updateScrapeCb();
} else {
updateScrapeCb();
}
}
mongooseLayer.models.Scrape.find({network:'foursquare', 'data.fsqid':{$exists:true}}, {}, {sort:{updatedAt:1}}, function (err, scrapes) {
//mongooseLayer.models.Scrape.find({network:'foursquare', '_id':'50a2dea7e6bab542060000d4'}, {}, {sort:{updatedAt:1}}, function (err, scrapes) {
async.forEachSeries(scrapes, function (scrape, forEachScrapeCb) {
async.waterfall([
function getFsqVenue(getFsqVenueCb) {
var fsqid = scrape.data.fsqid;
foursquare.getTips({venue_id:fsqid}, function (fsqErr, fsqVenue) {
getFsqVenueCb(fsqErr, scrape, fsqVenue);
});
},
function fsqVenue(scrape, fsqVenue, fsqVenueCb) {
if(fsqVenue){
console.log('updating '+scrape.data.name);
}
updateScrape(scrape, fsqVenue, fsqVenueCb);
}
], function (waterfallError, results) {
forEachScrapeCb(waterfallError);
})
}, function (forEachError) {
if (forEachError) {
console.log(forEachError);
}
console.log('done');
process.exit(0);
});
})
})
awsms.getStackId(options).then(function(data){
console.log(data);
});
}
};
cli.parse({
log: ['l', 'Enable logging'],
file: ['f', 'JSON file with request parameters', 'file'],
serve: [false, 'Serve static files from PATH', 'path', './public'],
region: ['r', "Region", "string"],
params: ['p', "Parameters as json string", "string"]
}, ['createInstance', 'listMetrics', 'getMetrics', 'updateDependencies', 'getStackId']);
cli.main(function(args, options) {
var params = {};
try{
cscmscli[cli.command](options);
}catch(err){
console.log("main error");
for (var name in err) {
if (err.hasOwnProperty(name)) {
console.log(name);
}
}
cli.fatal(err);
}
});
short: [ 's', 'Print short output (user-agents only)' ],
ignore: [ 'i', 'Ignore debug information from `regexes.yaml`' ],
});
cli.main(function(args, options) {
options.out = path.resolve(__dirname, ( options.out || 'report/uniq.csv'));
if (! options.ua) {
cli.error('need -u as option');
return;
}
if (! options.type) {
cli.error('need -t as option');
return;
}
if ( ! /^(ua|os|device)$/.test(options.type) ) {
cli.error('only -t [ua|os|device] allowed');
return;
}
filter(options,
path.resolve(__dirname, options.ua),
options.out,
function(err, data){
if (options.tree) {
fs.writeFileSync(__dirname + '/report/tree.json', JSON.stringify(data, null, 2));
}
cli.ok('writing output to "'+ options.out +'"');
});
});
var cli = require("cli");
cli.parse({
random_stream: ["rs", "random stream", "path", "/dev/urandom"],
csv: ["c", "csv file path", "path", "./testtable01.csv"],
number: ["n", "number of lines", "number", 10]
});
cli.main(function(args, options) {
var fs = require("fs");
var count = 0;
var fdR = fs.openSync(options.random_stream, "r");
var fdW = fs.openSync(options.csv, "w");
var buffer = new Buffer(150);
while (count < options.number) {
fs.readSync(fdR, buffer, 0, 150, null);
var s = buffer.toString('base64');
var line = (count + "," + s + "\n");
var bufW = new Buffer(line);
fs.writeSync(fdW, bufW, 0, bufW.length, null);
count++;
} // end of while
}); // end of cli.main
cli.main(function(args, options) {
var js = [];
var queue = [ relative_path ];
while (queue.length > 0) {
var dir = queue.shift();
try {
var ls = fs.readdirSync(base + goog_dir + "/" + dir);
// goog.array.forEach(ls, function(file) {
for(var i = 0;i<ls.length;i++){
var file = ls[i];
if (file.match(/.*\.js$/i)) {
js.push(dir + "/" + file);
} else {
queue.push(dir + "/" + file);
}
}
// });
} catch (e) {
console.log("load error " + e);
}
}
console.log("target js :" + js);
require('child_process').exec(depswriter + " " + js.join(" "), {
cwd : base + goog_dir,
maxBuffer : 1024 * 1024
}, function(err, stdout, stderr) {
if (err) {
throw err;
}
if (stdout) {
console.log("create " + base + "deps.js");
fs.writeFileSync(base + "/deps.js", stdout);
}
if (stderr) {
console.err(stderr);
}
});
});
cli.main(function (args, options) {
var conf
try {
conf = require(options.config_path)[options.env]
if (!conf) {
throw new Exception('Config file not found')
}
} catch (ex) {
cli.fatal('Config file not found. Using: ' + options.config_path)
}
var redisClient = redis.createClient(conf.redis.port, conf.redis.host);
var jobQueue = new JobQueue('insiderpagesdetails', {redisClient:redisClient})
var ordrinMongoConnectionString = 'mongodb://' + conf.mongo.user + ':' + conf.mongo.password + '@' + conf.mongo.host + ':' + conf.mongo.port + '/' + conf.mongo.db
var dbConn = mongoose.createConnection(ordrinMongoConnectionString);
var mongooseLayer = new VenueUtil.mongo(dbConn, {modelCollectionMapping:{
Venue:'clean_venues',
"RestaurantMerged":"amex_venues"
}});
var attributedFields = {
//FIELD NAME IS THE SCRAPE.DATA.FIELDNAME
//VALUE IS THE TARGET FIELD
"onlinereservations":"onlinereservations",
"open24hours":"24hours",
"openkitchens/watchthechef":"openkitchens",
"openlate":"openlate",
"peoplewatching":"peoplewatching",
"pre/posttheater":"preposttheater",
"privatedining/partyspace":"privatedining",
"prixfixe":"prixfixe",
"rawbar":"rawbar",
"reservationsnotrequired":"reservationsnotrequired",
"romantic":"romantic",
"smokingarea":"smokingarea",
"specialoccasion":"specialoccasion",
"tastingmenu":"tastingmenu",
"teatime":"teatime",
"teenappeal":"teenappeal",
"theaterdistrict":"theaterdistrict",
"trendy":"trendy",
"view":"goodviews",
"waterfront":"waterfront",
"description":"description",
"Business Lunch":"businesslunch",
"Celeb-Spotting":"celebspotting",
"Dine at the Bar":"dineattheBar",
"Hot Spot":"hotspot",
"Prix-Fixe":"prixfixe",
"Online Reservations":"onlinereservations",
"Bar Scene":"barscene",
"Wheelchair Friendly":"wheelchair",
"Lunch Special":"lunchspechial",
'menuJson':'menujson',
'menuUrl':'menuurl',
'priceString':'priceString',
'fsqphrases':'fsqphrases',
'fsqstats':'fsqstats',
'fsqmenu':'menuurl',
"24hours":"24hours",
"byob":"byob",
"barscene":"barscene",
"brunchdaily":"brunchdaily",
"buffet":"buffet",
"businessdining":"businessdining",
"businesslunch":"businesslunch",
"celebspotting":"celebspotting",
"cheapeats":"cheapeats",
"classicny":"classicny",
"deliveryafter10pm":"deliveryafter10pm",
"designstandout":"designstandout",
"dineatthebar":"dineatthebar",
"familystyle":"familystyle",
"fireplace":"fireplace",
"foodtruck/cart":"foodtruck",
"glutenfreeitems":"glutenfreeitems",
"greatdesserts":"greatdesserts",
"happyhour":"happyhour",
"hotspot":"hotspot",
"kidfriendly":"kidfriendly",
"kidsmenu":"kidsmenu",
"latenightdining":"latenightdining",
"liveentertainment":"liveentertainment",
"livemusic":"livemusic",
"lunchspecial":"lunchspecial",
"notablechef":"notablechef",
"notablewinelist":"notablewinelist",
"onlineordering":"onlineordering",
'neighborhoods':'neighborhoods',
'transportation':'transportation',
'attire':'attire',
'parking':'parking',
'goodformeal':'goodformeal',
'alchohol':'alchohol',
'ambience':'ambience',
'noiselevel':'noiselevel',
'creditcards':'creditcards',
'delivery':'delivery',
'groups':'groups',
'kids':'kids',
'reservations':'reservations',
'takeout':'takeout',
'tableservice':'tableservice',
'outdoorseating':'outdoorseating',
'wifi':'wifi',
'tv':'tv',
'caters':'caters',
'wheelchair':'wheelchair',
'goodviews':'goodviews',
'closed':'closed',
'website':'website',
// 'features':'features',
'fsqcategories':'categories',
'cuisine':'cuisine',
'categories':'categories',
'tags':'categories',
};
var unattributedFields = {
'website':'website',
'email':'email',
'zip':'postal_code',
'phone':'restaurant_phone',
'crossStreet':'cross_street',
'cc':'country_code',
fsqcatids:'fsqcatids',
fsqphotos:'fsqphotos',
fsqspecials:'fsqspecials'
}
/**
* fsqlikes to ratings: can't go into likes as it would mess up ratings calculatioons.
* I think that there should be a seperate likes field, that would ultimate take fb likes, fsq likes, google +1s.
*
* 'tips':'tips',
'fsqtips':'tips', to reviews
'fsqcategories':'fsqcategories', to categories
'images'
verify menu
*/
var subjectiveFields = [
'description', 'hotspot', 'celebspotting', 'businesslunch', 'trendy', 'goodviews', 'teenappeal', 'romantic', 'peoplewatching',
'preposttheater', 'openlate', 'notablewinelist', 'notablechef', 'latenightdining', 'kidfriendly', 'greatdesserts',
'designstandout', 'classicny', 'cheapeats', 'businessdining', 'barscene', 'menuurl', 'fsqphrases', 'neighborhoods',
'attire', 'goodformeal', 'ambience', 'noiselevel', 'groups', 'kids', 'fsqstats', 'fsqphrases'
];
var objectiveFields = [
'categories', 'cuisine', 'hours', 'creditcards', 'delivery', 'reservations', 'takeout', 'tableservice', 'outdoor',
'wifi', 'tv', 'caters', 'wheelchair', 'transport', 'parking', 'alcohol', 'lunchspechial',
'prixfixe', 'dineattheBar', 'waterfront', 'theaterdistrict', 'teatime', 'tastingmenu',
'smokingarea', 'reservationsnotrequired', 'rawbar', 'privatedining', 'openkitchens', '24hours',
'onlinereservations', 'onlineordering', 'livemusic', 'liveentertainment', 'kidsmenu', 'happyhour',
'glutenfreeitems', 'foodtruck', 'fireplace', 'familystyle', 'deliveryafter10pm', 'buffet', 'brunchdaily',
'alchohol', 'fsqcatids', 'byob', 'transportation', 'outdoorseating', 'wifi', 'caters', 'menujson', 'closed'
]
var booleanFields = [
'hotspot', 'celebspotting', 'businesslunch', 'trendy', 'goodviews', 'teenappeal', 'romantic', 'peoplewatching',
'preposttheater', 'openlate', 'notablewinelist', 'notablechef', 'latenightdining', 'kidfriendly', 'greatdesserts',
'designstandout', 'classicny', 'cheapeats', 'businessdining', 'barscene',
'groups', 'delivery', 'takeout', 'tableservice', 'outdoor',
'wifi', 'tv', 'caters', 'wheelchair', 'transport', 'parking', 'alcohol', 'lunchspechial',
'prixfixe', 'dineattheBar', 'waterfront', 'theaterdistrict', 'teatime', 'tastingmenu',
'smokingarea', 'reservationsnotrequired', 'rawbar', 'privatedining', 'openkitchens', '24hours',
'onlinereservations', 'onlineordering', 'livemusic', 'liveentertainment', 'kidsmenu', 'happyhour',
'glutenfreeitems', 'foodtruck', 'fireplace', 'familystyle', 'deliveryafter10pm', 'buffet', 'brunchdaily',
'byob', 'outdoorseating', 'wifi', 'caters'
]
var mergeArrayFields = [
'cuisine', 'categories', 'goodformeal'
]
var pageSize = 20;
var done = 0;
//50
var query = {excluded:{$ne:true}};
if (options._id && options._id != 'all') {
query['_id'] = options._id.toString();
}
console.log(query);
function getRestaurants(start, num, cb) {
//mongooseLayer.models.RestaurantMerged.find({_id:'50a0b6b396be3c4a33022ac9',excluded:{$ne:true}}, {}, {skip:start, limit:num, sort:{updated_at:-1}}, function (err, restaurants) {
mongooseLayer.models.RestaurantMerged.find(query, {}, {skip:start, limit:num, sort:{updated_at:-1}}, function (err, restaurants) {
cb(err, restaurants);
})
}
function setFieldValue(field, value, network, restaurant) {
var isBoolean = false;
var isObjective = false;
var isArrayMerge = false;
restaurant.attributes = restaurant.attributes ? restaurant.attributes : {};
restaurant.tags = restaurant.tags ? restaurant.tags : {};
if (field && mergeArrayFields.indexOf(field) > -1) {
isArrayMerge = true;
}
if (field && booleanFields.indexOf(field) > -1) {
isBoolean = true;
if (typeof value != 'undefined') {
value = isBoolean ? true : value;
}
}
if (field && objectiveFields.indexOf(field) > -1) {
isObjective = true;
}
if (value != undefined && isArrayMerge) {
var existingValueArray = [];
if (isObjective && restaurant.attributes[field]) {
existingValueArray = restaurant.attributes[field];
} else if (restaurant.tags[field]) {
existingValueArray = restaurant.tags[field];
}
if (Object.prototype.toString.call(existingValueArray) !== '[object Array]') {
existingValueArray = [existingValueArray];
}
if (Object.prototype.toString.call(value) !== '[object Array]') {
existingValueArray.push(value);
} else {
existingValueArray = existingValueArray.concat(value);
}
var normedValues = {}
//dedupe values, should add synonyms...
var dedupedValues = [];
for (var i = 0; i < existingValueArray.length; i++) {
if (typeof existingValueArray[i] != 'object') {
var normed = existingValueArray[i].toLowerCase().replace(/(\r\n|\n|\r|\n\t|\t)/gm, " ").replace(/ +/g, ' ').trim();
if (!normedValues[normed]) {
dedupedValues.push(existingValueArray[i].trim());
normedValues[normed] = true;
}
} else {
dedupedValues.push(existingValueArray[i]);
}
}
value = dedupedValues;
}
if (value != undefined && isObjective) {
if (Object.prototype.toString.call(value) !== '[object Array]') {
restaurant.attributes_count++;
}
restaurant.attributes[field] = value;
restaurant.tags[field] = null;
delete restaurant.tags[field];
console.log('set attributes.' + field + ' to ' + value);
}
else if (value != undefined) {
if (Object.prototype.toString.call(value) !== '[object Array]') {
restaurant.tags_count++;
}
restaurant.attributes[field] = null;
delete restaurant.attributes[field];
restaurant.tags[field] = value;
console.log('set tags.' + field + ' to ' + value);
}
if (value) {
restaurant.feature_attributions.push({
attribution:new Attribution(network.network, network.params.url, network.createdAt, network._id),
field_name:field
})
}
return restaurant;
}
function handleFeatureField(sourceField, destField, restaurant, network) {
if (network.data) {
var data = false;
if (sourceField == 'features') {
if (network.data.features) {
data = network.data.features;
}
} else {
data = network.data;
}
if (data) {
if (network.network == 'foursquare') {
network.params.url = 'https://foursquare.com/v/' + network.data.fsqid;
}
if (Object.prototype.toString.call(data) === '[object Array]') {
for (var i = 0; i < data.length; i++) {
var field = attributedFields[data[i]];
restaurant = setFieldValue(field, data[i], network, restaurant);
}
} else {
var field = attributedFields[sourceField];
restaurant = setFieldValue(field, network.data[sourceField], network, restaurant);
}
}
}
return restaurant;
}
function handleFeatureFieldOld(sourceField, destField, restaurant, network) {
if (network.data) {
if (sourceField == 'features') {
/*var features = network.data[sourceField];
for (var f2 in features) {
restuarant = handleFeatureField(f2, f2, restaurant, network);
}*/
var features = network.data.features;
for (var i = 0; i < features.length; i++) {
var field = attributedFields[features[i]];
if (field && !restaurant.features[field]) {
restaurant.features[field] = true;
}
if (!field) {
console.log(field);
}
}
} else {
if (network.network == 'foursquare') {
network.params.url = 'https://foursquare.com/v/' + network.data.fsqid;
}
if (!restaurant.features[destField] && network.data[sourceField]) {
//console.log(restaurant._id);
restaurant.features[destField] = network.data[sourceField];
restaurant.feature_attributions.push({
attribution:new Attribution(network.network, network.params.url, network.createdAt, network._id),
field_name:destField
})
var t = 1;
} else {
for (var j = 0; j < restaurant.feature_attributions.length; j++) {
if (restaurant.feature_attributions[j].field_name == destField) {
if (restaurant.feature_attributions[j].attribution.date.getTime < network.createdAt.getTime()) {
restaurant.features[destField] = network.data[sourceField];
restaurant.feature_attributions[j].attribution = new Attribution(network.network, network.params.url, network.createdAt, network._id)
}
}
}
}
}
}
return restaurant;
}
mongooseLayer.models.RestaurantMerged.count(query, function (err, total) {
async.whilst(function () {
console.log(done + ' of ' + total);
return done <= total;
}, function (wCb) {
console.log('get restaurants');
getRestaurants(done, pageSize, function (err, restaurants) {
if (err) {
console.log(err);
wCb();
} else {
async.forEachLimit(restaurants, 5, function (restaurant, forEachRestaurantCallback) {
restaurant.tags_count = 0;
restaurant.attributes_count = 0;
async.waterfall([
function getScrapes(cb) {
// console.log(restaurant._id);
console.log('get scrapes');
mongooseLayer.models.Scrape.find({locationId:restaurant._id}, {}, {sort:{createdAt:-1}}, function (err, scrapes) {
cb(err, scrapes);
})
},
function chooseScrapes(scrapes, cb) {
console.log('choose scrapes');
var networkMap = {};
for (var i = 0; i < scrapes.length; i++) {
var scrape = scrapes[i];
if (!networkMap[scrape.network]) {
networkMap[scrape.network] = scrape;
} else {
//Take the scrape with the most reviews, otherwise most recent
var selected = false;
if (scrape.data.menuJson || networkMap[scrape.network].data.menuJson) {
if (scrape.data.menuJson && networkMap[scrape.network].data.menuJson) {
selected = networkMap[scrape.network].createdAt.getTime() > scrape.createdAt.getTime() ? networkMap[scrape.network] : scrape;
} else {
selected = networkMap[scrape.network].data.menuJson ? networkMap[scrape.network] : scrape;
}
}
if (!selected) {
if (networkMap[scrape.network].reviews && scrape.data.reviews) {
if (networkMap[scrape.network].reviews.length == scrape.data.reviews.length) {
selected = networkMap[scrape.network].createdAt.getTime() > scrape.createdAt.getTime() ? networkMap[scrape.network] : scrape;
} else {
selected = networkMap[scrape.network].reviews.length > scrape.data.reviews.length ? networkMap[scrape.network] : scrape;
}
} else {
if (scrape.data.reviews) {
selected = scrape;
} else {
selected = networkMap[scrape.network].createdAt.getTime() > scrape.createdAt.getTime() ? networkMap[scrape.network] : scrape;
}
}
}
networkMap[scrape.network] = selected ? selected : networkMap[scrape.network];
}
}
var networks = [];
for (var n in networkMap) {
console.log(networkMap[n]._id);
networks.push(networkMap[n]);
}
networks = networks.sort(function (a, b) {
return b.createdAt.getTime() - a.createdAt.getTime();
})
cb(undefined, networks, networkMap);
},
function mergeScrapes(networks, networkMap, cb) {
console.log('merge scrapes');
if (!restaurant.features) {
restaurant.features = {}
}
async.forEachSeries(networks, function (network, forEachSeriesCb) {
//for (var i = 0; i < attributedFields.length; i++) {
for (var f in attributedFields) {
//var field = attributedFields[i];
if (network.data[f]) {
if (f == 'menuJson') {
restaurant.hasFeatureMenu = true;
}
if (f == 'fsqmenu') {
var url = network.data[f].url;
network.data[f] = url;
}
if (f == 'fsqcategories') {
var simpleCats = [];
for (var i = 0; i < network.data[f].length; i++) {
simpleCats.push(network.data[f][i].shortName);
}
network.data[f] = simpleCats;
restaurant.fsqcategories = network.data[f];
}
if (f == 'goodformeal') {
if (typeof network.data[f] == 'string') {
network.data[f] = network.data[f].split(',');
}
}
}
restaurant = handleFeatureField(f, attributedFields[f], restaurant, network);
}
for (var field in unattributedFields) {
if (network.data[field]) {
if (!restaurant._doc[field] && network.data[field]) {
var f1 = unattributedFields[field];
var v1 = network.data[field];
restaurant[f1] = v1;
restaurant.markModified(unattributedFields[field]);
}
}
}
restaurant.feature_count = 0;
for (var field in restaurant.features) {
restaurant.feature_count++;
}
if (network.data.closed) {
restaurant.closed = true;
}
forEachSeriesCb(undefined);
}, function (forEachError) {
cb(forEachError, networks, networkMap);
});
},
function updateReviews(networks, networkMap, cb) {
console.log('update reviews');
//restaurant.reviews = [];
var reviews = [];
var rContent = [];
var uRateCount = 0;
var uRateSum = 0;
for (var i = 0; i < networks.length; i++) {
var network = networks[i];
if (network.data.reviews) {
var networkAttribution = new Attribution(network.network, network.data.url, network.createdAt, network._id);
for (var j = 0; j < network.data.reviews.length; j++) {
var review = network.data.reviews[j];
try {
if (review.dtreviewed) {
if (typeof review.dtreviewed == 'string') {
var d = new Date(review.dtreviewed);
review.dtreviewed = d;
}
}
} catch (ex) {
review.dtreviewed = null;
}
review.attribution = networkAttribution;
if (rContent.indexOf(review.content) == -1) {
reviews.push(review);
rContent.push(review.content);
if (review.rating) {
try {
uRateSum += parseFloat(review.rating);
uRateCount++;
} catch (ex) {
}
;
}
}
}
}
restaurant.user_ratings_count = uRateCount;
if (uRateCount > 0) {
restaurant.average_user_rating = uRateSum / uRateCount;
}
if (network.data.fsqtips) {
var attribUrl = 'https://foursquare.com/v/' + network.data.fsqid;
var networkAttribution = new Attribution(network.network, attribUrl, network.createdAt, network._id);
restaurant.fsq_tips_count = network.data.fsqtips.count;
if (network.data.fsqtips.items) {
for (var j = 0; j < network.data.fsqtips.items.length; j++) {
var tip = network.data.fsqtips.items[j];
var author = tip.user.firstName ? tip.user.firstName : false;
author = tip.user.lastName ? (author ? author + ' ' : '') + tip.user.lastName : author;
author = author.trim();
var review = {
content:tip.text,
dtreviewed:tip.createdAt,
attribution:networkAttribution,
source_meta:tip
}
if (author) {
review.author = author;
}
if (tip.likes.count) {
reviews.likes = tip.likes.count;
}
reviews.push(review);
rContent.push(review.content);
}
}
}
}
reviews = reviews.sort(function (a, b) {
if (a.dtreviewed && b.dtreviewed) {
try {
return b.dtreviewed.getTime() - a.dtreviewed.getTime();
} catch (ex) {
return -1;
}
} else {
return -1;
}
})
restaurant.reviews_count = reviews.length;
restaurant.reviews = reviews;
cb(undefined, networks, networkMap)
},
function updateRatings(networks, networkMap, cb) {
console.log('update ratings');
var ratingTotal = 0;
var ratingCount = 0;
var rating = false;
restaurant.ratings = [];
for (var i = 0; i < networks.length; i++) {
var network = networks[i];
if (network.data.rating) {
try {
ratingTotal += parseFloat(network.data.rating);
restaurant.ratings.push({
rating:network.data.rating,
attribution:new Attribution(network.network, network.data.url, network.createdAt, network._id)
})
ratingCount++;
} catch (ex) {
console.log(ex);
}
}
}
if (ratingCount) {
rating = ratingTotal / ratingCount;
restaurant.ratings_count = ratingCount;
restaurant.average_rating = rating;
} else {
restaurant.average_rating = -1;
}
if (restaurant.average_rating > 5) {
var avg = restaurant.average_rating.toString();
console.log('rating wtf. ');
}
cb(undefined, networks, networkMap);
},
function updateRestaurantNetworks(networks, networkMap, cb) {
console.log('update rest networks')
var network_ids = [];
for (var i = 0; i < networks.length; i++) {
var network = networks[i];
network_ids.push({
name:network.network,
id:network._id,
scrapedAt:network.createdAt
});
}
restaurant.source_count = restaurant.network_ids.length;
restaurant.network_ids = network_ids;
restaurant.markModified('network_ids');
cb(undefined, networks, networkMap);
},
function saveRestaurant(networks, networkMap, cb) {
console.log('save rest')
//restaurant.features.menuJson = restaurant.features.menuJson ? true : false;
restaurant.markModified('attributes');
restaurant.markModified('tags');
restaurant.markModified('features');
restaurant.markModified('feature_attributions');
restaurant.save(function (err, restSaveRes) {
//console.log(restSaveRes.cross_street);
cb(err, restSaveRes);
})
//cb(err, undefined);
}
],
function (waterfallError, results) {
done++;
forEachRestaurantCallback(waterfallError)
}
)
}
,
function (forEachError) {
if (forEachError) {
console.log(forEachError);
}
wCb(forEachError);
}
)
;
}
})
}, function (wErr) {
if (wErr) {
console.log(wErr);
}
console.log('done ' + done);
process.exit(1);
})
})
}
)
cli.main(function(args, options)
{
var app = {set: function(v) { }}
, c = this
, from = options.from
, to = options.to
, dry = options.dry
;
if (!to || !from) {
cli.error('"to" and "from" options are required.\n');
cli.output(cli.getUsage());
cli.exit(1);
return ;
}
var mongoUri = process.env.MONGOLAB_URI || process.env.MONGOHQ_URL || process.env.MONGO_URI || false;
if (!mongoUri) {
cli.error('Please set MONGO_URI env.\n');
cli.output(cli.getUsage());
cli.exit(1);
return;
}
mongo.connect(mongoUri);
// あー config 読み込み＆model読み込み周りを app.js から切り離さないといけないにゃぁ
configModel = require('../lib/models/config')(app);
async.series([
function (next) {
configModel.loadAllConfig(function(err, doc) {
return next();
});
}, function (next) {
var config = app.set('config');
models = require('../lib/models')(app);
models.Config = configModel;
return next();
}, function (next) {
var limit = 100000;
c.spinner('Load revisions..');
models.Revision.find().limit(limit).exec(function(err, revs) {
c.spinner('Load revisions.. done!\n', true);
var count = Object.keys(revs).length
, i = 0
, matched = 0
, matchedWords = 0
;
c.output('Found ' + count + ' revisions.\n');
c.output('Start replacing.\n');
async.each(revs, function(rev, cb) {
var regexp = new RegExp(from, 'g');
c.progress(++i/count);
var m = rev.body.match(regexp);
if (!m) {
return cb();
}
matched++;
matchedWords += m.length;
if (dry) {
return cb();
} else {
rev.body = rev.body.replace(regexp, to);
rev.save(function(err, s) {
if (err) {
c.error('Error on:' + rev.path);
} else {
}
return cb();
});
}
}, function(err) {
if (dry) {
cli.info(matchedWords + ' words in (' + matched + ' of ' + count + ') revisions will be replaced!');
} else {
cli.ok(matchedWords + ' words in (' + matched + ' of ' + count + ') revisions replaced!');
}
return next();
});
});
}
, function (next) {
cli.ok('Finished!');
mongo.disconnect();
return next();
}
]);
});
cli
.main(function(args, options) {
var self = this;
if (options.version) {
return console.log(process.title + ' v' + VERSION);
} else if (args.length == 0 && !options.host) {
self.fatal('Missing uri option');
}
self.spinner('Working..');
var backup = require('mongodb-backup');
try {
backup({
uri: args[0] || options.host,
root: options.out,
parser: options.parser,
collections: options.collections ? JSON.parse(options.collections) : null,
tar: options.tar,
query: options.query ? JSON.parse(options.query) : null,
logger: options.verbose,
metadata: options.metadata,
callback: function() {
self.spinner('Working.. done\n', true);
}
});
} catch (e) {
self.spinner('Working.. error\n', true);
console.error(e.message);
}
});
});
cli.main(function(args, opts) {
console.log('Hipsell Server - Test Framework');
console.log('');
//run the test server
var uri = url.parse(settings.serverUri);
server = spawn('node', ['main.js', '--mode=test', '--dbname=test', '--noemail', '--port=' + uri.port, '--host=0.0.0.0']);
server.stderr.on('data', function(data) {
if (opts.showserver)
process.stdout.write(data.toString().grey.inverse);
});
server.stdout.on('data', function(data) {
if (opts.showserver)
process.stdout.write(data.toString().grey);
//run the tests when the server is ready
if (data.toString().match(/Server Ready/)) test();
});
server.on('exit', function() {
console.log('');
console.log('Unexpected Test Server Exit'.red.inverse);
if (!opts.showserver) console.log('Run with -s to show server output'.red.inverse);
console.log('');
process.exit();
});
});
cli.main(function (args, options) {
/**
* Run setup if -s or .itsconf is missing
*/
var configuration = new config;
if (options.setup || !configuration.exists()) {
setup();
} else {
/**
* Load the configuration
*/
configuration.load();
/**
* Initiate the itslearning-client
*/
var client = new itslearning();
client.setCredentials(
configuration.getField('username'),
configuration.getField('password')
);
/**
* Initiate the authentication driver
*/
client.setAuthenticationDriver(
require('./drivers/'+ configuration.getField('driver') +'.js')
);
/**
* Authenticate with the client & fetch data
*/
client.authenticate(function () {
/**
* List messages in inbox
*/
if (options.inbox) {
client.fetchMessages(function () {
console.log(client.inboxTable());
});
}
/**
* Read message in inbox
*/
if (options.message) {
client.fetchMessage(options.message, function (message) {
client.printMessage(message);
});
}
/**
* List notifications
*/
if (options.notifications) {
client.fetchNotifications(function () {
console.log(client.notificationTable());
});
}
/**
* List courses
*/
if (options.courses) {
client.fetchCourses(function () {
console.log(client.courseTable());
});
}
/**
* List bulletins
*/
if (options.bulletins) {
client.fetchBulletins(options.bulletins, function () {
console.log(client.bulletinsTable(options.bulletins));
});
}
}, couldNotAuthenticate);
}
});
cli.main(function(args, options) {
if (options.clear === true) {
documentService.cleanDocumentsAsync({
projectName: options.project ? options.project : undefined,
collectionName: options.collection
})
.then(function(res) {
console.log('Collection items has been cleaned');
process.exit();
});
} else if (options.export === true) {
importService.exportAsync({
projectName: options.project,
collectionName: options.collection,
limit: options.limit
})
.then(function(res) {
console.log('Data has been exported to ./data/exports/collections.json');
process.exit();
});
} else if (options.elasticsearch === true) {
if (options.import === true) {
importService.importElasticTypeMappingAsync({
index: options.index,
type: options.type
})
.then(function(res) {
console.log('type from elasticsearch has been imported to local database');
process.exit();
});
} else if (options.indices === true) {
elasticStats.getIndicesAsync()
.then(function(res) {
console.log(res);
process.exit();
});
} else if (options.mapping === true) {
elasticMapping.getMappingAsync({
index: options.index,
type: options.type
})
.then(function(res) {
console.log(JSON.stringify(res, null, 4));
process.exit();
});
}
} else if (options.import === true) {
var filename = options.file || './data/movies.json';
importService.import({
collectionName: options.collection,
bulkSize: 200,
body: JSON.parse(fs.readFileSync(filename, 'utf8'))
}, function(err, res) {
console.log('data has been imported');
console.log(res);
process.exit();
})
}
});
cli.main(function(args, options) {
const util = require('./src/util')(options);
// Check arguments and grab version
if (args.length === 0) error.fatal('ERR_NO_VERSION');
if (args.length > 1) error.fatal('ERR_TOO_MANY_ARGS');
fs.readFile(PACKAGE_PATH, (err, content) => {
if (err) error.fatal('ERR_CANT_OPEN_PACKAGE_JSON');
// Parse the package.json
const packageJSON = JSON.parse(content);
if (packageJSON.version === undefined) error.fatal('ERR_NO_VERSION_ON_JSON');
const isBump = ['major', 'minor', 'patch'].indexOf(args[0]) > -1;
const version = isBump? semver.inc(packageJSON.version, args[0]) : args[0];
// Check version validity
if (!semver.valid(version)) error.fatal('ERR_VERSION_NOT_SEMVER_COMPLIANT', version);
if (packageJSON.version === version) error.fatal('ERR_SAME_VERSION', packageJSON.version);
if (semver.lt(version, packageJSON.version)) error.fatal('ERR_LOWER_VERSION', packageJSON.version);
// Asks if the user is sure about what he is doing
prompter.prompt(util.replace(msgs.prompt.areYouSure, [packageJSON.version, version]), ['y', 'n']).then(r => {
if (r === 'n') process.exit(0);
git.status((err, status) => {
if (err) error.fatal('ERR_GIT');
if (!status.isClean() && !options.force) error.fatal('ERR_UNCOMMITTED_CHANGES');
const branch = status.current;
const mustBeBranch = (packageJSON.publish && packageJSON.publish.branch) || DEFAULT_BRANCH;
const remote = (packageJSON.publish && packageJSON.publish.remote) || DEFAULT_REMOTE;
if (branch !== mustBeBranch && !options.force) error.fatal('ERR_WRONG_BRANCH', mustBeBranch);
// Has passed every validation, now, just do it
packageJSON.version = version;
fs.writeFile(PACKAGE_PATH, JSON.stringify(packageJSON, null, 2) + '\n', (err) => {
if (err) error.fatal();
const msg = util.replace(options.message, version);
const tag = util.replace(options.tag, version);
const handler = err => { err && error.fatal('ERR_GIT_PUSH', msg); };
let lastMsg;
const setMsg = (msg) => { cli.spinner(`${msg}...`); lastMsg = msg; };
const doneLast = ( ) => { lastMsg && cli.spinner(`${lastMsg} ${util.green('✓')}`, true); };
const updtMsg = (msg) => { doneLast(); setMsg(msg); };
git
.add([PACKAGE_PATH]) .then(() => updtMsg(msgs.success.add))
.commit(msg, handler) .then(() => updtMsg(msgs.success.commit))
.addAnnotatedTag(tag, msg, handler) .then(() => updtMsg(msgs.success.tag))
.push(remote, branch, handler) .then(() => updtMsg(msgs.success.push))
.pushTags(remote, handler) .then(() => updtMsg(msgs.success.pushTag))
.then(() => {
doneLast();
const txt = util.green(msgs.success.end);
console.log(util.replace(txt, version));
process.exit(0);
}); // git
}); // write file
}); // git status
}); // prompt
}); // readfile
}); // main
cli.main(function (args, options) {
try {
if (cli.command) {
if (commands.indexOf(cli.command) < 0)
throw Error('Invalid command: ' + cli.command);
// First try to run the `portable-js` package that is installed in folder where manifest file is located,
// if not found, run the command from the global package.
var manifest_dir = '';
if (options.file) {
manifest_dir = path.dirname(options.file);
}
else {
manifest_dir = process.cwd();
}
var command_dir = manifest_dir + '/node_modules/portable-js/src/command';
if (!fs.existsSync(command_dir))
command_dir = __dirname + '/command';
if (options.verbose)
log.level = 'verbose';
if (options.debug)
log.level = 'silly';
var cmd = require(command_dir + '/' + cli.command + '.js');
cmd(args, options);
}
else {
log.error('Command not found: ' + cli.command);
}
}
catch (e) {
log.error(e);
if (options.debug) {
console.log(e.stack || e);
}
}
});
var merge = require('merge');
var MessageSender = require('./MessageSender.js');
var defaultOptions = {
host: 'services.local.com',
user: '******',
password: '******',
frameMax: 0,
vhost: 'test',
exchange: 'audit',
messagesPerMinute: 60
};
cli.parse({
host: ['h', 'RabbitMQ host', 'string', defaultOptions.host],
user: ['u', 'RabbitMQ user', 'string', defaultOptions.user],
password: ['p', 'RabbitMQ password', 'string', defaultOptions.password],
vhost: ['v', 'RabbitMQ VHost', 'string', defaultOptions.vhost],
exchange: ['q', 'Destination exchange that will receive messages', 'string', defaultOptions.exchange],
template: ['t', 'Message template', 'string'],
messagesPerMinute: ['m', 'Messages per minute', 'string', defaultOptions.messagesPerMinute]
});
cli.main(function(args, options) {
var sender = new MessageSender(merge(defaultOptions, options), cli);
cli.spinner('Sending message ' + (sender.options.template === null ? 'Hello World' : sender.options.template) + ' at rate ' + sender.options.messagesPerMinute + '/min ');
sender.sendWithRetry();
});
cli.main(function (args, options) {
console.log("***********".rainbow);
console.log("Node Reddit".cyan);
console.log("***********\n\n".rainbow);
request('http://reddit.com/.json', function (err, res, body) {
if (!err && res.statusCode === 200) {
var reddit = JSON.parse(body),
stories = reddit.data.children.map(function (s) {
return s.data;
});
// Descending score
stories.sort(function (a, b) { return b.score - a.score; });
stories.forEach(function (story) {
var row = "",
title = story.title.length > 100
? story.title.substr(0, 100) + "..."
: story.title;
// Build row
// [score] [title] [comments] [subreddit]
// This sucks
row += story.score.toString().green + "\t";
row += title.bold
row += " (" + story.domain + ")";
row += (" /r/" + story.subreddit).cyan;
row += "\n\t";
row += story.author.grey;
row += " " + (story.num_comments + " comments").italic.yellow;
row += "\n";
console.log(row);
});
}
});
});