/**
* Creates workers or parses files and extracts metadata in-process.
*/
_getWorker(
options: ?{forceInBand: boolean},
): (message: WorkerMessage) => Promise<WorkerMetadata> {
if (!this._workerPromise) {
let workerFn;
if (
(options && options.forceInBand) ||
this._options.maxWorkers <= 1
) {
workerFn = worker;
} else {
this._workerFarm = workerFarm(
{
maxConcurrentWorkers: this._options.maxWorkers,
},
require.resolve('./worker'),
);
workerFn = this._workerFarm;
}
this._workerPromise = (message: WorkerMessage) => new Promise(
(resolve, reject) => workerFn(message, (error, metadata) => {
if (error || !metadata) {
reject(error);
} else {
resolve(metadata);
}
}),
);
}
return this._workerPromise;
}
constructor(options) {
const opts = this._opts = validateOpts(options);
this._cache = opts.cache;
this._transformModulePath = opts.transformModulePath;
this._projectRoots = opts.projectRoots;
if (opts.transformModulePath != null) {
let transformer;
if (opts.disableInternalTransforms) {
transformer = opts.transformModulePath;
} else {
transformer = this._workerWrapperPath = temp.path();
fs.writeFileSync(
this._workerWrapperPath,
`
module.exports = require(${JSON.stringify(require.resolve('./worker'))});
require(${JSON.stringify(String(opts.transformModulePath))});
`
);
}
this._workers = workerFarm({
autoStart: true,
maxConcurrentCallsPerWorker: 1,
maxCallsPerWorker: MAX_CALLS_PER_WORKER,
maxCallTime: opts.transformTimeoutInterval,
maxRetries: MAX_RETRIES,
}, transformer);
this._transform = Promise.denodeify(this._workers);
}
}
/**
* Creates workers or parses files and extracts metadata in-process.
*/
_getWorker() {
if (!this._workerPromise) {
if (this._options.maxWorkers === 1) {
this._workerPromise = message => new Promise(
(resolve, reject) => worker(message, (error, metadata) => {
if (error) {
reject(error);
} else {
resolve(metadata);
}
})
);
} else {
this._workerFarm = workerFarm(
{
maxConcurrentWorkers: this._options.maxWorkers,
},
require.resolve('./worker')
);
this._workerPromise = denodeify(this._workerFarm);
}
}
return this._workerPromise;
}
/**
* Creates workers or parses files and extracts metadata in-process.
*/
_getWorker() {
if (!this._workerPromise) {
let workerFn;
if (this._options.maxWorkers <= 1) {
workerFn = worker;
} else {
this._workerFarm = workerFarm(
{
maxConcurrentWorkers: this._options.maxWorkers },
require.resolve('./worker'));
workerFn = this._workerFarm;
}
this._workerPromise = message => new Promise(
(resolve, reject) => workerFn(message, (error, metadata) => {
if (error || !metadata) {
reject(error);
} else {
resolve(metadata);
}
}));
}
return this._workerPromise;
}
constructor(options) {
const opts = this._opts = validateOpts(options);
const {transformModulePath} = opts;
if (transformModulePath) {
this._transformModulePath = require.resolve(transformModulePath);
this._workers = workerFarm(
{
autoStart: true,
maxConcurrentCallsPerWorker: 1,
maxConcurrentWorkers: maxConcurrentWorkers,
maxCallsPerWorker: MAX_CALLS_PER_WORKER,
maxCallTime: opts.transformTimeoutInterval,
maxRetries: MAX_RETRIES,
},
require.resolve('./worker'),
['minify', 'transformAndExtractDependencies']
);
this._transform = Promise.denodeify(this._workers.transformAndExtractDependencies);
this.minify = Promise.denodeify(this._workers.minify);
}
}
_createParallelTestRun(testPaths, onTestResult, onRunFailure) {
const farm = workerFarm({
maxConcurrentCallsPerWorker: 1,
// We allow for a couple of transient errors. Say something to do
// with loading/serialization of the resourcemap (which I've seen
// happen).
maxRetries: 2,
maxConcurrentWorkers: this._opts.maxWorkers,
}, TEST_WORKER_PATH);
const runTest = promisify(farm);
return this._getModuleLoaderResourceMap()
.then(() => Promise.all(testPaths.map(
testFilePath => runTest({config: this._config, testFilePath})
.then(testResult => onTestResult(testFilePath, testResult))
.catch(err => {
onRunFailure(testFilePath, err);
if (err.type === 'ProcessTerminatedError') {
// Initialization error or some other uncaught error
console.error(
'A worker process has quit unexpectedly! ' +
'Most likely this an initialization error.'
);
process.exit(1);
}
})
))).then(() => workerFarm.end(farm));
}
_createParallelTestRun(
testPaths: Array<Path>,
onTestResult: OnTestResult,
onRunFailure: OnRunFailure,
) {
const config = this._config;
const farm = workerFarm({
autoStart: true,
maxConcurrentCallsPerWorker: 1,
maxRetries: 2, // Allow for a couple of transient errors.
maxConcurrentWorkers: this._options.maxWorkers,
}, TEST_WORKER_PATH);
const runTest = promisify(farm);
return Promise.all(testPaths.map(
path => runTest({path, config})
.then(testResult => onTestResult(path, testResult))
.catch(err => {
onRunFailure(path, err);
if (err.type === 'ProcessTerminatedError') {
console.error(
'A worker process has quit unexpectedly! ' +
'Most likely this an initialization error.',
);
process.exit(1);
}
})),
)
.then(() => workerFarm.end(farm));
}
var runInProcess = function(file, args, callback) {
var worker = workerFarm(file);
args.push(function (err, result) {
callback(err, result);
workerFarm.end(worker);
});
worker.apply(worker, args);
};
Preprocessor.setWorkers = function(){
workers = workerFarm({
maxCallsPerWorker: 100,
maxConcurrentWorkers: 4,
maxConcurrentCallsPerWorker: -1,
maxCallTime: 1000
}, require.resolve('./preprocessor_worker.js'));
};
extract.init = () => {
if (ENABLE_WORKERS) {
workers = workerFarm({
maxConcurrentCallsPerWorker: npm.limit.fetch,
maxRetries: 1
}, WORKER_PATH)
}
return BB.resolve()
}
_createParallelTestRun(
testPaths,
watcher,
onResult,
onFailure)
{
const config = this._config;
const farm = workerFarm({
autoStart: true,
maxConcurrentCallsPerWorker: 1,
maxConcurrentWorkers: this._options.maxWorkers,
maxRetries: 2 },
TEST_WORKER_PATH);
const mutex = throat(this._options.maxWorkers);
// Send test suites to workers continuously instead of all at once to track
// the start time of individual tests.
const runTestInWorker = (_ref) => {let path = _ref.path,config = _ref.config;return mutex(() => {
if (watcher.isInterrupted()) {
return Promise.reject();
}
this._dispatcher.onTestStart(config, path);
return promisify(farm)({ config, path });
});};
const onError = (err, path) => {
onFailure(path, err);
if (err.type === 'ProcessTerminatedError') {
console.error(
'A worker process has quit unexpectedly! ' +
'Most likely this an initialization error.');
process.exit(1);
}
};
const onInterrupt = new Promise((_, reject) => {
watcher.on('change', state => {
if (state.interrupted) {
reject(new CancelRun());
}
});
});
const runAllTests = Promise.all(testPaths.map(path => {
return runTestInWorker({ config, path }).
then(testResult => onResult(path, testResult)).
catch(error => onError(error, path));
}));
return Promise.race([
runAllTests,
onInterrupt]).
then(() => workerFarm.end(farm));
}
function makeFarm(worker, methods, timeout) {
return workerFarm(
{
autoStart: true,
maxConcurrentCallsPerWorker: 1,
maxConcurrentWorkers: maxConcurrentWorkers,
maxCallsPerWorker: MAX_CALLS_PER_WORKER,
maxCallTime: timeout,
maxRetries: MAX_RETRIES,
},
worker,
methods,
);
}
return new Promise(async (resolve, reject) => {
const startTime = Date.now();
let count = 0;
async function countToFinish() {
if (++count === calls) {
workerFarm.end(api);
const endTime = Date.now();
// Let all workers go down.
await sleep(2000);
resolve({
globalTime: endTime - startTime - 2000,
processingTime: endTime - startProcess,
});
}
}
const api = workerFarm(
{
autoStart: true,
maxConcurrentCallsPerWorker: 1,
maxConcurrentWorkers: threads,
},
require.resolve('./workers/worker_farm'),
[method],
);
// Let all workers come up.
await sleep(2000);
const startProcess = Date.now();
for (let i = 0; i < calls; i++) {
const promisified = new Promise((resolve, reject) => {
api[method]((err, result) => {
if (err) {
reject(err);
} else {
resolve(result);
}
});
});
promisified.then(countToFinish);
}
});
module.exports = function(options) {
options = options || {};
var workerPool = workerFarm({
maxConcurrentWorkers: options.poolSize || require('os').cpus().length-1 || 1
}, require.resolve('./worker'));
return {
optimize: function(config, assetGroup, callback) {
workerPool(config, assetGroup, callback);
},
shutdown: function() {
workerFarm.end(workerPool);
}
};
};
constructor(options) {
const opts = this._opts = validateOpts(options);
this._cache = opts.cache;
if (opts.transformModulePath != null) {
this._workers = workerFarm({
autoStart: true,
maxConcurrentCallsPerWorker: 1,
maxCallsPerWorker: MAX_CALLS_PER_WORKER,
maxCallTime: opts.transformTimeoutInterval,
}, opts.transformModulePath);
this._transform = Promise.denodeify(this._workers);
}
}
/**
* Runs the specified webpack configuration in parallel.
* @param {String} configPath The path to the webpack.config.js
* @param {Object} options
* @param {Boolean} [options.watch=false] If `true`, Webpack will run in `watch-mode`.
* @param {boolean} [options.maxRetries=Infinity] The maximum amount of retries on build error
* @param {Number} [options.maxConcurrentWorkers=require('os').cpus().length] The maximum number of parallel workers
* @param {Function} [callback] A callback to be invoked once the build has been completed
* @return {Promise} A Promise that is resolved once all builds have been created
*/
function run(configPath, options, callback) {
var config,
argvBackup = process.argv;
if(!options.argv) {
options.argv = [];
}
options.argv = ['node', 'parallel-webpack'].concat(options.argv);
try {
process.argv = options.argv;
config = require(configPath);
process.argv = argvBackup;
} catch(e) {
throw {
message: chalk.red('[WEBPACK]') + ' Could not load configuration file ' + chalk.underline(configPath),
error: e
};
}
var maxRetries = options && parseInt(options.maxRetries, 10) || Infinity,
maxConcurrentWorkers = options
&& parseInt(options.maxConcurrentWorkers, 10)
|| require('os').cpus().length,
workers = workerFarm({
maxRetries: maxRetries,
maxConcurrentWorkers: maxConcurrentWorkers
}, require.resolve('./src/webpackWorker')),
done = closeFarm(workers, options, callback, +new Date());
process.on('SIGINT', function() {
console.log(chalk.red('[WEBPACK]') + ' Forcefully shutting down');
done();
});
return startFarm(
config,
configPath,
options || {},
promisify(workers)
).then(function(err, res) {
return done(null, res);
}, function(err) {
throw done(err);
});
}
function Transformer(options) {
var opts = validateOpts(options);
this._cache = opts.nonPersistent
? new DummyCache()
: new Cache({
resetCache: options.resetCache,
cacheVersion: options.cacheVersion,
projectRoots: options.projectRoots,
});
if (options.transformModulePath != null) {
this._workers = workerFarm(
{autoStart: true, maxConcurrentCallsPerWorker: 1},
options.transformModulePath
);
this._transform = Promise.promisify(this._workers);
}
}
return function webpack(config, pckg) {
const runnerPath = require.resolve(path.resolve(__dirname, 'forkableRunner'));
const workers = workerFarm(runnerPath);
return groupEntries(config, pckg).then(results => (
Promise.all(results.map(result => (
Promise.fromCallback(callback => (
workers({
config: {
baseDir: config.baseDir,
developmentDevtool: config.developmentDevtool,
productionDevtool: config.productionDevtool,
},
entries: result.entries,
pckg: result.pckg,
variant,
}, callback)
))
)))
)).finally(() => workerFarm.end(workers));
};
var glob = require("glob"),
path = require("path"),
_ = require("lodash"),
Q = require("q"),
qlimit = require("qlimit"),
fs = require("graceful-fs"),
readFile = qlimit(1000)(Q.nbind(fs.readFile, fs)),
Timerish = require("timerish"),
workerFarm = require("worker-farm"),
parseIndexHtml = workerFarm(require.resolve("./lib/parseIndex/parseIndexHtml")),
es = require("event-stream"),
JSONStream = require("JSONStream");
var args = require("yargs")
.usage('Usage: $0 [options]')
.option("index-path", {type: "string", demand: true})
.option("save-to", {type: "string"})
.argv;
function parseFile(fn) {
console.log("Reading", {fn: fn});
return readFile(fn).then(function (html) {
return Q.nfcall(parseIndexHtml, fn, html.toString());
});
}
var tick = Timerish();
Q.nfcall(glob, "**/*.html", {cwd: args["index-path"]})
.then(function (res) {
tick("glob");
return Q.all(res.map(function (fn) {
* and all available reports in that year.
*/
var async = require('async'),
fs = require('fs'),
glob = require('glob'),
path = require('path'),
workerFarm = require('worker-farm');
var config = require("../config/config"),
boot = require("../app/boot"),
es = boot.es;
var farm = workerFarm({
maxCallsPerWorker: 1000,
maxConcurrentWorkers: 1,
maxRetries: 20
}, require.resolve('./inspectors-worker'));
function loadReportProxy(details, done) {
farm(details, config, function(err) {
if (err) {
console.log("\tEr what!!");
console.log("\t" + err);
process.exit(1);
}
done();
});
}
prepare(emitter) {
this._workers = workerFarm(require.resolve('./job'));
emitter.on(RunnerEvents.END, () => workerFarm.end(this._workers));
}
/**
* Runs the specified webpack configuration in parallel.
* @param {String} configPath The path to the webpack.config.js
* @param {Object} options
* @param {Boolean} [options.watch=false] If `true`, Webpack will run in
* `watch-mode`.
* @param {boolean} [options.maxRetries=Infinity] The maximum amount of retries
* on build error
* @param {Number} [options.maxConcurrentWorkers=require('os').cpus().length] The
* maximum number of parallel workers
* @param {Function} [callback] A callback to be invoked once the build has
* been completed
* @return {Promise} A Promise that is resolved once all builds have been
* created
*/
function run(configPath, options, callback) {
var config,
argvBackup = process.argv;
options = options || {};
if(options.colors === undefined) {
options.colors = chalk.supportsColor;
}
if(!options.argv) {
options.argv = [];
}
options.argv.unshift(process.execPath, 'parallel-webpack');
try {
process.argv = options.argv;
config = loadConfigurationFile(configPath);
process.argv = argvBackup;
} catch(e) {
process.argv = argvBackup;
return Promise.reject(new Error(
chalk.red('[WEBPACK]') + ' Could not load configuration file ' + chalk.underline(configPath) + "\n"
+ e
));
}
var maxRetries = parseInt(options.maxRetries, 10) || 0,
maxConcurrentWorkers = parseInt(options.maxConcurrentWorkers, 10)
|| require('os').cpus().length,
workers = workerFarm({
maxRetries: maxRetries,
maxConcurrentWorkers: maxConcurrentWorkers
}, require.resolve('./src/webpackWorker'));
process.on('SIGINT', function() {
if (notSilent(options)) {
console.log(chalk.red('[WEBPACK]') + ' Forcefully shutting down');
}
workerFarm.end(workers);
});
var startTime = Date.now();
return startFarm(
config,
configPath,
options,
Promise.promisify(workers)
).error(function(err) {
if(notSilent(options)) {
console.log('%s Build failed after %s seconds', chalk.red('[WEBPACK]'), chalk.blue((Date.now() - startTime) / 1000));
}
return Promise.reject(err);
}).then(function (results) {
if(notSilent(options)) {
console.log('%s Finished build after %s seconds', chalk.blue('[WEBPACK]'), chalk.blue((Date.now() - startTime) / 1000));
}
results = results.filter(function(result) {
return result;
});
if(results.length) {
return results;
}
}).finally(function () {
workerFarm.end(workers);
}).asCallback(callback);
}
},
destruct() {
if (this._workers) {
workerFarm.end(this._workers);
this._workers = null;
}
},
/**
* Добавить задачу в очередь исполнения
* @param {String} path путь к модулю-обработчику
* @return {Promise} промис с результатом выполнения задачи
*/
push() {
if (!this._workers) {
this._workers = workerFarm(require.resolve('./processor'));
this.push = this._process;
}
return this._process.apply(this, arguments);
},
_process(path) {
const args = [].slice.call(arguments, 1);
const workers = this._workers.bind(this._workers, path, args);
return vowNode.invoke(workers);
}
});
var RUNS = 50
var x
console.log('creating %s jobs', RUNS)
JOBS = []
for (var i = RUNS-1; i >= 0; i--) {
// just to swing the number a bit
JOBS.push( 0-( i/10000000 ) )
};
console.log('jobs created!')
d0 = Date.now()
var workerFarm = require('worker-farm')
, workers = workerFarm(require.resolve('./worker'))
, ret = 0
for (var i = 0; i < JOBS.length-1; i++) {
workers(JOBS[i], function (err, byPid) {
console.log("done", ret)
if (++ret == JOBS.length-1){
workerFarm.end(workers)
console.log( 'took '+ (Date.now() - d0), ret )
}
})
}
'use strict'
const BB = require('bluebird')
const npa = require('npm-package-arg')
const workerFarm = require('worker-farm')
const extractionWorker = require('./worker.js')
const WORKER_PATH = require.resolve('./worker.js')
module.exports = {
startWorkers () {
this._workers = workerFarm({
maxConcurrentCallsPerWorker: 20,
maxRetries: 1
}, WORKER_PATH)
},
stopWorkers () {
workerFarm.end(this._workers)
},
child (name, child, childPath, config, opts) {
const spec = npa.resolve(name, child.version)
const childOpts = config.toPacote(Object.assign({
integrity: child.integrity,
resolved: child.resolved
}, {
dirPacker: opts.dirPacker
}))
const args = [spec, childPath, childOpts]
_addUntestedFiles(globalConfig: GlobalConfig, contexts: Set<Context>) {
const files = [];
contexts.forEach(context => {
const config = context.config;
if (
globalConfig.collectCoverageFrom &&
globalConfig.collectCoverageFrom.length
) {
context.hasteFS
.matchFilesWithGlob(globalConfig.collectCoverageFrom, config.rootDir)
.forEach(filePath =>
files.push({
config,
path: filePath,
}),
);
}
});
if (!files.length) {
return Promise.resolve();
}
if (isInteractive) {
process.stderr.write(
RUNNING_TEST_COLOR('Running coverage on untested files...'),
);
}
let worker;
let farm;
if (this._globalConfig.maxWorkers <= 1) {
worker = pify(CoverageWorker);
} else {
farm = workerFarm(
{
autoStart: true,
maxConcurrentCallsPerWorker: 1,
maxConcurrentWorkers: this._globalConfig.maxWorkers,
maxRetries: 2,
},
require.resolve('./coverage_worker'),
);
worker = pify(farm);
}
const instrumentation = [];
files.forEach(fileObj => {
const filename = fileObj.path;
const config = fileObj.config;
if (!this._coverageMap.data[filename]) {
const promise = worker({
config,
globalConfig,
path: filename,
})
.then(result => {
if (result) {
this._coverageMap.addFileCoverage(result.coverage);
if (result.sourceMapPath) {
this._sourceMapStore.registerURL(
filename,
result.sourceMapPath,
);
}
}
})
.catch((error: SerializableError) => {
console.error(chalk.red(error.message));
});
instrumentation.push(promise);
}
});
const cleanup = () => {
if (isInteractive) {
clearLine(process.stderr);
}
if (farm) {
workerFarm.end(farm);
}
};
return Promise.all(instrumentation).then(cleanup).catch(cleanup);
}
var mysql = require('mysql');
var config = require('../config/config');
var workerFarm = require('worker-farm')
var companyPageScrapper = workerFarm(require.resolve('../kompass2/scrap-company-page'));
var activityUrlUpdate = workerFarm(require.resolve('../queries/update-scraping-status'));
var db = mysql.createConnection({
host : config.db.host,
user : config.db.user,
password : config.db.password,
port : config.db.port,
database : config.db.database
});
module.exports = function(callback) {
var query = db.query('SELECT url FROM companyUrls WHERE scrapped IS NULL');
query.on('error', function(err) { console.log('QUERY ERROR ' + JSON.stringify(err)); });
query.on('result', function(result) {
activityPageScrapper(result.url, function(err) {
console.log('SCRAPE COMP ' + result.url + ' : ' + JSON.stringify(err));
});
activityUrlUpdate('companyUrls', result.url, function(err, activityUrl) {
console.log('UPDATE COMP ' + result.url + ' : ' + JSON.stringify(err));
});
});
query.on('end', function() { console.log('fini !!'); });
};
_createParallelTestRun(
testPaths: Array<Path>,
watcher: TestWatcher,
onResult: OnTestResult,
onFailure: OnRunFailure,
) {
const config = this._config;
const farm = workerFarm({
autoStart: true,
maxConcurrentCallsPerWorker: 1,
maxConcurrentWorkers: this._options.maxWorkers,
maxRetries: 2, // Allow for a couple of transient errors.
}, TEST_WORKER_PATH);
const mutex = throat(this._options.maxWorkers);
const worker = promisify(farm);
// Send test suites to workers continuously instead of all at once to track
// the start time of individual tests.
const runTestInWorker = ({config, path}) => mutex(() => {
if (watcher.isInterrupted()) {
return Promise.reject();
}
this._dispatcher.onTestStart(config, path);
return worker({
config,
path,
rawModuleMap: watcher.isWatchMode()
? this._hasteContext.moduleMap.getRawModuleMap()
: null,
});
});
const onError = (err, path) => {
onFailure(path, err);
if (err.type === 'ProcessTerminatedError') {
console.error(
'A worker process has quit unexpectedly! ' +
'Most likely this an initialization error.',
);
process.exit(1);
}
};
const onInterrupt = new Promise((_, reject) => {
watcher.on('change', state => {
if (state.interrupted) {
reject(new CancelRun());
}
});
});
const runAllTests = Promise.all(testPaths.map(path => {
return runTestInWorker({config, path})
.then(testResult => onResult(path, testResult))
.catch(error => onError(error, path));
}));
const cleanup = () => workerFarm.end(farm);
return Promise.race([
runAllTests,
onInterrupt,
]).then(cleanup, cleanup);
}
return true;
}
*/
var gcm = require('node-gcm');
var _ = require('underscore')._ ;
var Message = require('./Message.js');
var crypto = require('jsrsasign');
var forge = require('node-forge')({disableNativeCode: true});
var pg = require('pg');
var when = require('when');
var squel = require("squel");
var config = require('./Config.js');
var workerFarm = require('worker-farm');
var workers = workerFarm(require.resolve('./KeysGenerator'));
function PostMan(_io, _logger) {
var io = _io; //pointer to io.sockets
var listOfMessages = []; //array of Message.js (DB)
var listOfACKs = []; //array of {msgID ,to ,from } (DB)
var clientOfDB = null;
var self = this;
var lastServerAsigned = 0;
var logger = _logger;
var listOfAsimetricKeys = [];
this.createAsymetricKeys = function() {
var options = {};
__constructor: function(emitter) {
this._workers = workerFarm(require.resolve('./compare-adapter'), ['compare', 'buildDiff']);
emitter.on(RunnerEvents.END, function() {
workerFarm.end(this._workers);
}.bind(this));
},