Example #1
0
  readdir(path, function(err, versions){
    var batch = new Batch;
    batch.concurrency(6);

    versions.forEach(function(version){
      batch.push(function(done){
        var file = join(path, version, 'component.json');

        read(file, 'utf8', function(err, json){
          if (err) return done(err);

          try {
            var obj = JSON.parse(json);
          } catch (err) {
            return done(err);
          }

          tree[version] = obj;
          done();
        });
      });
    });

    batch.end(fn);
  });
Example #2
0
  builder.hook('before styles', function (pkg, cb) {
    if (!pkg.config.styles) return cb();

    var stylusFiles = pkg.config.styles.filter(endsWith('styl'))
      , batch = new Batch();

    stylusFiles.forEach(function (styl) {
      batch.push(function (done) {
        var stylPath = pkg.path(styl)
          , name = styl.split('.')[0] + '.css';

        debug('compiling: %s', styl);

        stylus(fs.readFileSync(stylPath, 'utf-8'))
          .set('include css', true)
          .set('filename', styl)
          .use(nib())
          .render(function (err, css) {
            pkg.addFile('styles', name, css);
            pkg.removeFile('styles', styl);

            done();
          });

      });
    });

    batch.end(cb);
  });
Example #3
0
  this.getJSON(function(err, json){
    if (err) return self.emit('error', err);

    var files = [];
    if (json.scripts) files = files.concat(json.scripts);
    if (json.styles) files = files.concat(json.styles);
    if (json.templates) files = files.concat(json.templates);
    if (json.files) files = files.concat(json.files);
    json.repo = json.repo || self.remote + '/' + self.name;

    if (json.dependencies) {
      batch.push(function(done){
        self.getDependencies(json.dependencies, done);
      });
    }

    batch.push(function(done){
      self.mkdir(self.dirname(), function(err){
        json = JSON.stringify(json, null, 2);
        self.writeFile('component.json', json, done);
      });
    }); 

    batch.push(function(done){
      self.mkdir(self.dirname(), function(err){
        self.getFiles(files, done);
      });
    });

    batch.end(function(err){
      if (err) return self.emit('error', err);
      self.emit('end');
    });
  });
Example #4
0
File: queue.js Project: jbt/qp
  self.setTTLLock(function(err, lockAcquired){
    if (err){
      debug('unable to set ttl check lock');
      return scheduleNext();
    }

    if (!lockAcquired) return scheduleNext();

    var batch = new Batch();
    batch.concurrency(1);

    self.states.forEach(function(state){
      var ttl = self.getOption(state+'TTL');

      // not set or invalid
      if (typeof ttl != 'number') return;

      batch.push(function(done){
        self.getJobsByTime(state, 0, Date.now() - ttl, function(err, jobs){
          if (err || !jobs || !jobs.length) return done();

          debug('removing %d jobs in %s state past ttl of %d', jobs.length, state, ttl);
          self.removeJobs(jobs, done);
        });
      });
    });

    batch.end(function(err){
      if (err) debug(err);

      scheduleNext();
    });
  });
Example #5
0
File: queue.js Project: jbt/qp
Queue.prototype.multiSave = function(jobs, cb) {

  var self = this;

  for (var i = 0; i < jobs.length; i++) {
    if (!(jobs[i] instanceof Job)) {
      jobs[i] = self.create(jobs[i]);
    }
  }

  var r = this.redis.multi();

  var batch = new Batch();

  // stop redis storming if uniqueness is enabled or IDs arent set
  batch.concurrency(3);

  jobs.forEach(function(job) {
    batch.push(function(done) {
      job._save(r, done);
    });
  });

  batch.end(function() {
    // how is best to handle errors from _save here?
    // we probably still want to exec the rest otherwise
    // one bad job stops the entire batch from being saved
    r.exec(cb);
  });

};
Example #6
0
exports.stats = function(obj, fn){
  var scripts = obj.scripts || [];
  var batch = new Batch;
  var ret = {};

  // throw
  batch.throws(true);

  // stats `file`
  scripts.forEach(function(src){
    batch.push(stats(obj, src));
  });

  // all done
  batch.end(function(err, arr){
    if (err) return fn(err);
    ret.sloc = 0;
    ret.size = 0;

    arr.forEach(function(obj){
      ret.sloc += obj.sloc;
      ret.size += obj.size;
    });

    fn(null, ret);
  });
};
Example #7
0
  builder.hook('before styles', function (builder, callback) {
    if (!builder.config.styles) return callback();

    var files = builder.config.styles.filter(sassFilter)
      , batch = new Batch();

    files.forEach(function (file) {
      batch.push(function (done) {
        debug('compiling: %s', file);

        sass.compile(builder.path(file), options, function (err, css) {
          if (err) {
            debug('error compiling: %s, %s', file, err);
            return done(err);
          }

          var newFile = path.basename(file, path.extname(file)) + '.css';
          builder.addFile('styles', newFile, css);
          builder.removeFile('styles', file);
          done();
        });
      });
    });

    batch.end(callback);
  });
Example #8
0
  builder.hook('before styles', function(pkg, fn){

    var styles = pkg.config.styl;
    if (!styles) return fn()

    var batch = new Batch;

    if (!pkg.config.styles){
      batch.push(function(done){
        addConfProperty(pkg, 'styles', done);
      });
    }

    styles.forEach(function(file){

      batch.push(function(done){

        var stylfile = pkg.path(file) + '.styl';
        var cssfile =  file + '.css';
        var styl = fs.readFileSync(stylfile, 'utf8');
        var opts = { whitespace: true };
        var style = new Style(styl, opts);
        style.use(variant());

        style.vendors(vendors);
        pkg.addFile('styles', cssfile, style.toString());
        done();
        
      });

    });

    batch.end(fn);

  });
Example #9
0
Package.prototype.getFiles = function(files, fn){
  var self = this;
  var batch = new Batch;

  files.forEach(function(file){
    batch.push(function(done){
      var url = self.url(file);
      debug('fetching %s', url);
      self.emit('file', file, url);
      var dst = self.join(file);

      // mkdir
      // TODO: only mkdir when necessary
      mkdir(dirname(dst), function(err){
        if (err) return fn(err);

        // pipe file
        request
        .get(url)
        .buffer(false)
        .end(function(res){
          res.pipe(fs.createWriteStream(dst));
          res.on('error', done);
          res.on('end', done);
        });
      });
    });
  });

  batch.end(fn);
};
Example #10
0
 this.query(start, end, 0, function (err, res) {
   if (err) return callback(err);
   var count = res.count;
   customers.push.apply(customers, res.data);
   var got = res.data.length;
   var left = res.count - got;
   // check if we grabbed everything in the first query
   if (0 === left) return callback(null, customers);
   // there's more, we have to paginate query
   var pages = Math.ceil(left / page);
   debug('loaded %d customers, %d left in %d pages of %d', got, left, pages, page);
   var batch = new Batch();
   batch.concurrency(self.options.concurrency);
   range(0, pages).forEach(function (i) {
     batch.push(function (done) { self.query(start, end, got + (i * page), done); });
   });
   batch.end(function (err, results) {
     if (err) return callback(err);
     results.forEach(function (res) {
       customers.push.apply(customers, res.data);
     });
     debug('finished loading all customers in cohort');
     callback(null, customers);
   });
 });
Example #11
0
  builder.hook('before styles', function(pkg, fn){

    var styles = pkg.config.styles;
    if (!styles) return fn()

    var batch = new Batch;
    
    styles.forEach(function(file){

      batch.push(function(done){

        var cssfile = pkg.path(file);
        var styl = fs.readFileSync(cssfile, 'utf8');
        var style = new Style(styl);
        
        style.vendors(vendors);
        pkg.removeFile('styles', file);
        pkg.addFile('styles', file, style.toString());
        done();

      });
    });

    batch.end(fn);

  });
Example #12
0
module.exports = function (html, options, callback) {
    var batch = new Batch();
    var data = getStylesheetList(html, options);
    batch.push(function (callback) {
        getStylesData(data.html, options, callback);
    });
    if (data.hrefs.length) {
        assert.ok(options.url, 'options.url is required');
    }
    data.hrefs.forEach(function (stylesheetHref) {
        batch.push(function (callback) {
            getHrefContent(stylesheetHref, options.url, callback);
        });
    });
    batch.end(function (err, results) {
        if (err) {
            return callback(err);
        }
        var stylesData = results.shift();
        results.forEach(function (content) {
            stylesData.css.push(content);
        });
        var css = stylesData.css.join('\n');
        callback(null, stylesData.html, css);
    });
};
Example #13
0
  function startServer(propOverrides, done) {
    var props = {};
    var prop;
    for (prop in defaultServerProps) {
      if (!defaultServerProps.hasOwnProperty(prop)) continue;

      props[prop] = defaultServerProps[prop];
    }
    for (prop in propOverrides) {
      if (!propOverrides.hasOwnProperty(prop)) continue;

      props[prop] = propOverrides[prop];
    }
    var batch = new Batch();
    batch.push(function(cb) { mkdirp(MC_SERVER_PATH, cb); });
    batch.push(function(cb) {
      var str = "";
      for (var prop in props) {
        if (!props.hasOwnProperty(prop)) continue;

        str += prop + "=" + props[prop] + "\n";
      }
      fs.writeFile(path.join(MC_SERVER_PATH, "server.properties"), str, cb);
    });
    batch.end(function(err) {
      if (err) return done(err);
      mcServer = spawn('java', [ '-jar', MC_SERVER_JAR, 'nogui'], {
        stdio: 'pipe',
        cwd: MC_SERVER_PATH,
      });
      mcServer.stdin.setEncoding('utf8');
      mcServer.stdout.setEncoding('utf8');
      mcServer.stderr.setEncoding('utf8');
      var buffer = "";
      mcServer.stdout.on('data', onData);
      mcServer.stderr.on('data', onData);
      function onData(data) {
        buffer += data;
        var lines = buffer.split("\n");
        var len = lines.length - 1;
        for (var i = 0; i < len; ++i) {
          mcServer.emit('line', lines[i]);
        }
        buffer = lines[lines.length - 1];
      }
      mcServer.on('line', onLine);
      mcServer.on('line', function(line) {
        process.stderr.write('.');
        // uncomment this line when debugging for more insight as to what is
        // happening on the minecraft server
        //console.error("[MC]", line);
      });
      function onLine(line) {
        if (/\[Server thread\/INFO\]: Done/.test(line)) {
          mcServer.removeListener('line', onLine);
          done();
        }
      }
    });
  }
Example #14
0
  fs.readFile(dataset_file, 'utf8', function (err, data) {
    if (err) {
      console.log('Error: ' + err);
      return callback(err);
    }
    datasets = JSON.parse(data).datasets;
    
    var batch = new Batch;
    batch.concurrency(concurency);

    // process all datasets.
    var count = 0;
    var length = datasets.length;
    for(var i=0; i < length; i++){
      (function(dataset){
        batch.push(function(done){
          datasetProcessor(dataset, done);       
        });
      })(datasets[i]);
    }
    
    // end.
    batch.end(function(err, data){
      if(err) {
        console.log("PROCESS ALL DATASET ERROR",err);
        if(callback) callback(err);
        return;
      }
      console.log("all datasets processed");
      if(callback)
        return callback();
    });
  });
Example #15
0
  builder.hook('before scripts', function(pkg, fn){

    var templates = pkg.config.html;
    if (!templates) return fn()

    batch = new Batch;

    if (!pkg.config.scripts){
      batch.push(function(done){
        addConfProperty(pkg, 'scripts', done);
      });
    }

    templates.forEach(function(file){

      batch.push(function(done){

        var htmlfile = pkg.path(file) + '.html';
        var jsfile = file + '.js';
        var str = fs.readFileSync(htmlfile, 'utf8');

        pkg.addFile('scripts', jsfile, str2js(str));
        done();

      });
    
    });

    batch.end(fn);

  });
  builder.hook('before styles', function(pkg, next) {
    var styles = pkg.config.styles;
    if (!styles) return next();

    var stylusFiles = styles.filter(function(file) { return path.extname(file) == '.styl';});

    if(!stylusFiles.length) return next();

    batch = new Batch();
    var includeText = "";
    stylPlugin.includes.forEach(function(include) {
      contents = read(include, 'utf-8');
      includeText += contents + '\n';
    });

    stylusFiles.forEach(function(styl) {


      batch.push(function(done) {
        var filePath = pkg.path(styl);
        pkg._files[styl] = stylPlugin.compileFunction(filePath, includeText);
        done();
      });
    });
    batch.end(next);
  });
Example #17
0
 Opportunity.getFull = function(id, cb) {
   var batch = new Batch();
   batch.push(function(cb) {
     Opportunity.get(id, cb);
   });
   batch.push(function(cb) {
     lydian.get("/opportunities/" + id + "/brief", {}, cb);
   });
   batch.push(function(cb) {
     lydian.get("/opportunities/" + id + "/faqs", {}, cb);
   });
   batch.push(function(cb) {
     lydian.get("/opportunities/" + id + "/terms", {}, cb);
   });
   batch.push(function(cb) {
     lydian.get("/opportunities/" + id + "/host", {}, cb);
   });
   batch.end(function(err, results) {
     if (err) return cb(err);
     var opp = results.shift();
     opp.brief = results.shift();
     opp.faqs = results.shift();
     opp.terms = results.shift();
     opp.host = results.shift();
     cb(null, opp);
   });
 };
Example #18
0
Builder.prototype.build = function(fn){
  var self = this;
  var batch = new Batch;
  debug('building %s', this.dir);
  batch.push(this.buildScripts.bind(this));
  batch.push(this.buildTemplates.bind(this));
  batch.push(this.buildAliases.bind(this));
  batch.push(this.buildStyles.bind(this));
  batch.push(this.buildImages.bind(this));
  batch.push(this.buildFonts.bind(this));
  batch.push(this.buildFiles.bind(this));
  batch.end(function(err, res){
    if (err) return fn(err);

    var scripts = res.shift();
    var require = res.shift();
    var templates = res.shift();
    var custom = self._js;
    var js = [scripts, require, templates, custom].filter(empty).join('\n')

    fn(null, {
      js: js,
      css: res.shift(),
      images: res.shift(),
      fonts: res.shift(),
      files: res.shift(),
      require: requirejs
    });
  });
};
Example #19
0
var batchGet = exports.batchGet = function(urls, progressback, callback) {
  var batch = new Batch;
  batch.concurrency(5);
  urls.forEach(function(url) {
    batch.push(function(done) {
      request
        .get(url)
        .set('User-Agent', 'curl/7.24.0 (x86_64-apple-darwin12.0) libcurl/7.24.0 OpenSSL/0.9.8r zlib/1.2.5')
        .end(function(error, response) {
          console.log(url);
          if (error) throw new Error(error);
          if (response.error) {
            if (response.status === 404) {
              done();
            } else {
              throw [response.error,response.text].join("\n");
            }
          }
          var result;
          try {
            result = progressback(response.text);
          } catch (err) {
            error = err;
          }
          done(error, result);
        });
    });
  });

  batch.end(function(error, all) {
    if (error) throw new Error(error);
    callback(all);
  });
};
Example #20
0
function verify(deps, options, fn) {
  console.log('Verifying packages...');
  var batch = new Batch();

  Object.keys(deps).forEach(function(name) {
    batch.push(function(fn) {
      var version = deps[name],
        pkg = new Package(name, version, {
          remotes: options.remotes
        });
      pkg.getJSON(function(err, json) {
        if (err) {
          console.error('%s cannot be reliably shrinkwrapped: setting version to *', name);
          deps[name] = '*';
        } else if (version !== json.version) {
          console.error('%s version discrepancy - tag %s vs. source %s', name, json.version, version);
        }
        fn();
      });
    });
  });

  batch.end(function() {
    fn(null, deps);
  });
}
  provider.handleUpload = function(req, res, next) {
    var form = new multiparty.Form(),
      batch = new Batch();

    var file = {
      id: req.params.challengeId   // all files have the id that's same as challenge.id in json file.
    };

    batch.push(function(cb) {
      // add field parameters
      form.on('field', function(name, value) {
        file[name] = value
        cb(null, '');
      });
    });
    batch.push(function(cb) {
      form.on('part', function(part) {
        if (!part.filename) {
          return;
        }
        cb(null, part);
      });
    });

    batch.end(function(err, results) {
      if(err) {
        routeHelper.addError(req, err);
        return next();
      }
      form.removeListener('close', onEnd);
      var part = results[1];
      var fileName = part.filename;
      headers['Content-Length'] = part.byteCount;
      var targetPath = '/challenges' + '/' + req.params.challengeId + '/' + fileName;
      _.extend(file, {
        filePath : targetPath,
        fileName : fileName,
        size : part.byteCount,
        // storageLocation configured in config. same as name of storage provider
        // challenge service is using storageLocation so chaned storageType to storage location
        storageLocation : config.uploads.storageProvider
      });
      s3Client.putStream(part, targetPath, headers, function(err, s3Response) {
        if (err) {
          routeHelper.addError(req, err, s3Response.statusCode);
        } else {
          console.log('s3 response code' + s3Response.statusCode);
          if(s3Response.statusCode === HTTP_OK) {
            req.body = file;
          } else {
            // S3 response code is not HTTP OK error occured during upload
            routeHelper.addError(req, new Error('upload failed'), s3Response.statusCode);
          }
        }
        next();
      });
    });
    form.on('close', onEnd);
    form.parse(req);
  };
Example #22
0
function identify(inputFile, callback){
  var results = {}
    , batch = new Batch()

  soxInfo('-t', function(value) { results.format        = value; });
  soxInfo('-r', function(value) { results.sampleRate    = value; });
  soxInfo('-c', function(value) { results.channelCount  = value; });
  soxInfo('-s', function(value) { results.sampleCount   = value; });
  soxInfo('-D', function(value) { results.duration      = value; });
  soxInfo('-B', function(value) { results.bitRate       = value; });

  batch.end(function(err) {
    if (err) return callback(err);
    for (var k in conversions) {
      results[k] = conversions[k](results[k])
    }
    callback(null, results);
  });

  function soxInfo(arg, assign) {
    batch.push(function(cb) {
      capture('sox', ['--info', arg, inputFile], function(err, value) {
        if (err) return cb(err);
        assign(value);
        cb();
      });
    });
  }
}
Example #23
0
  self.performHook('before ' + type, function(err) {
    if (err) return fn(err);

    // build files
    if (conf[type]) {
      conf[type].forEach(function(file){
        var path = self.path(file);
        batch.push(function(done){
          var val = self._files[file];

          // on disk
          if (null == val) {
            debug('read file %s', path);
            fs.readFile(path, 'utf8', function(err, str){
              if (err) return fn(err);
              done(null, process(self, file, str));
            });
            return
          }

          // fabricated
          done(null, process(self, file, val));
        });
      });
    }

    batch.end(function(err, res){
      if (err) return fn(err);
      fn(null, res.join('\n'));
    });
  });
Example #24
0
Package.prototype.getFiles = function(files, fn){
  var self = this;
  var batch = new Batch;

  files.forEach(function(file){
    batch.push(function(done){
      var url = self.url(file);
      debug('fetching %s', url);
      self.emit('file', file, url);
      var dst = self.join(file);

      // mkdir
      self.mkdir(dirname(dst), function(err){
        if (err) return fn(err);

        // pipe file
        var req = request.get(url);
        req.set('Accept-Encoding', 'gzip');
        req.buffer(false);

        if (self.auth) req.auth(self.auth.user, self.auth.pass);

        req.end(function(res){
          if (res.error) return done(error(res, url));
          res.pipe(fs.createWriteStream(dst));
          res.on('error', done);
          res.on('end', done);
        });
      });
    });
  });

  batch.end(fn);
};
Example #25
0
Mixpanel.prototype.track = function (track, settings, callback) {
  var imported = shouldImport(track)
    , endpoint = imported ? '/import/' : '/track/';

  var payload = {
    event      : track.event(),
    properties : formatProperties(track, settings)
  };

  extend(payload.properties, superProperties(track));

  var req = {
    url : this.baseUrl + endpoint,
    qs  : {
      ip      : track.ip() || 0,
      verbose : 1,
      data    : b64encode(payload),
      api_key : settings.apiKey
    }
  };

  var batch = new Batch()
    , self  = this;

  debug('making track request', track.userId() || track.sessionId());
  batch.push(function (done) { request.post(req, self._parseResponse(done)); });

  if (track.revenue()) {
    batch.push(function (done) { self.revenue(track, settings, done); });
  }

  batch.end(callback);
};
Example #26
0
Mixpanel.prototype.increment = function(track, fn){
  var increments = getIncrements(track, this.settings);
  var batch = new Batch;
  var self = this;

  // ignore
  if (!increments) return tick(fn);
  if (!track.userId()) return tick(fn);

  // send
  batch.push(send('$add'));
  batch.push(send('$set'));
  batch.end(fn);

  // send `type`
  function send(type){
    return function(done){
      var payload = {};
      payload.$distinct_id = track.userId();
      payload.$token = self.settings.token;
      payload.mp_lib = 'Segment.io';
      payload[type] = increments[type];
      var b64 = b64encode(payload);

      return self
        .get('/engage')
        .query({ ip: 0 })
        .query({ verbose: 1 })
        .query({ data: b64 })
        .end(self._parseResponse(done));
    };
  }
};
Example #27
0
Mixpanel.prototype.page = function(msg, fn){
  var batch = new Batch;
  var category = msg.category();
  var name = msg.fullName();
  var self = this;

  // all
  if (this.settings.trackAllPages) {
    batch.push(track(msg.track()));
  }

  // categorized
  if (category && this.settings.trackCategorizedPages) {
    batch.push(track(msg.track(category)));
  }

  // named
  if (name && this.settings.trackNamedPages) {
    batch.push(track(msg.track(name)));
  }

  // call track with `msg`.
  function track(msg){
    return function(done){
      self.track(msg, function(err, arr){
        if (err) return done(err);
        done(null, arr[1]);
      });
    };
  }

  batch.end(fn);
};
Example #28
0
  builder.hook('before styles', function (pkg, cb) {
    if (!pkg.config.styles) return cb();

    var stylusFiles = pkg.config.styles.filter(endsWith('styl'));
    var batch = new Batch();

    stylusFiles.forEach(function (styl) {

      // only load index.styl
      if (!~styl.indexOf('index.styl')) return pkg.removeFile('styles', styl);

      batch.push(function (done) {
        var stylPath = pkg.path(styl);
        var name = styl.split('.')[0] + '.css';

        debug('compiling: %s', styl);
        var paths = [
          cwd,
          cwd + '/components',
          cwd + '/node_modules'
        ];

        var render = stylus(fs.readFileSync(stylPath, 'utf-8'))
          .set('paths', paths)
          .set('include css', true)
          .set('filename', stylPath)
          .use(nib());

        for(var key in options.set) {
          render.set(key, options.set[key]);
        }
        for(var k in options.use) {
          render.use(options.use[k]);
        }

        if (pkg.parent && pkg.parent.config.build && pkg.parent.config.build.stylus && pkg.parent.config.build.stylus.imports) {
          pkg.parent.config.build.stylus.imports.forEach(function (path) {
            var imp = pkg.parent.dir + '/' + path;
            debug('importing: %s', imp);

            render.import(imp);
          });
        }

        render.render(function (err, css) {

          if (err) return done(err);

          pkg.addFile('styles', name, css);
          pkg.removeFile('styles', styl);

          done();
        });

      });
    });

    batch.end(cb);
  });
Example #29
0
 after(function(done) {
   var batch = new Batch();
   batch.push(function(done) { fs.unlink("naught.log", done); });
   batch.push(function(done) { fs.unlink("stderr.log", done); });
   batch.push(function(done) { fs.unlink("stdout.log", done); });
   batch.push(function(done) { fs.unlink("test/settings.json", done); });
   batch.end(done);
 });
Example #30
0
QP.prototype.stop = function(cb) {
  var batch = new Batch();

  for (var i in this.queues) {
    var q = this.queues[i];
    batch.push(q.stop.bind(q));
  }
  batch.end(cb);
};