tributary.loadPlugins = function(plugins, options, cb) {
var q;
if (options.serial) {
q = queue(1);
} else {
q = queue();
}
plugins.forEach(function(plugin) {
var opts = plugin.options;
if (!opts) opts = {};
q.defer(tributary.loadPlugin, plugin.url, opts);
});
q.awaitAll(cb);
};
dynamodb.test('[requests] batchGetItemRequests (single table)', fixtures, function(assert) {
var dyno = Dyno({
table: dynamodb.tableName,
region: 'local',
endpoint: 'http://localhost:4567'
});
var params = { RequestItems: {} };
params.RequestItems[dynamodb.tableName] = {
Keys: _.range(150).map(function(i) {
return { id: i.toString() };
})
};
var found = dyno.batchGetItemRequests(params);
assert.equal(found.length, 2, 'split 150 keys into two requests');
queue()
.defer(found[0].send.bind(found[0]))
.defer(found[1].send.bind(found[1]))
.awaitAll(function(err, results) {
assert.ifError(err, 'requests were sent successfully');
results = results[0].Responses[dynamodb.tableName].concat(results[1].Responses[dynamodb.tableName]);
assert.equal(results.length, 150, 'all responses were recieved');
assert.end();
});
});
function idGeocode(geocoder, asId, options, callback) {
var q = queue();
var extid = asId.dbname + '.' + asId.id;
var indexes = geocoder.byname[asId.dbname];
for (var i = 0; i < indexes.length; i++) {
q.defer(function(source, feat, done) {
feature.getFeature(source, feat, function (err, data) {
if (err) return done(err);
if (!data) return done();
data = data[asId.id];
data._extid = extid;
done(null, data);
});
}, indexes[i], termops.feature(asId.id));
}
q.awaitAll(function(err, features) {
if (err) return callback(err);
var result = {
"type": "FeatureCollection",
"query": [extid],
"features": []
};
for (var i = 0; i < features.length; i++) {
if (!features[i]) continue;
var f = ops.toFeature([features[i]]);
f.relevance = features[i]._score || 0;
result.features.push(f);
}
return callback(null, result);
});
}
test('bin.mapbox-upload-validate: invalid', function(t) {
function validate(type, filepath, reason, callback) {
if (!callback) {
callback = reason;
reason = undefined;
}
exec(['NODE_NO_WARNINGS=1', run, filepath].join(' '), function(err, stdout, stderr) {
console.log('# bin.mapbox-upload-validate: invalid', path.basename(filepath));
t.equal(err.code, 3, 'exit 3');
t.equal(stdout, '', [type, reason].join('.') + ': nothing logged to stdout');
var expect = expected[type + 'Errors'];
if (reason) expect = expect[reason];
t.ok(stderr.indexOf(expect) > -1, [type, reason].join('.') + ': expected error message');
callback();
});
}
var q = queue();
_(fixtures.invalid).forEach(function(val, type) {
if (typeof val === 'string') return q.defer(validate, type, val);
_(val).forEach(function(filepath, reason) {
q.defer(validate, type, filepath, reason);
});
});
q.await(function() {
t.end();
});
});
query: function (opts) {
if (!opts) opts = {};
if (!this.queryable()) return this;
if (this._query) {
this._query.abort();
}
if (this._requests && this._requests.length) this._requests.forEach(function(request) {
request.abort();
});
this._requests = [];
var q = queue();
var pts = [this.origin, this.destination].concat(this._waypoints);
for (var i in pts) {
if (!pts[i].geometry.coordinates) {
q.defer(L.bind(this._geocode, this), pts[i], opts.proximity);
}
}
q.await(L.bind(function(err) {
if (err) {
return this.fire('error', {error: err.message});
}
this._query = request(this.queryURL(), L.bind(function (err, resp) {
this._query = null;
if (err) {
return this.fire('error', {error: err.message});
}
this.directions = resp;
this.directions.routes.forEach(function (route) {
route.geometry = {
type: "LineString",
coordinates: polyline.decode(route.geometry, 6).map(function (c) { return c.reverse(); })
};
});
if (!this.origin.properties.name) {
this.origin = this.directions.origin;
} else {
this.directions.origin = this.origin;
}
if (!this.destination.properties.name) {
this.destination = this.directions.destination;
} else {
this.directions.destination = this.destination;
}
this.fire('load', this.directions);
}, this), this);
}, this));
return this;
},
resolve : function(hosts, dnets, callback) {
var resolveQueue = queue();
/** get the IP address for each host **/
hosts.forEach(function(h) {
resolveQueue.defer(function(callback) {
dns.lookup(h, function (err, addy) {
resolved.hosts[h] = addy;
var res = city.lookupSync(addy);
if (res) {
res.lonlat = [res.latitude, res.longitude];
resolved.geo[h] = res;
}
callback();
});
});
});
/** get the edges in each dnet **/
dnets.forEach(function(c) {
resolveQueue.defer(function(callback) {
dns.resolve4(c, function(err, addies) {
resolved.dnets[c] = addies;
callback()
});
});
});
resolveQueue.awaitAll(function(error) {
callback(resolved);
});
}, getDnet : function(host) {
module.exports = function(geocoder, lon, lat, maxtype, full, callback) {
var context = [];
var indexes = geocoder.indexes;
var types = Object.keys(indexes);
types = types.slice(0, maxtype ? types.indexOf(maxtype) : types.length);
// No-op context.
if (!types.length) return callback(null, context);
var q = queue();
types.forEach(function(type) {
var source = indexes[type];
var bounds = source._geocoder.bounds;
var method = source._geocoder.format === 'pbf' ? contextVector : contextGrid;
if (lat >= bounds[1] && lat <= bounds[3] && lon >= bounds[0] && lon <= bounds[2])
q.defer(method, source, lon, lat, full);
});
q.awaitAll(function(err, res) {
if (err) return callback(err);
res = res.reverse().filter(identity);
callback(null, res);
});
};
function (error, stdout, stderr) {
if (error) {
console.log('password', error, stderr);
res.render("login.ejs", {message: stderr || "Login failed.",
project: project});
return;
}
var author = req.body.username;
queue().defer(db.get_groupid, project)
.defer(AuthorManager.createAuthorIfNotExistsFor, author, author)
.await(function (error, groupid, authordata) {
if (error) { console.log('prelogin', error); return; }
SessionManager.createSession(
groupid, authordata.authorID,
Math.floor(new Date().getTime()/1000 + 7 * 24 * 60 * 60),
function (error, data) {
if (error) { console.log('login', error); return; }
var new_cookie;
if (req.cookies.sessionID) {
new_cookie = req.cookies.sessionID + ','
+ data["sessionID"];
} else {
new_cookie = data["sessionID"];
}
res.cookie("sessionID", new_cookie);
var dest = req.params[1];
if (!dest) {
dest = '/g/' + project + '/e/';
}
res.redirect(dest);
});
});
});
function fetch(base_url, callback) {
var results = {},
q = queue(4);
// Get the layer's metadata
request.get({ url: base_url + '?f=json', json: true }, onload);
function onload(error, response, metadata) {
if (error || response.statusCode != 200) {
return callback(error);
}
if (metadata.capabilities && metadata.capabilities.indexOf('Query') < 0) {
return callback("Layer doesn't support query operation.");
}
var oidField = findOidField(metadata.fields);
if (!metadata.extent) return callback("Layer doesn't list an extent.");
if (!oidField) return callback("Could not find an OID field.");
if ('subLayers' in metadata && metadata.subLayers.length > 0) {
return callback("Specified layer has sublayers.");
}
// Start by requesting the world bounding box for the layer
q.defer(fetchBbox, base_url, metadata.extent, oidField.name, results, q);
q.awaitAll(function(error, data) {
var features = [];
Object.keys(results).forEach(function(key) {
features.push(results[key]);
});
return callback(error, metadata.geometryType, features);
});
}
}
function wipe(source, callback) {
var s = source;
var q = queue(100);
var shardlevel = s._geocoder.shardlevel;
var types = ['degen','term','freq','phrase','grid','feature'];
s.startWriting(function(err) {
if (err) return callback(err);
for (var j = 0; j < types.length; j++) {
var type = types[j];
var limit = type === 'feature' ?
Math.pow(16,shardlevel+1) :
Math.pow(16,shardlevel);
for (var i = 0; i < limit; i++) {
q.defer(transferGeocoderData, type, i);
}
}
function transferGeocoderData(type, shard, callback) {
s.getGeocoderData(type, shard, function(err, buffer) {
if (!buffer) return callback();
s.putGeocoderData(type, shard, type === 'feature' ? '{}' : new Buffer(0), callback);
});
}
q.awaitAll(function(err) {
if (err) return callback(err);
s.stopWriting(callback);
});
});
}
s3.listObjects(opts, function(err, data){
if (err) return callback(err);
if (!data.Contents.length) return callback();
var q = queue(10);
var emitted = 0;
_(data.Contents).each(function(obj) {
q.defer(function(callback) {
emit(obj.Key, function(err, e) {
if (e) emitted++;
callback(err, e);
});
});
});
q.awaitAll(function(err) {
if (err) return callback(err);
log('emitted %d of %d keys', emitted, data.Contents.length);
saveState(_(data.Contents).last().Key, function(err) {
if (err) return callback(err);
if (data.IsTruncated) {
opts.Marker = _(data.Contents).last().Key;
fetch(opts);
} else {
callback();
}
});
});
});
Cache.prototype.loadall = function(getter, type, ids, callback) {
var cache = this;
var queues = Cache.shards(type, ids);
var allshards = Object.keys(queues);
var shards = [];
for (var i = 0; i < allshards.length; i++) {
if (!cache.has(type, +allshards[i])) shards.push(allshards[i]);
}
var q = queue(10);
for (var i = 0; i < shards.length; i++) {
q.defer(getter, type, +shards[i]);
}
q.awaitAll(function(err, buffers) {
if (err) return callback(err);
for (var i = 0; i < buffers.length; i++) {
if (!buffers[i]) continue;
// Sync load is used because async call is
// experimental/not yet stable
try {
cache.loadSync(buffers[i], type, +shards[i]);
} catch (e) {
return callback(e);
}
}
callback(err, allshards, queues);
});
};
Store.prototype.init = function(db, cb) {
var SlowResource = this.SlowResource = db.define('SlowResource', {
key: { type: 'text', index: true },
url: { type: 'text' },
mtime: { type: 'date', time: true, index: true },
maxage: { type: 'integer' },
headers: { type: 'object' }, // a hash
valid: { type: 'boolean', defaultValue: true },
data: { type: 'binary' },
code: { type: 'integer', defaultValue: 200 },
builder: { type: 'text' }
}, {
table: "raja_cached_resources",
cache: false, // we cache better :)
hooks: {
beforeSave: beforeSaveResource
}
});
SlowResource.hasMany("children", SlowResource, {order: {type: 'integer'}}, {
reverse : "parents",
key: true
});
var self = this;
var q = queue(1);
if (this.reset) {
q.defer(SlowResource.drop);
}
q.defer(SlowResource.sync).awaitAll(function(err) {
cb(err, self);
});
};
function checkS3(keys) {
assert.equal(keys.length, 50, 'all records written to S3');
var q = queue(20);
records.forEach(function(expected) {
var key = crypto.createHash('md5')
.update(Dyno.serialize({ id: expected.id }))
.digest('hex');
key = [prefix, dynamodb.tableName, key].join('/');
expected = Dyno.serialize(expected);
assert.ok(keys.indexOf(key) > -1, 'expected item written for ' + key);
q.defer(function(next) {
s3.getObject({
Bucket: bucket,
Key: key
}, function(err, data) {
if (err) throw err;
assert.equal(data.Body.toString(), expected, 'expected data for ' + key);
next();
});
});
});
q.awaitAll(function() {
assert.end();
});
}
module.exports = function(files, options, callback) {
if (typeof options === "function") callback = options, options = {};
var fileMap = {};
function readRecursive(file, callback) {
if (file in fileMap) return callback(null);
readImports(file, function(error, files) {
if (error) {
if (options["ignore-missing"] && error.code === "ENOENT") files = [];
else return void callback(error);
}
var q = queue(1);
fileMap[file] = files;
files.forEach(function(file) {
q.defer(readRecursive, file);
});
q.awaitAll(callback);
});
}
var q = queue(1);
files.forEach(function(file) {
q.defer(readRecursive, expandFile(file));
});
q.awaitAll(function(error) {
callback(error, error ? null : fileMap);
});
};
test('insert datasets and listDatasets', function(t) {
var cardboard = Cardboard(config);
var q = queue(1);
q.defer(function(cb) {
cardboard.put(fixtures.haiti, 'haiti', function() {
cb();
});
});
q.defer(function(cb) {
cardboard.put(fixtures.dc, 'dc', function() {
cb();
});
});
q.awaitAll(getDatasets);
function getDatasets() {
cardboard.listDatasets(function(err, res) {
t.notOk(err, 'should not return an error');
t.ok(res, 'should return a array of datasets');
t.equal(res.length, 2);
t.equal(res[0], 'dc');
t.equal(res[1], 'haiti');
t.end();
});
}
});
to.startWriting(function(err) {
if (err) return callback(err);
var q = queue(2);
var types = ['freq','grid','stat','feature'];
for (var j = 0; j < types.length; j++) {
var type = types[j];
var limit = type === 'feature' ? Math.pow(2,20) : cache.shard(type, Math.pow(2,52)) || 1;
// Chunk shards ops in groups of 10000 to avoid
// exceeding call stack.
for (var i = 0; i < limit; i = i + 10000) q.defer(function(type, i, callback) {
var subq = queue(200);
for (var k = 0; k < 10000; k++) subq.defer(function(type, shard, callback) {
from.getGeocoderData(type, shard, function(err, buffer) {
if (err) return callback(err);
if (!buffer) return callback();
to.putGeocoderData(type, shard, buffer, callback);
});
}, type, i + k);
subq.awaitAll(callback);
}, type, i);
}
q.awaitAll(function(err) {
if (err) return callback(err);
to.stopWriting(function(err) {
if (err) return callback(err);
to._geocoder.unloadall('freq');
to._geocoder.unloadall('grid');
to._geocoder.unloadall('stat');
callback();
});
});
});
process.on('message', function(tile) {
var q = queue();
for (var i = 0; i < sources.length; i++) {
q.defer(sources[i].getTile, tile);
}
q.awaitAll(gotData);
function gotData(err, results) {
if (err) throw err;
var data = {};
for (var i = 0; i < results.length; i++) {
data[sources[i].name] = results[i];
if (!results[i]) return process.send({reduce: true});
}
function gotResults(err, value) {
if (err) throw err;
process.send({reduce: true, value: value, tile: tile});
}
map(data, tile, write, gotResults);
}
});
q.await(function (err, archives) {
if (err) {
//TODO better error handling?
res.send(err);
return;
}
//console.log('archives', archives);
q = queue();
q.defer(pmweb.getContexts, archives);
q.await(function(err, contexts) {
//console.log('contexts', contexts);
q = queue();
q.defer(pmweb.getMetrics, {metric: ctx.metric, archives: archives});
q.await(function(err, metrics) {
//console.log('metrics', metrics);
res.send(metrics);
});
});
});
function query(err) {
if (err) throw err;
var readResult = [];
var writeResult = [];
queue(1)
.defer(function(next) {
s.dyno.query({id: {EQ: readItems[0].id}, range: {EQ: readItems[0].range}})
.on('data', function(item) {
readResult.push(item);
})
.on('end', next);
})
.defer(function(next) {
s.dyno.query({id: {EQ: writeItems[4].id}, range: {EQ: writeItems[4].range}})
.on('data', function(item) {
writeResult.push(item);
})
.on('end', next);
})
.await(function(err) {
t.ifError(err, 'no error');
t.deepEqual(readResult, [readItems[0]], 'queried read table');
t.deepEqual(writeResult, [], 'did not query write table');
t.end();
});
}
var schedule = function(items, wait, samples, done) {
var start = +(new Date);
var q = queue();
_(items).each(function(v) {
q.defer(function(ts, cb) {
v.run(ts, function(err) {
if (err) return cb(err)
if (argv.verbose) v.report();
cb();
});
}, (start / 1000 | 0));
});
q.await(function(err){
if (err) return done(err);
if (samples !== false) {
samples--;
if (samples <= 0) return done();
}
var delay = wait - (+(new Date) - start);
if (delay < 0) {
return done(new Error('command is too slow.'));
}
setTimeout(function() {
schedule(items, wait, samples, done);
}, delay);
});
};
function parseResults(result, done) {
var candidates = {};
resultIndex += 1;
// console.log(result.link);
if (!notFalsePositiveURL(result.link) ||
!searchResultIsNotFromADictionary(result)) {
// console.log('Filtering result from:', result.link);
done();
return;
}
var q = queue();
var categorizedResult = categorizeByPrecedingOf(result);
q.defer(
getNounCandidates, wordnok, categorizedResult.wordsNotPrecedingOf
);
q.defer(getNounCandidates, wordnok, categorizedResult.wordsPrecedingOf);
q.await(sumUp);
function sumUp(error, reportNotPrecedingOf, reportPrecedingOf) {
if (error) {
done(error);
}
else {
candidates.nouns = reportNotPrecedingOf;
candidates.beforeOfNouns = reportPrecedingOf;
applyFiltersToCandidates(candidates);
done(null, candidates);
}
}
}
initialize: function(options) {
_.bindAll(this);
var app = options.app;
app.loader.start();
this.app = app;
this.branch = options.branch || this.model.get('master_branch');
this.model = options.model;
this.nav = options.nav;
this.path = options.path || '';
this.router = options.router;
this.sidebar = options.sidebar;
// Init subviews
this.initBranches();
this.initHeader();
var q = queue();
q.defer(this.initSearch);
q.defer(this.initHistory);
q.awaitAll(this.initFiles);
// Events from sidebar
this.listenTo(this.sidebar, 'destroy', this.destroy);
this.listenTo(this.sidebar, 'cancel', this.cancel);
this.listenTo(this.sidebar, 'confirm', this.updateFile);
app.loader.done();
},
function putFeatures(source, docs, callback) {
var byshard = {};
for (var i = 0; i < docs.length; i++) {
var doc = docs[i];
var s = shard(source._geocoder.shardlevel, doc._hash);
byshard[s] = byshard[s] || [];
byshard[s].push(doc);
}
var q = queue(100);
for (var s in byshard) q.defer(function(s, docs, callback) {
source.getGeocoderData('feature', s, function(err, buffer) {
if (err) return callback(err);
try {
var current = buffer && buffer.length ? JSON.parse(buffer) : {};
for (var i = 0; i < docs.length; i++) {
var doc = docs[i];
var entry = current[doc._hash] ? JSON.parse(current[doc._hash]) : {};
entry[doc._id] = doc;
current[doc._hash] = JSON.stringify(entry, cleanDoc);
}
source.putGeocoderData('feature', s, JSON.stringify(current), callback);
} catch (err) { return callback(err) }
});
}, s, byshard[s]);
q.awaitAll(callback);
}
it('.listWorkInProgressJobs() should retrieve WIP users', function (done) {
var jobs = [{ user: '******', id: 'jobId1' }, { user: '******', id: 'jobId2' }, { user: '******', id: 'jobId3' }];
var testStepsQueue = queue(1);
jobs.forEach(function (job) {
testStepsQueue.defer(jobBackend.addWorkInProgressJob.bind(jobBackend), job.user, job.id);
});
testStepsQueue.awaitAll(function (err) {
if (err) {
done(err);
}
jobBackend.listWorkInProgressJobs(function (err, users) {
if (err) {
return done(err);
}
assert.ok(users.userA);
assert.deepEqual(users.userA, [ 'jobId1', 'jobId2' ]);
assert.ok(users.userB);
assert.deepEqual(users.userB, [ 'jobId3' ]);
done();
});
});
});
function inserted() {
var q = queue(1);
queries.forEach(function(query) {
q.defer(function(query, callback) {
t.equal(cardboard.bboxQuery(query.query, 'default', function(err, resp) {
t.ifError(err, 'no error for ' + query.query.join(','));
if (err) return callback(err);
t.equal(resp.features.length, query.length, query.query.join(',') + ' finds ' + query.length + ' data with a query');
callback();
}), undefined, '.bboxQuery');
}, query);
});
q.awaitAll(function(err) {
t.ifError(err, 'queries passed');
t.equal(cardboard.list('default', function(err, resp) {
t.ifError(err, 'no error for list');
if (err) throw err;
var length = queries.reduce(function(memo, query) {
return memo + query.length;
}, 0);
t.equal(resp.features.length, length, 'list finds ' + length + ' data with a query');
t.end();
}), undefined, '.list');
});
}
var compareAll = function(newVersion, oldVersion, callback) {
var q = queue(4);
var comparators = [
landmarkScore,
nameModified,
disputedBorderTagChanged,
disputedBorderDeleted,
invalidHighwayTags,
largeBuilding,
majorRoadChanged,
pathRoadChanged
];
comparators.forEach(function(comparator) {
q.defer(comparator, newVersion, oldVersion);
});
q.awaitAll(function(err, results) {
if (err) {
callback(err);
}
callback(
err,
results.reduce(
function(problems, result) {
return extend(problems, result);
},
{}
)
);
});
};
function inserted(err) {
t.notOk(err, 'no error returned');
var queries = [
{
query: [-10, -10, 10, 10],
length: 0
},
{
query: [-76.0, 38.0, -79, 40],
length: 0
}
];
var q = queue(1);
queries.forEach(function(query) {
q.defer(function(query, callback) {
t.equal(cardboard.bboxQuery(query.query, 'default', function(err, resp) {
t.equal(err, null, 'no error for ' + query.query.join(','));
t.equal(resp.features.length, query.length, 'finds ' + query.length + ' data with a query');
callback();
}), undefined, '.bboxQuery');
}, query);
});
q.awaitAll(function() { t.end(); });
}
module.exports = function(files, argv, callback) {
var fileMap = {},
requiresKey = argv.requiresKey || "requires";
pathKey = argv.pathKey || "path",
targetDir = argv.targetDir || "",
modulePrefix = argv.modulePrefix || "",
separator = argv.separator || "-",
basePath = argv.basePath || "";
function readRecursive(file, parent, callback) {
var moduleKey = fileToModuleKey(file, basePath, separator, modulePrefix);
if (typeof parent === "string") fileMap[parent][requiresKey].push(moduleKey);
if (moduleKey in fileMap) return callback(null);
readImports(file, function(error, files) {
if (error) return void callback(error);
var q = queue(1);
fileMap[moduleKey] = {};
fileMap[moduleKey][pathKey] = targetDir + stripBasePath(file, basePath);
fileMap[moduleKey][requiresKey] = files.length > 0 ? [] : undefined;
files.forEach(function(file) {
q.defer(readRecursive, file, moduleKey);
});
q.awaitAll(callback);
});
}
var q = queue(1);
files.forEach(function(file) {
q.defer(readRecursive, expandFile(file), null);
});
q.awaitAll(function(error) {
callback(error, error ? null : fileMap);
});
};
function load(id) {
legID = id;
queue()
.defer(d3.csv, 'assets/crp_categories_sector-industries1.csv')
.defer(d3.json, 'assets/114_legislator_sector_support.json')
.await(render);
// d3.csv('assets/crp_categories_sector-industries.csv', function(error, data) {
// comboChart(data);
// });
// d3.json('assets/114_top10_oppose_industries.json', function(error, data) {
// if (error) {
// renderError();
// } else {
// opposeChart(data[id]);
// }
// });
// d3.json('assets/114_top10_support_industries.json', function(error, data) {
// if (error) {
// renderError();
// } else {
// supportChart(data[id]);
// }
// });
}