fs.readdir(path.join(test_root, test_path), function (err, files) {
if (err) return cb(err);
files.sort();
if (! /\.gz$/.test(files[0])) {
files.push(files.shift());
}
var pend = new Pend();
var results;
files.forEach(function(file) {
pend.go(function(cb) {
fs.readFile(path.join(test_root, test_path, file), function (err, data) {
if (err) return cb(err);
if (/\.gz$/.test(file)) {
zlib.gunzip(data, function (err, data) {
if (err) return cb(err);
results = {file: file, data: data};
cb();
});
} else {
results = {file: file, data: data};
cb();
}
});
});
});
pend.wait(function(err) {
if (err) return cb(err);
var fullData = "";
results.forEach(function(item) {
fullData += item.data.toString();
});
cb(null, results, fullData);
});
});
form.on('close', function() {
res.end('OK');
pend.wait(function(err) {
if (err) throw err;
callback(null, parts);
});
});
fn: function (cb) {
var pend = new Pend();
files.forEach(function(file) {
rimraf(path.join(test_root, file), pend.hold());
});
pend.wait(cb);
},
PlayerServer.prototype.init = function(cb) {
var self = this;
var pend = new Pend();
pend.go(loadAllUsers);
pend.go(loadAllEvents);
pend.go(loadLastFmState);
pend.wait(cb);
function loadAllUsers(cb) {
dbIterate(self.db, USERS_KEY_PREFIX, processOne, function(err) {
if (err) return cb(err);
self.ensureGuestUser();
self.computeUsersIndex();
self.emit('users');
self.emit('haveAdminUser');
cb();
});
function processOne(key, value) {
var user = deserializeUser(value);
self.users[user.id] = user;
}
}
function loadAllEvents(cb) {
dbIterate(self.db, EVENTS_KEY_PREFIX, processOne, function(err) {
if (err) return cb(err);
self.cacheEventsArray();
self.emit('events');
cb();
});
function processOne(key, value) {
var ev = deserializeEvent(value);
self.events[ev.id] = ev;
}
}
function loadLastFmState(cb) {
self.db.get(LASTFM_DB_KEY, function(err, value) {
if (err) {
var notFoundError = /^NotFound/.test(err.message);
if (!notFoundError) return cb(err);
} else {
var state;
try {
state = JSON.parse(value);
} catch (err) {
cb(new Error("unable to parse lastfm state: " + err.message));
return;
}
self.scrobblers = state.scrobblers;
self.scrobbles = state.scrobbles;
}
// in case scrobbling fails and then the user presses stop, this will still
// flush the queue.
setInterval(self.flushScrobbleQueue.bind(self), 120000);
cb();
});
}
};
function deployStart(cb){
if (waitingFor === 'shutdown') {
return cb('ErrorShuttingDown');
} else if (waitingFor != null) {
return cb('ErrorDeployInProgress');
}
assert.strictEqual(workers.booting.count, 0);
waitingFor = 'new';
var pend = new Pend();
var count = workerCount;
for (var i = 0; i < count; i += 1) {
spawnNew(i, pend.hold());
}
pend.wait(function() {
if (workers.new_online.count !== workerCount) {
deployAbort(function() {
cb('DeployFailed');
});
return;
}
waitingFor = 'old';
var onlineCount = workers.online.count;
for (var i = 0; i < onlineCount; ++i) {
pend.go(shutdownOneWorker('online'));
}
pend.wait(function() {
assert.strictEqual(workers.online.count, 0);
waitingFor = null;
forEachWorker('new_online', function(pid, worker){
setWorkerStatus(worker, 'online');
});
cb('Ready');
});
});
}
function readKeyAndCert(cb) {
var pend = new Pend();
var options = {};
pend.go(function(cb) {
fs.readFile(self.config.sslKey, function(err, data) {
if (err) {
console.error("Unable to read SSL key file: " + err.message);
process.exit(1);
return;
}
options.key = data;
cb();
});
});
pend.go(function(cb) {
fs.readFile(self.config.sslCert, function(err, data) {
if (err) {
console.error("Unable to read SSL cert file: " + err.message);
process.exit(1);
return;
}
options.cert = data;
cb();
});
});
pend.wait(function() {
self.httpServer = https.createServer(options, self.app);
cb();
});
}
PlayerServer.deleteAllUsers = function(db) {
var cmds = [];
var usersDeleted = 0;
var eventsDeleted = 0;
var pend = new Pend();
pend.go(function(cb) {
dbIterate(db, USERS_KEY_PREFIX, processOne, cb);
function processOne(key, value) {
cmds.push({type: 'del', key: key});
usersDeleted += 1;
}
});
pend.go(function(cb) {
dbIterate(db, EVENTS_KEY_PREFIX, processOne, cb);
function processOne(key, value) {
cmds.push({type: 'del', key: key});
eventsDeleted += 1;
}
});
pend.wait(function(err) {
if (err) throw err;
db.batch(cmds, function(err) {
if (err) throw err;
log.info("Users deleted: " + usersDeleted);
log.info("Events deleted: " + eventsDeleted);
process.exit(0);
});
});
};
function createLogsAndIpcServer(cb) {
var pend = new Pend();
createLogs(pend.hold());
mkdirp(path.dirname(socketPath), pend.hold());
pend.wait(function(err) {
if (err) return cb(err);
server = net.createServer(function(newSocket){
if (socket != null) {
log("Warning: Only one connection to daemon allowed. Terminating old connection.\n");
socket.destroy();
}
socket = newSocket;
socket.on('error', function(err){
log("Error: ipc channel socket: " + err.stack + "\n");
});
socket.once('end', function(){
socket = null;
});
jsonSocket.listen(socket, function(msg){
var response = handleSocketMessage(msg);
if (response) {
jsonSocket.send(response);
}
});
});
server.listen(socketPath, cb);
});
}
function deployAbort(cb){
switch (waitingFor) {
case 'new':
var pend = new Pend();
var i;
var newOnlineCount = workers.new_online.count;
var bootingCount = workers.booting.count;
for (i = 0; i < newOnlineCount; i += 1) {
pend.go(destroyWorkers('new_online'));
}
for (i = 0; i < bootingCount; i += 1) {
pend.go(destroyWorkers('booting'));
}
pend.wait(function() {
waitingFor = null;
cb();
});
break;
case 'old':
destroyDying();
cb();
break;
default:
event('ErrorNoDeployInProgress');
}
}
form.parse(request, function(err, fields, files) {
if (err) return next(err);
var keys = [];
var pend = new Pend();
for (var key in files) {
var arr = files[key];
for (var i = 0; i < arr.length; i += 1) {
var file = arr[i];
pend.go(makeImportFn(file));
}
}
pend.wait(function() {
response.json(keys);
});
function makeImportFn(file) {
return function(cb) {
self.player.importFile(file.path, file.originalFilename, function(err, dbFile) {
if (err) {
console.error("Unable to import file:", file.path, "error:", err.stack);
} else if (!dbFile) {
console.error("Unable to locate new file due to race condition");
} else {
keys.push(dbFile.key);
}
cb();
});
};
}
});
JobQueue.prototype.shutdown = function(callback) {
var self = this;
self.shuttingDown = true;
clearInterval(self.flushStaleInterval);
var pend = new Pend();
pend.go(function(cb) {
self.redisClient.quit(function(err) {
/* We want to kill this instance of the redis client, so at at this point we
* are not interested in what connection errors Redis comes up with.
*/
self.redisClient.end();
cb();
});
});
self.childProcesses.forEach(function(child) {
pend.go(function(cb) {
child.on('exit', cb);
child.send('shutdown');
});
});
if (self.childWorker) {
pend.go(shutdownChildWorker);
}
pend.wait(function() {
callback();
});
function shutdownChildWorker(cb) {
self.childWorker.shutdown(cb);
}
};
form.on('close', function() {
pend.wait(function() {
if (allDbFiles.length > 0) {
playerServer.handleImportedTracks(request.client, allDbFiles, autoQueue);
}
response.json({});
});
});
walker.on('file', function(file, stat, linkPath) {
var usePath = linkPath || file;
if (ignoreFile(usePath)) return;
var relName = '/' + path.relative(dir, usePath).replace('\\', '/');
var compressedSink = new StreamSink();
var uncompressedSink = new StreamSink();
var hashSink = new StreamSink();
var inStream = fs.createReadStream(file);
var cacheObj;
cache[relName] = cacheObj = {
sink: null,
mime: mime.lookup(relName),
mtime: stat.mtime,
hash: null,
compressed: null,
};
var fileDone = pend.hold();
var thisPend = new Pend();
var gzipPendCb = thisPend.hold();
var hashPendCb = thisPend.hold();
var uncompressedPendCb = thisPend.hold();
inStream.on('error', function(err) {
if (err.code === 'EISDIR') {
delete cache[relName];
gzipPendCb();
uncompressedPendCb();
hashPendCb();
} else {
walker.stop();
gzipPendCb(err);
uncompressedPendCb(err);
hashPendCb(err);
}
});
inStream.pipe(zlib.createGzip()).pipe(compressedSink);
compressedSink.on('finish', gzipPendCb);
inStream.pipe(uncompressedSink);
uncompressedSink.on('finish', uncompressedPendCb);
inStream.pipe(crypto.createHash('sha1')).pipe(hashSink);
hashSink.on('finish', function() {
cacheObj.hash = hashSink.toString('base64');
hashPendCb();
});
thisPend.wait(function(err) {
if (err) return fileDone(err);
var compressionRatio = compressedSink.length / uncompressedSink.length;
if (compressionRatio >= 0.95) {
// 95% of original size or worse. discard compressed sink
cacheObj.sink = uncompressedSink;
cacheObj.compressed = false;
} else {
// better than 95% of original size. discard uncompressed sink
cacheObj.sink = compressedSink;
cacheObj.compressed = true;
}
fileDone();
});
});
zipfile.on("end", function() {
entryProcessing.wait(function() {
for (var fileName in expectedArchiveContents) {
throw new Error(testId + fileName + ": missing file");
}
console.log(testId + "pass");
zipfileCallback();
});
});
function readKeyAndCert(cb) {
var pend = new Pend();
var options = {};
pend.go(function(cb) {
fs.readFile(config.sslKey, function(err, data) {
if (err) {
log.fatal("Unable to read SSL key file: " + err.message);
process.exit(1);
return;
}
options.key = data;
cb();
});
});
pend.go(function(cb) {
fs.readFile(config.sslCert, function(err, data) {
if (err) {
log.fatal("Unable to read SSL cert file: " + err.message);
process.exit(1);
return;
}
options.cert = data;
cb();
});
});
pend.go(function(cb) {
if (!config.sslCaDir) return cb();
options.ca = [];
fs.readdir(config.sslCaDir, function(err, fileList) {
if (err) {
log.fatal("Unable to read SSL CA dir: " + err.message);
process.exit(1);
return;
}
fileList.forEach(function(file) {
pend.go(function(cb) {
var caPath = path.join(config.sslCaDir, file);
fs.readFile(caPath, function(err, data) {
if (err) {
log.fatal("Unable to read SSL CA file: " + err.message);
process.exit(1);
return;
}
options.ca.push(data);
cb();
});
});
});
cb();
});
});
pend.wait(function() {
httpServer = httpolyglot.createServer(options, app);
cb();
});
}
db.open(function(err) {
if (err) {
if (exitGracefullyIfRunning &&
/^IO error: lock.*: Resource temporarily unavailable$/.test(err.message))
{
return;
} else {
throw err;
}
}
if (deleteAllUsers || deleteAllEvents) {
// this will call process.exit when done
PlayerServer.deleteUsersAndEvents(db, deleteAllUsers);
return;
}
var app = express();
var sslEnabled = !!(config.sslKey && config.sslCert);
if (sslEnabled) {
app.use(redirectHttpToHttps);
}
var pend = new Pend();
pend.go(function(cb) {
var options = {
dir: path.join(__dirname, "../public"),
aliases: [],
};
createGzipStatic(options, function(err, middleware) {
if (err) return cb(err);
app.use(middleware);
cb();
});
});
pend.go(function(cb) {
createGzipStatic({dir: path.join(__dirname, "../src/public")}, function(err, middleware) {
if (err) return cb(err);
app.use(middleware);
cb();
});
});
pend.wait(function(err) {
if (err) throw err;
var player = new Player(db, config.musicDirectory,
config.encodeQueueDuration, config.googleApiKey);
player.initialize(function(err) {
if (err) throw err;
log.debug("Player initialization complete.");
startServer(app, db, player, config, sslEnabled);
});
});
});
return function(done) {
this.timeout(4000);
var testPath = path.join(templatePath, testName);
var expectedHtml, actualHtml;
var expectedText, actualText;
var pend = new Pend();
// Read the JSON file & render the HTML
pend.go(function(cb) {
attemptReadFile(testPath + '.json', 'utf8', function(err, data) {
var context = {};
if (err) return cb(err);
if (data) context = JSON.parse(data);
templates.render(testName + '.html', context, function(err, html, text) {
actualHtml = html;
actualText = text;
cb(err);
});
});
});
// Load the expected HTML
pend.go(function(cb) {
attemptReadFile(testPath + '.out.html', 'utf8', function(err, html) {
expectedHtml = html;
cb(err);
});
});
// Try loading the expected text (may not exist)
pend.go(function(cb) {
attemptReadFile(testPath + '.out.txt', 'utf8', function(err, text) {
expectedText = text;
cb(err);
});
});
// And when they're all done...
pend.wait(function(err) {
if (err) return done(err);
if (expectedHtml)
assert.strictEqual(actualHtml.trim(), expectedHtml.trim());
if (expectedText)
assert.strictEqual(actualText.trim(), expectedText.trim());
done();
});
};
function getDeployDiff(packageJson, targetName, branch, format, cb) {
var exec = require('child_process').exec;
var pend = new Pend();
var revision;
pend.go(function(cb) {
var sshConf = packageJson.rodent.targets[targetName].ssh;
var firstHost = sshConf.hosts[0];
var destAppPath = appPath(packageJson, targetName);
var cmd = "ssh " +
"-o ForwardAgent=yes " +
"-p " + sshConf.port + " " +
sshConf.user + "@" + firstHost + " " +
"'cd " + destAppPath + " && git rev-parse HEAD'";
exec(cmd, function(err, stdout, stderr) {
if (err) {
err.stderr = stderr;
err.stdout = stdout;
err.cmd = cmd;
cb(err);
} else {
revision = stdout.trim();
cb();
}
});
});
pend.go(function(cb) {
var cmd = "git fetch origin";
exec(cmd, function(err, stdout, stderr) {
if (err) {
err.stderr = stderr;
err.stdout = stdout;
err.cmd = cmd;
cb(err);
} else {
cb();
}
});
});
pend.wait(function(err) {
if (err) return cb(err);
var cmd = "git log --pretty=format:\"" + format + "\" " + revision + "..origin/" + branch;
exec(cmd, function(err, stdout, stderr) {
if (err) {
err.stderr = stderr;
err.stdout = stdout;
err.cmd = cmd;
cb(err);
} else {
cb(null, stdout.trim());
}
});
});
}
form.on('close', function() {
pend.wait(function() {
if (allDbFiles.length >= 1) {
var user = request.client && request.client.user;
playerServer.addEvent(user, 'import', null, allDbFiles[0].key, allDbFiles.length);
if (autoQueue) {
player.sortAndQueueTracks(allDbFiles);
playerServer.addEvent(user, 'queue', null, allDbFiles[0].key, allDbFiles.length);
}
}
response.json({});
});
});
Client.prototype.deleteObjects = function(s3Params) {
var self = this;
var ee = new EventEmitter();
var params = {
Bucket: s3Params.Bucket,
Delete: extend({}, s3Params.Delete),
MFA: s3Params.MFA,
};
var slices = chunkArray(params.Delete.Objects, MAX_DELETE_COUNT);
var errorOccurred = false;
var pend = new Pend();
ee.progressAmount = 0;
ee.progressTotal = params.Delete.Objects.length;
slices.forEach(uploadSlice);
pend.wait(function(err) {
if (err) {
ee.emit('error', err);
return;
}
ee.emit('end');
});
return ee;
function uploadSlice(slice) {
pend.go(function(cb) {
doWithRetry(tryDeletingObjects, self.s3RetryCount, self.s3RetryDelay, function(err, data) {
if (err) {
cb(err);
} else {
ee.progressAmount += slice.length;
ee.emit('progress');
ee.emit('data', data);
cb();
}
});
});
function tryDeletingObjects(cb) {
self.s3Pend.go(function(pendCb) {
params.Delete.Objects = slice;
self.s3.deleteObjects(params, function(err, data) {
pendCb();
cb(err, data);
});
});
}
}
};
function boot() {
var pend = new Pend();
db.open(pend.hold());
pend.go(createHttpServer);
pend.wait(function(err) {
if (err) throw err;
cacheAllDb(function(err) {
if (err) throw err;
httpServer.listen(httpPort, httpHost, function() {
console.info("HTTP server listening at http://" + httpHost + ":" + httpPort + "/");
});
});
});
}
return function(cb) {
var pend = new Pend();
var expectedHtml, actualHtml;
var expectedText, actualText;
pend.go(function(cb) {
swigEmails.render(templateName + '.html', {
context: context,
urlRewrite: rewrite
}, function(err, html, text) {
actualHtml = html;
actualText = text;
cb(err);
});
});
pend.go(function(cb) {
var filename = path.join(__dirname, "templates", templateName + ".out.html");
fs.readFile(filename, 'utf8', function(err, val) {
expectedHtml = val;
cb(err);
});
});
pend.go(function(cb) {
var filename = path.join(__dirname, "templates", templateName + ".out.txt");
fs.readFile(filename, 'utf8', function(err, val) {
if (err) {
if (err.code === 'ENOENT') {
expectedText = null;
cb();
} else {
cb(err);
}
} else {
expectedText = val;
cb();
}
});
});
pend.wait(function(err) {
if (err) return cb(err);
if (actualHtml.trim() !== expectedHtml.trim()) {
return cb(new Error("Invalid html"));
}
if (expectedText) {
if (actualText.trim() !== expectedText.trim()) return cb(new Error("Invalid text"));
}
cb();
});
};
function cleanup() {
playlist.clear();
files.forEach(function(o) {
pend.go(function(cb) {
o.file.close(cb);
});
});
pend.wait(function(err) {
if (err) throw err;
player.detach(function(err) {
if (err) throw err;
playlist.destroy();
});
});
}
function shutdownAll(cb){
if (waitingFor === 'shutdown') {
event('AlreadyShuttingDown');
return;
}
waitingFor = 'shutdown';
var pend = new Pend();
statusesToShutdown.forEach(function(status) {
var count = workers[status].count;
for (var i = 0; i < count; ++i) {
pend.go(shutdownOneWorker(status));
}
});
pend.wait(cb);
}
function createLogs(cb) {
var pend = new Pend();
pend.go(function(cb) {
maybeCreateLog(logNaughtPath, function(err, results) {
naughtLog = results.log;
naughtLogBehavior = results.behavior;
cb(err);
});
});
pend.go(function(cb) {
maybeCreateLog(logStderrPath, function(err, results) {
stderrLog = results.log;
stderrBehavior = results.behavior;
cb(err);
});
});
pend.go(function(cb) {
maybeCreateLog(logStdoutPath, function(err, results) {
stdoutLog = results.log;
stdoutBehavior = results.behavior;
cb(err);
});
});
pend.wait(function(err) {
if (err) return cb(err);
if (naughtLogBehavior === 'inherit') {
naughtLog = process.stderr;
}
if (stderrBehavior === 'pipe') {
stderrLog.on('error', function(err) {
log("Error writing to " + logStderrPath + ": " + err.stack + "\n");
});
}
if (stdoutBehavior === 'pipe') {
stdoutLog.on('error', function(err) {
log("Error writing to " + logStdoutPath + ": " + err.stack + "\n");
});
}
if (naughtLogBehavior === 'pipe') {
naughtLog.on('error', function(err){
process.stderr.write("Error writing to " + logNaughtPath + ": " + err.stack + "\n");
});
}
cb();
});
}
self.s3.putObject(s3Params, function(err, data) {
pendCb();
if (fatalError) return;
if (err) {
cb(err);
return;
}
pend.wait(function() {
if (fatalError) return;
if (!compareMultipartETag(data.ETag, localFileStat.multipartETag)) {
cb(new Error("ETag does not match MD5 checksum"));
return;
}
cb(null, data);
});
});
function assertFilesMd5(list, cb) {
var pend = new Pend();
list.forEach(function(o) {
pend.go(function(cb) {
var inStream = fs.createReadStream(o.path);
var hash = crypto.createHash('md5');
inStream.pipe(hash);
hash.on('data', function(digest) {
var hexDigest = digest.toString('hex');
assert.strictEqual(hexDigest, o.md5, o.path + " md5 mismatch");
cb();
});
});
});
pend.wait(cb);
}
self.s3.putObject(s3Params, function(err, data) {
pendCb();
if (errorOccurred) return;
if (err) {
errorOccurred = true;
cb(err);
return;
}
pend.wait(function() {
if (!compareETag(data.ETag, localFileStat.md5sum)) {
errorOccurred = true;
cb(new Error("ETag does not match MD5 checksum"));
return;
}
cb(null, data);
});
});
extractRequires(source, function(err, requireList) {
if (err) return cb(err);
requireList.forEach(function(requireItem) {
pend.go(function(cb) {
requireResolve(requireItem, path.dirname(canonicalSourcePath), function(err, canonicalDepPath) {
if (err) return cb(err);
deps[canonicalSourcePath][canonicalDepPath] = true;
depMap[canonicalSourcePath][requireItem] = canonicalDepPath;
sourceQueue.push(canonicalDepPath);
cb();
});
});
});
pend.wait(function(err) {
if (err) return cb(err);
collectDependencies(cb);
});
});
function doStatAndMd5Sum() {
var md5sum;
var pend = new Pend();
pend.go(doStat);
pend.go(doMd5Sum);
pend.wait(function(err) {
if (err) {
uploader.emit('error', err);
return;
}
localFileStat.md5sum = md5sum;
startPuttingObject();
});
function doStat(cb) {
fs.stat(localFile, function(err, stat) {
if (!err) {
localFileStat = stat;
uploader.progressTotal = stat.size;
}
cb(err);
});
}
function doMd5Sum(cb) {
var inStream = fs.createReadStream(localFile);
var counter = new StreamCounter();
inStream.on('error', function(err) {
cb(err);
});
uploader.emit('stream', inStream);
var hash = crypto.createHash('md5');
hash.on('data', function(digest) {
md5sum = digest;
cb();
});
counter.on('progress', function() {
uploader.progressMd5Amount = counter.bytes;
uploader.emit('progress');
});
inStream.pipe(hash);
inStream.pipe(counter);
}
}