test('basic use with binary data in Buffers', function (done) {
var CHUNK_SIZE = 64 * 1024; // 64KB
var DATA_LENGTH = 2 * 1024 * 1024 + 25; // 2,025 KB
var shasum = crypto.createHash('sha1');
var resultDigest;
var bytesToGenerate = DATA_LENGTH;
var stream = passStream();
stream
.pipe(redisWStream(client, KEY))
.on('finish', function () {
client.get(new Buffer(KEY), function (err, data) { // use Buffer key so returns Buffer data
if (err) return done(err);
var dataDigest = crypto.createHash('sha1').update(data).digest('base64');
t.equal(resultDigest, dataDigest);
done();
});
});
function gen() {
var size = (bytesToGenerate > CHUNK_SIZE) ? CHUNK_SIZE : bytesToGenerate;
var buff = crypto.randomBytes(size);
shasum.update(buff);
stream.write(buff);
bytesToGenerate -= size;
if (!bytesToGenerate) {
stream.end();
resultDigest = shasum.digest('base64');
return;
}
immediate(gen); // use next tick so doesnt blow stack
}
immediate(gen); // kick it off
});
test('options.clientMulti provided so rename added to it, user must exec when ready', function (done) {
var stream = passStream();
var clientMulti = client.multi(); // my batch
stream
.pipe(redisWStream(client, KEY, { clientMulti: clientMulti }))
.on('finish', function () {
// exec not called on clientMulti so won't exist yet
client.get(KEY, function (err, data) {
if (err) return done(err);
t.isNull(data);
clientMulti.exec(function (err) {
if (err) return done(err);
client.get(KEY, function (err, data) {
if (err) return done(err);
t.deepEqual(data, 'abcdefghi');
done();
});
});
});
});
immediate(function () {
stream.write('abc');
stream.write('def');
stream.end('ghi');
});
});
test('accum(listenerFn) with various types of data, results with concatenated raw array to listenerFn before end', function (done) {
var DATA = [
1,
true,
false,
'hello',
[2, 3],
{ a: 1, b: 2 },
new Buffer('buff'),
'finished'
];
var stream = passStream(null, null, { objectMode: true });
stream
.pipe(accum({ objectMode: true }, function (alldata) {
t.ok(Array.isArray(alldata));
t.deepEqual(alldata, DATA);
done();
}));
process.nextTick(function () {
DATA.forEach(function (x) {
stream.write(x);
});
stream.end();
});
});
test('options.tempKeySuffix is provided so use `saveKey + tempKeySuffix` for tempKey ', function (done) {
var stream = passStream();
var clientMulti = client.multi(); // my batch
var myTempKeySuffix = '.myUniqueTempSuffix';
stream
.pipe(redisWStream(client, KEY, { clientMulti: clientMulti, tempKeySuffix: myTempKeySuffix }))
.on('finish', function () {
// exec not called on clientMulti so saveKey won't exist yet, but saveKey+tempKeySuffix will
client.get(KEY + myTempKeySuffix, function (err, data) {
if (err) return done(err);
t.deepEqual(data, 'abcdefghi');
clientMulti.exec(function (err) {
if (err) return done(err);
client.get(KEY, function (err, data) {
if (err) return done(err);
t.deepEqual(data, 'abcdefghi');
client.get(KEY + myTempKeySuffix, function (err, data) {
if (err) return done(err);
t.isNull(data);
done();
});
});
});
});
});
immediate(function () {
stream.write('abc');
stream.write('def');
stream.end('ghi');
});
});
storage.read(guid, (err, data) => {
if (err) {
return res.status(500).json({
statusCode: 500,
error: 'Internal Server Error',
message: 'We encountered an unexpected error. Please try again later.'
})
}
var decryptStream = passStream()
if (file.metadata.secure) {
const { algorithm = null, passphrase: key = null } = request.params
decryptStream = new cryptostream.DecryptStream({ algorithm, key })
}
// set headers
res.set({
'Cache-Control': 'public, must-revalidate, proxy-revalidate',
'Content-Disposition': utils.generateContentDisposition(file),
'Content-Length': file.length,
'Content-Type': (file.contentType || 'application/octet-stream'),
'ETag': file.md5 || file.hash,
'Last-Modified': file.uploadDate.toUTCString(),
})
data.pipe(res)
})
/**
* returns gzipped data read stream
*/
function getDataStream(key) {
var ps = passStream();
getContentIdForKey(key, function (err, cid) {
if (err) return ps.emit('error', err);
redisRStream(rc, cns(cid))
.pipe(ps);
});
return ps;
}
test('accum(listenerFn) with string data, results with concatenated string to listenerFn before end', function (done) {
var stream = passStream();
stream
.pipe(accum(function (alldata) {
t.equal(alldata, 'abcdefghi');
done();
}));
process.nextTick(function () {
stream.write('abc');
stream.write('def');
stream.end('ghi');
});
});
sequenceIntervalToFileInterval(start, stop, sequenceFilePath, function (fileStartPos, fileStopPos) {
res.setHeader("content-type", "text/plain");
res.setHeader("Access-Control-Allow-Origin", "*");
res.setHeader("Access-Control-Allow-Headers", "X-Requested-With");
promisePipe(
fs.createReadStream(sequenceFilePath, {start: fileStartPos, end: fileStopPos}),
passStream(newLineFilter),
res
).then(function () {
console.log('Succesfully streamed sequence '+ start +'-'+ stop +' on chromosome '+ req.params.chromosomeId +'.');
});
});
test('accum(listenerFn) with number data, results with concatenated raw array to listenerFn before end', function (done) {
var stream = passStream(null, null, { objectMode: true });
stream
.pipe(accum({ objectMode: true }, function (alldata) {
t.ok(Array.isArray(alldata));
t.deepEqual(alldata, [1, 2, 3]);
done();
}));
process.nextTick(function () {
stream.write(1);
stream.write(2);
stream.end(3);
});
});
/**
* save data into redis, can pass data as string or stream.
* @param key string host:urlpath
* @param data string or stream
* @param type string content type - saved as meta data `type` - required
* @param meta optional object with additional meta data (`type`, `len` and `digest` will be overriden with actual values)
* @param cb function callback(err, digest, gzipLength, cid)
*/
function set(key, data, type, meta, cb) {
if (typeof cb === 'undefined' && typeof meta === 'function') { // meta not provided
cb = meta;
meta = null;
}
if (!data.on) { // is not a stream, but a string, so create one
var dataString = data;
data = passStream();
data.pause();
data.end(dataString);
process.nextTick(function () { data.resume(); });
}
setStream(key, data, type, meta, cb);
}
/**
* digest, gzip, and store
*/
function setStream(key, inStream, type, meta, cb) {
meta = meta || {};
var pstream = passStream(); // pausable stream
pstream.pause();
inStream
.pipe(pstream);
// create clone so we are not modifying original
var metaObj = Object.keys(meta).reduce(function (accum, k) { accum[k] = meta[k]; return accum; }, {});
getNextContentId(function (err, contentId) {
if (err) return cb(err);
var digest;
function digestFn(resultDigest, len) {
digest = resultDigest;
}
var length;
function lengthFn(len) {
length = len;
}
pstream
.pipe(digestStream('sha1', 'base64', digestFn))
.pipe(zlib.createGzip())
.pipe(lengthStream(lengthFn))
.pipe(redisWStream(rc, cns(contentId)))
.on('error', function (err) { cb(err); })
.on('end', function () {
metaObj[TYPE] = type;
metaObj[LEN] = length.toString();
metaObj[DIGEST] = digest;
metaObj[MOD_TIME] = metaObj[MOD_TIME] || (new Date()).toISOString();
metaObj['Content-Encoding'] = 'gzip';
rc.multi()
.hmset(mns(contentId), metaObj)
.lpush(urlns(key), contentId)
.exec(function (err, result) {
if (err) return cb(err);
purgeVersions(key, content.currentConfig.revisions, function (err, result) {
if (err) console.error('Error purging versions for key: %s, err:', key, err); // log and cont
cb(null, digest, length, contentId);
});
});
});
pstream.resume();
});
}
function CryptoStream(algorithm, encoding, callback){
var hash = crypto.createHash(algorithm);
function write(data, encoding, fn){
hash.update(data);
this.push(data);
return fn();
}
function end(fn){
callback(hash.digest(encoding));
return fn();
}
return passthrough_stream(write, end);
}
test('basic use with string, stream data is stored and finish is fired', function (done) {
var stream = passStream();
stream
.pipe(redisWStream(client, KEY))
.on('finish', function () {
client.get(KEY, function (err, data) {
if (err) return done(err);
t.deepEqual(data, 'abcdefghi');
done();
});
});
immediate(function () {
stream.write('abc');
stream.write('def');
stream.end('ghi');
});
});
auth.getUserCtx(req, function(err, ctx) {
if (err || !ctx || !ctx.name) {
self.emit('not-authorized');
return;
}
var dataBuffer = [];
var writeFn = function(data, encoding, cb) {
dataBuffer.push(data);
cb();
};
var endFn = function(cb) {
var self = this;
var options = { buffer: buffer };
var data = dataBuffer.join('');
var doc = parse(data);
if (!doc) {
// ignore non-json requests
proxy.web(req, res, options);
self.push(data);
return cb();
}
couchdbAudit.withNode(db, ctx.name).log([doc], function(err) {
if (err) {
return cb(err);
}
data = JSON.stringify(doc);
// audit might modify the doc (eg: generating an id)
// so we need to update the content-length header
req.headers['content-length'] = Buffer.byteLength(data);
proxy.web(req, res, options);
self.push(data);
return cb();
});
};
var ps = passStream(writeFn, endFn);
var buffer = req.pipe(ps);
buffer.on('error', function(e) {
self.emit('error', e);
});
buffer.destroy = function() {};
});
test('accum(listenerFn) with Buffer data, results with concatenated Buffer to listenerFn before end', function (done) {
var DATA = new Buffer('abcdefghi');
var stream = passStream();
stream
.pipe(accum(function (alldata) {
t.ok(Buffer.isBuffer(alldata));
t.equal(alldata.length, DATA.length);
var digest = crypto.createHash('sha1').update(alldata).digest('base64');
var expectedDigest = crypto.createHash('sha1').update(DATA).digest('base64');
t.equal(digest, expectedDigest);
done();
}));
process.nextTick(function () {
stream.write(new Buffer('abc'));
stream.write(new Buffer('def'));
stream.end(new Buffer('ghi'));
});
});
function lengthStream(options, lengthListener) {
if (arguments.length === 1) { // options not provided, shift
lengthListener = options;
options = {};
}
options = options || {};
if (typeof lengthListener !== 'function') throw new Error('lengthStream requires a lengthListener fn');
var length = 0;
function writeFn(data, encoding, cb) {
/*jshint validthis:true */
length += data.length;
this.push(data);
cb();
}
function endFn(cb) {
/*jshint validthis:true */
lengthListener(length); // call with resultant length
cb();
}
var stream = passStream(writeFn, endFn, options);
return stream;
}
test('basic use, results with length being provided to listener before end', function (done) {
var resultantLength;
function lengthListener(length) {
resultantLength = length;
}
var ls = lengthStream(lengthListener);
var stream = passStream();
var accumData = [];
stream
.pipe(ls)
.on('error', function (err) { done(err); })
.on('data', function (data) { accumData.push(data); })
.on('end', function () {
t.deepEqual(Buffer.concat(accumData).toString(), 'abcdefghi');
t.equal(resultantLength, 9);
done();
});
process.nextTick(function () {
stream.write('abc');
stream.write('def');
stream.end('ghi');
});
});
detectMime(readStream, (err, mime, dataStream) => {
if (err) {
return cb(err)
}
// set default mime value if libmagic fails
if (mime === null) {
mime = { type: 'application/octet-stream' }
}
// perform additional lookup when mime type is text/plain
mime.type = mime.type === 'text/plain'
? mimeType.lookup(file.filename) || 'text/plain'
: mime.type
// detect if file is blacklisted
const { blacklist } = this._storageConfig
if (blacklist.fileExtensions.includes(utils.getFileExtension(file.filename)) || blacklist.mimeTypes.includes(mime.type)) {
return cb('This file contains a file extension or mime type that is blacklisted.')
}
// image post-processing optimizations
let imageProcessingStream = passStream()
if (this._storageConfig.imageOptimization) {
switch (mime.type) {
case 'image/jpeg':
imageProcessingStream = new JpegTran(['-copy', 'all', '-optimize', '-progressive'])
break
case 'image/gif':
imageProcessingStream = new Gifsicle(['-w', '-O3'])
break
case 'image/png':
imageProcessingStream = new OptiPng()
break
}
}
// file encryption
let encryptStream = passStream()
if (file.metadata.secure && file.crypto.algorithm && file.crypto.passphrase) {
if (crypto.getCiphers().indexOf(file.crypto.algorithm) === -1) {
const algorithms = crypto.getCiphers().toString().toUpperCase()
return cb(`You must use one of the following algorithms: ${algorithms}.`)
}
const { algorithm, passphrase: key } = file.crypto
encryptStream = new cryptostream.EncryptStream({ algorithm, key })
}
// store file
bcrypt.hash(file.deleteHash, 8, (err, hash) => {
file.content_type = mime.type
file.metadata.deleteHash = hash
this._gridfs.createWriteStream(file, (err, writeStream) => {
writeStream.on('error', (err) => {
return cb(err)
})
writeStream.on('close', (res) => {
return cb(null, res)
})
dataStream.pipe(imageProcessingStream).pipe(encryptStream).pipe(writeStream)
})
})
})
var http = require('http'),
cicada = require('cicada'),
ws = require('websocket.io'),
stream = require('stream'),
moment = require('moment'),
passStream = require('pass-stream');
/**
* Setup output stream
*/
var output = passStream(),
buffer = [];
output.on('data', function (data) {
buffer.push(''+data);
});
output.log = function () {
var args = [].slice.call(arguments);
output.write(args.join(' ') + '\n');
};
output.pipe(process.stdout);
/**
* Setup CI
*/
var ci = cicada('/tmp/nanoci');
ci.on('commit', function (commit) {
// Run npm install