walker.on('file', function(file, stat, linkPath) {
var usePath = linkPath || file;
if (ignoreFile(usePath)) return;
var relName = '/' + path.relative(dir, usePath).replace('\\', '/');
var compressedSink = new StreamSink();
var uncompressedSink = new StreamSink();
var hashSink = new StreamSink();
var inStream = fs.createReadStream(file);
var cacheObj;
cache[relName] = cacheObj = {
sink: null,
mime: mime.lookup(relName),
mtime: stat.mtime,
hash: null,
compressed: null,
};
var fileDone = pend.hold();
var thisPend = new Pend();
var gzipPendCb = thisPend.hold();
var hashPendCb = thisPend.hold();
var uncompressedPendCb = thisPend.hold();
inStream.on('error', function(err) {
if (err.code === 'EISDIR') {
delete cache[relName];
gzipPendCb();
uncompressedPendCb();
hashPendCb();
} else {
walker.stop();
gzipPendCb(err);
uncompressedPendCb(err);
hashPendCb(err);
}
});
inStream.pipe(zlib.createGzip()).pipe(compressedSink);
compressedSink.on('finish', gzipPendCb);
inStream.pipe(uncompressedSink);
uncompressedSink.on('finish', uncompressedPendCb);
inStream.pipe(crypto.createHash('sha1')).pipe(hashSink);
hashSink.on('finish', function() {
cacheObj.hash = hashSink.toString('base64');
hashPendCb();
});
thisPend.wait(function(err) {
if (err) return fileDone(err);
var compressionRatio = compressedSink.length / uncompressedSink.length;
if (compressionRatio >= 0.95) {
// 95% of original size or worse. discard compressed sink
cacheObj.sink = uncompressedSink;
cacheObj.compressed = false;
} else {
// better than 95% of original size. discard uncompressed sink
cacheObj.sink = compressedSink;
cacheObj.compressed = true;
}
fileDone();
});
});
request.on('httpHeaders', function(statusCode, headers, resp) {
if (statusCode >= 300) {
handleError(new Error("http status code " + statusCode));
return;
}
var contentLength = parseInt(headers['content-length'], 10);
downloader.progressTotal = contentLength;
downloader.progressAmount = 0;
downloader.emit('progress');
downloader.emit('httpHeaders', statusCode, headers, resp);
var eTag = cleanETag(headers.etag);
var eTagCount = getETagCount(eTag);
var outStream = new StreamSink();
var multipartETag = new MultipartETag({size: contentLength, count: eTagCount});
var httpStream = resp.httpResponse.createUnbufferedStream();
httpStream.on('error', handleError);
outStream.on('error', handleError);
hashCheckPend.go(function(cb) {
multipartETag.on('end', function() {
if (multipartETag.bytes !== contentLength) {
handleError(new Error("Downloaded size does not match Content-Length"));
return;
}
if (eTagCount === 1 && !multipartETag.anyMatch(eTag)) {
handleError(new Error("ETag does not match MD5 checksum"));
return;
}
cb();
});
});
multipartETag.on('progress', function() {
downloader.progressAmount = multipartETag.bytes;
downloader.emit('progress');
});
outStream.on('finish', function() {
if (errorOccurred) return;
hashCheckPend.wait(function() {
cb(null, outStream.toBuffer());
});
});
httpStream.pipe(multipartETag);
httpStream.pipe(outStream);
multipartETag.resume();
});
it( "createReadStream", function( done ) {
var str = "I never conquered rarely came, 16 just held such better days";
var buf = new Buffer( str );
var slicer = fdSlicer.createFromBuffer( buf );
var inStream = slicer.createReadStream();
var sink = new StreamSink();
inStream.pipe( sink );
sink.on( 'finish', function() {
assert.strictEqual( sink.toString(), str );
inStream.destroy();
done();
} );
} );
it("downloadStream", function(done) {
var client = createClient();
var downloadStream = client.downloadStream({Key: remoteFile, Bucket: s3Bucket});
downloadStream.on('error', done);
var gotHttpHeaders = false;
downloadStream.on('httpHeaders', function(statusCode, headers, resp) {
var contentType = headers['content-type'];
assert.strictEqual(contentType, "image/png");
gotHttpHeaders = true;
});
var sink = new StreamSink();
downloadStream.pipe(sink);
sink.on('finish', function() {
var md5sum = crypto.createHash('md5');
md5sum.update(sink.toBuffer());
assert.strictEqual(md5sum.digest('hex'), hexdigest)
assert.ok(gotHttpHeaders);
done();
});
});