walker.on('file', function(file, stat, linkPath) { var usePath = linkPath || file; if (ignoreFile(usePath)) return; var relName = '/' + path.relative(dir, usePath).replace('\\', '/'); var compressedSink = new StreamSink(); var uncompressedSink = new StreamSink(); var hashSink = new StreamSink(); var inStream = fs.createReadStream(file); var cacheObj; cache[relName] = cacheObj = { sink: null, mime: mime.lookup(relName), mtime: stat.mtime, hash: null, compressed: null, }; var fileDone = pend.hold(); var thisPend = new Pend(); var gzipPendCb = thisPend.hold(); var hashPendCb = thisPend.hold(); var uncompressedPendCb = thisPend.hold(); inStream.on('error', function(err) { if (err.code === 'EISDIR') { delete cache[relName]; gzipPendCb(); uncompressedPendCb(); hashPendCb(); } else { walker.stop(); gzipPendCb(err); uncompressedPendCb(err); hashPendCb(err); } }); inStream.pipe(zlib.createGzip()).pipe(compressedSink); compressedSink.on('finish', gzipPendCb); inStream.pipe(uncompressedSink); uncompressedSink.on('finish', uncompressedPendCb); inStream.pipe(crypto.createHash('sha1')).pipe(hashSink); hashSink.on('finish', function() { cacheObj.hash = hashSink.toString('base64'); hashPendCb(); }); thisPend.wait(function(err) { if (err) return fileDone(err); var compressionRatio = compressedSink.length / uncompressedSink.length; if (compressionRatio >= 0.95) { // 95% of original size or worse. discard compressed sink cacheObj.sink = uncompressedSink; cacheObj.compressed = false; } else { // better than 95% of original size. discard uncompressed sink cacheObj.sink = compressedSink; cacheObj.compressed = true; } fileDone(); }); });
request.on('httpHeaders', function(statusCode, headers, resp) { if (statusCode >= 300) { handleError(new Error("http status code " + statusCode)); return; } var contentLength = parseInt(headers['content-length'], 10); downloader.progressTotal = contentLength; downloader.progressAmount = 0; downloader.emit('progress'); downloader.emit('httpHeaders', statusCode, headers, resp); var eTag = cleanETag(headers.etag); var eTagCount = getETagCount(eTag); var outStream = new StreamSink(); var multipartETag = new MultipartETag({size: contentLength, count: eTagCount}); var httpStream = resp.httpResponse.createUnbufferedStream(); httpStream.on('error', handleError); outStream.on('error', handleError); hashCheckPend.go(function(cb) { multipartETag.on('end', function() { if (multipartETag.bytes !== contentLength) { handleError(new Error("Downloaded size does not match Content-Length")); return; } if (eTagCount === 1 && !multipartETag.anyMatch(eTag)) { handleError(new Error("ETag does not match MD5 checksum")); return; } cb(); }); }); multipartETag.on('progress', function() { downloader.progressAmount = multipartETag.bytes; downloader.emit('progress'); }); outStream.on('finish', function() { if (errorOccurred) return; hashCheckPend.wait(function() { cb(null, outStream.toBuffer()); }); }); httpStream.pipe(multipartETag); httpStream.pipe(outStream); multipartETag.resume(); });
it( "createReadStream", function( done ) { var str = "I never conquered rarely came, 16 just held such better days"; var buf = new Buffer( str ); var slicer = fdSlicer.createFromBuffer( buf ); var inStream = slicer.createReadStream(); var sink = new StreamSink(); inStream.pipe( sink ); sink.on( 'finish', function() { assert.strictEqual( sink.toString(), str ); inStream.destroy(); done(); } ); } );
it("downloadStream", function(done) { var client = createClient(); var downloadStream = client.downloadStream({Key: remoteFile, Bucket: s3Bucket}); downloadStream.on('error', done); var gotHttpHeaders = false; downloadStream.on('httpHeaders', function(statusCode, headers, resp) { var contentType = headers['content-type']; assert.strictEqual(contentType, "image/png"); gotHttpHeaders = true; }); var sink = new StreamSink(); downloadStream.pipe(sink); sink.on('finish', function() { var md5sum = crypto.createHash('md5'); md5sum.update(sink.toBuffer()); assert.strictEqual(md5sum.digest('hex'), hexdigest) assert.ok(gotHttpHeaders); done(); }); });