module.exports = function (res) {
// TODO: use Array#includes when targeting Node.js 6
if (['gzip', 'deflate'].indexOf(res.headers['content-encoding']) === -1) {
return res;
}
var unzip = zlib.createUnzip();
var stream = new PassThrough();
stream.httpVersion = res.httpVersion;
stream.headers = res.headers;
stream.rawHeaders = res.rawHeaders;
stream.trailers = res.trailers;
stream.rawTrailers = res.rawTrailers;
stream.setTimeout = res.setTimeout.bind(res);
stream.statusCode = res.statusCode;
stream.statusMessage = res.statusMessage;
stream.socket = res.socket;
unzip.on('error', function (err) {
if (err.code === 'Z_BUF_ERROR') {
stream.end();
return;
}
stream.emit('error', err);
});
res.pipe(unzip).pipe(stream);
return stream;
};
var handleData = function handleData() {
var firstChunk = sourceStream.read();
// 'readable' is also called when a stream ends with no data
// this is how you tell when that's the case:
if (firstChunk === null) {
placeHolder.end();
return;
}
var decompressStream;
if (data.headers['content-encoding'] == 'deflate') {
// https://github.com/nfriedly/node-unblocker/issues/12
// inflateRaw seems to work here wheras inflate and unzip do not.
// todo: validate this against other sites - if some require raw and others require non-raw, then maybe just rewrite the accept-encoding header to gzip only
decompressStream = zlib.createInflateRaw();
} else {
decompressStream = zlib.createUnzip();
}
decompressStream.write(firstChunk);
// continue piping through the placeholder since the next stream in the chain is listening to it, not our new decompressStream
sourceStream.pipe(decompressStream).pipe(placeHolder);
};
this.end().req.on('response', function(res){
if (/^(deflate|gzip)$/.test(res.headers['content-encoding'])) {
res.pipe(zlib.createUnzip()).pipe(stream, options);
} else {
res.pipe(stream, options);
}
});
function unzipResponse(response) {
if (['gzip', 'deflate'].indexOf(response.get('Content-Encoding')) > -1) {
var unzip_1 = zlib_1.createUnzip();
response.body.pipe(unzip_1);
response.body = unzip_1;
}
}
function getNextFile () {
// Create request URL in the format xxx/xxx/xxx
var url = '';
for (i = 0; i < (stateStr.length / 3); i++) {
url += stateStr[i * 3] + stateStr[i * 3 + 1] + stateStr[i * 3 + 2] + '/';
}
var nodata = true;
var ss = request.get(that.baseURL + url.split('').reverse().join('') +
that.filetype)
.pipe(zlib.createUnzip())
.on('data', function (data) {
nodata = (data.length === 0) && nodata;
})
.on('end', function () {
if (nodata) {
next();
ss.end();
}
})
.pipe(xmlParser)
.on('data', function (data) {
that.push(data);
})
.on('end', next)
.on('error', function (err) {
throw new Error('from parser', err);
});
}
var unzipResponse = function (res) {
if (['gzip', 'deflate'].indexOf(res.get('Content-Encoding')) !== -1) {
var unzip = zlib.createUnzip()
res.body.pipe(unzip)
res.body = unzip
}
}
}, function(response) {
var decompressor = zlib.createUnzip(),
parser = new tar.Extract({ 'path': destPath });
parser.on('end', callback);
response.pipe(decompressor).pipe(parser);
});
(function appClosure() {
'use strict';
var fs = require('fs');
var zlib = require('zlib');
var count = 0;
// Callback Funktion
var echo = function(whatever) {
count += 1;
console.log(count);
};
// Datei aus einer GZIP Datei gelsen und als Stream verarbeiten
var readFile = fs.createReadStream('random_large_file.txt.gz').pipe(zlib.createUnzip());
// Was tun, wenn daten aus dem Stream ankommen?
readFile.on("data", echo);
// Was tun, wenn keine Daten zum lesen mehr da sind (EOF)?
readFile.on("end", function() { clearInterval(otherOperation) });
// Noch etwas anderes machen
var otherOperation = setInterval(function() {
console.log("\nANOTHER OPERATION\n");
}, 10);
})();
exports.run = function (opts) {
if (opts.users) {
exports.users(opts);
return;
}
if (typeof opts.gzipped === 'undefined')
opts.gzipped = data_file.path.match(/\.gz$/i);
opts.keypath = opts.keypath || [true];
var parser = JSONStream.parse(opts.keypath);
if (opts.fields)
exports.out(opts.fields);
parser.on('data', opts.on_data);
parser.on('error', function (error) {
// TODO: handle error
console.log('Error:', error);
});
parser.on('end', function () {
if (opts.on_end) opts.on_end();
out_file.end();
});
if (opts.gzipped)
data_file.pipe(zlib.createUnzip()).pipe(parser);
else
data_file.pipe(parser);
};
.on('response', function(response) {
if (response.statusCode === 200) {
if (response.headers['content-type'] === 'application/octet-stream' ||
response.headers['content-type'] === 'application/x-bittorrent') {
let filename = hash + '.torrent';
let dest;
switch (response.headers['content-encoding']) {
case 'gzip':
case 'deflate':
dest = response.pipe(zlib.createUnzip()).on('error', function(err) {
cb(err);
})
.pipe(fs.createWriteStream(filename));
break;
default:
dest = response.pipe(fs.createWriteStream(filename));
break;
}
dest.on('finish', function() {
cb(null, filename);
})
.on('error', function(err) {
cb(err);
});
} else {
cb('Invalid content type: ' + response.headers['content-type']);
}
} else {
cb('Error response: ' + response.statusCode);
}
});
Tuiter.prototype.post = function(url, params, callback){
var unzip = zlib.createUnzip();
var req = this.oa.post(
url
, this.oa.access_token_key
, this.oa.access_token_secret
, params);
req.on('response', function(res){
res.pipe(unzip);
var data = '';
unzip.on('data', function(chunk){
data += chunk;
});
unzip.on('end', function(){
try{
var json_data = JSON.parse(data);
callback(null, json_data);
} catch(e){
callback(e, null);
}
});
unzip.on('error', function(err){
callback(err, null);
});
});
req.end();
req.on('error', function(){
callback(null, error);
});
};
Tuiter.prototype.get = function(url, params, callback){
var unzip = zlib.createUnzip();
url += (Object.keys(params).length) ? '?' : '';
var req = this.oa.get(
url + qs.stringify(params)
, params.access_token_key || this.oa.access_token_key
, params.access_token_secret || this.oa.access_token_secret
);
req.on('response', function(res){
res.pipe(unzip);
var data = '';
unzip.on('data', function(chunk){
data += chunk;
});
unzip.on('end', function(){
try{
var json_data = JSON.parse(data);
callback(null, json_data);
} catch(e){
callback(e, null);
}
});
unzip.on('error', function(err){
callback(err, null);
});
});
req.end();
req.on('error', function(){
callback(null, error);
});
};
.on('response', function(res) {
if (supportGzip && ['gzip', 'deflate'].indexOf(res.headers['content-encoding']) > -1) {
var gunzip = zlib.createUnzip();
gunzip.request = res.request;
gunzip.statusCode = res.statusCode;
gunzip.headers = res.headers;
if (!options.asBuffer) {
gunzip.setEncoding("binary");
}
req.emit('response', gunzip);
res.pipe(gunzip);
} else {
if (!options.asBuffer) {
res.setEncoding("binary");
}
req.emit('response', res);
}
});
proxy.on('proxyRes', function(proxyRes, req, res) {
log('received response from upstream', req.proxy.proxyReq.method, req.cacheKey, proxyRes.statusCode, proxyRes.statusMessage);
log('storing headers');
var headersData = _.pick(proxyRes, ['statusCode', 'statusMessage', 'headers']);
cache.set('headers:'+req.cacheKey, headersData);
var dataStream = proxyRes.headers['content-encoding'] ? proxyRes.pipe(zlib.createUnzip()) : proxyRes;
var data = [];
dataStream.on('data', function(chunk) {
data.push(chunk);
});
dataStream.on('end', function() {
data = Buffer.concat(data).toString('base64');
if (proxyRes.statusCode === 200) {
log('storing %s response for %s', proxyRes.statusCode, req.cacheKey);
cache.set(req.cacheKey, data);
} else {
log('not storing %s response for %s ', proxyRes.statusCode, req.cacheKey);
}
});
});
this.req.once('response', res => {
// redirect
const redirect = isRedirect(res.statusCode);
if (redirect && this._redirects++ != this._maxRedirects) {
return this._redirect(res)._pipeContinue(stream, options);
}
this.res = res;
this._emitResponse();
if (this._aborted) return;
if (this._shouldUnzip(res)) {
const unzipObj = zlib.createUnzip();
unzipObj.on('error', err => {
if (err && err.code === 'Z_BUF_ERROR') { // unexpected end of file is ignored by browsers and curl
stream.emit('end');
return;
}
stream.emit('error', err);
});
res.pipe(unzipObj).pipe(stream, options);
} else {
res.pipe(stream, options);
}
res.once('end', () => {
this.emit('end');
});
});
utils.getJsFromXml = function (res, callback) {
var encoding = res.headers['content-encoding'],
stream = res,
xml = '';
if(encoding === 'gzip' || encoding === 'deflate') {
// Set the stream to zlib.Unzip object to handle both encodings.
stream = zlib.createUnzip();
// Pipe the compressed response from NextBus to the unzip stream.
res.pipe(stream);
}
stream.on('data', function(d) { if(d) xml += d.toString(); });
stream.on('end', function() {
try {
js4xml(xml, function(err, js) {
if(err) callback(err, {errtype: 'parse'});
else if(!js || typeof js.body === 'undefined') {
console.error(JSON.stringify(js));
callback({message: 'No "body" element in NextBus XML document'}, {errtype: 'parse'});
} else callback(null, js);
});
} catch (err) {
callback(err, {errtype: 'parse'});
}
});
stream.on('error', function(e) {
console.error(e.stack);
callback(e, {errtype: 'stream'});
});
};
var req = transport.request(options, function handleResponse(res) {
if (aborted) return;
// Response has been received so kill timer that handles request timeout
clearTimeout(timer);
timer = null;
// uncompress the response body transparently if required
var stream = res;
switch (res.headers['content-encoding']) {
/*eslint default-case:0*/
case 'gzip':
case 'compress':
case 'deflate':
// add the unzipper to the body stream processing pipeline
stream = stream.pipe(zlib.createUnzip());
// remove the content-encoding in order to not confuse downstream operations
delete res.headers['content-encoding'];
break;
}
var response = {
status: res.statusCode,
statusText: res.statusMessage,
headers: res.headers,
config: config,
request: req
};
if (config.responseType === 'stream') {
response.data = stream;
settle(resolve, reject, response);
} else {
var responseBuffer = [];
stream.on('data', function handleStreamData(chunk) {
responseBuffer.push(chunk);
// make sure the content length is not over the maxContentLength if specified
if (config.maxContentLength > -1 && Buffer.concat(responseBuffer).length > config.maxContentLength) {
reject(createError('maxContentLength size of ' + config.maxContentLength + ' exceeded', config));
}
});
stream.on('error', function handleStreamError(err) {
if (aborted) return;
reject(enhanceError(err, config));
});
stream.on('end', function handleStreamEnd() {
var responseData = Buffer.concat(responseBuffer);
if (config.responseType !== 'arraybuffer') {
responseData = responseData.toString('utf8');
}
response.data = responseData;
settle(resolve, reject, response);
});
}
});
request.once('response', (response) => {
if (socket)
socket.removeListener('error', handleError);
let stream = response;
if (shouldUnzip(response)) {
stream = response.pipe(zlib.createUnzip({
flush: zlib.Z_SYNC_FLUSH,
finishFlush: zlib.Z_SYNC_FLUSH,
}));
}
if (this.options.followRedirects !== false && [301, 302, 303, 307, 308].includes(response.statusCode)) {
resolve({
response,
redirect: URL.resolve(this.options.lastBuiltUrl, response.headers.location),
});
response.destroy();
} else {
const body = [];
stream.on('data', (chunk) => {
if (!this.push(chunk))
this.pause();
body.push(chunk);
});
stream.once('end', () => {
this.push(null);
const raw = Buffer.concat(body);
resolve({ response, raw, redirect: false });
});
}
});
/* "private bodyLogger " : {
━━━━━━━━━━━━━━━━━━━━━━━━━━*/
function bodyLogger(stream, type, callback) {
var data = [], length, unzipper;
var assembleBody = function() {
var buffer;
stream.body = new Buffer(stream.length);
for (var i = 0, len = data.length, offset = 0; i < len; i++) {
buffer = data[i];
buffer.copy(stream.body, offset);
offset += buffer.length;
}
return data = null;
};
callback || (callback = function() {
assembleBody();
stream.emit('body');
if (type === 'response') {
return trace.info("Captured " + stream.body.length + " bytes from " + stream.statusCode);
}
});
length = parseInt(stream.headers['content-length'], 10) || 0;
stream.body = new Buffer(parseInt(stream.headers['content-length'], 10));
stream.length = 0;
var unzipper = zlib.createUnzip();
unzipper.on('data', function(datum) {
data.push(datum);
return stream.length += datum.length;
});
unzipper.on('end', function() {
return callback();
});
unzipper.destroy = function() {
return trace.info(stream.headers);
};
switch (stream.headers['content-encoding']) {
case 'gzip':
trace.info("Unzipping");
stream.pipe(unzipper);
break;
case 'deflate':
trace.info("Deflating");
stream.pipe(unzipper);
break;
default:
stream.on('data', function(datum) {
data.push(datum);
return stream.length += datum.length;
});
stream.on('end', function() {
return callback();
});
break;
}
};
var req = transport.request(options, function (res) {
if (req.aborted) {
return;
}
// Uncompress the response body transparently if required.
var respStream = res;
var encodings = ['gzip', 'compress', 'deflate'];
if (encodings.indexOf(res.headers['content-encoding']) !== -1) {
// Add the unzipper to the body stream processing pipeline.
var zlib = require('zlib');
respStream = respStream.pipe(zlib.createUnzip());
// Remove the content-encoding in order to not confuse downstream operations.
delete res.headers['content-encoding'];
}
var response = {
status: res.statusCode,
headers: res.headers,
request: req,
data: undefined,
config: config,
};
var responseBuffer = [];
var boundary = getMultipartBoundary(res.headers);
if (boundary) {
var dicer = require('dicer');
var multipartParser = new dicer({ boundary: boundary });
multipartParser.on('part', function (part) {
var tempBuffers = [];
part.on('data', function (partData) {
tempBuffers.push(partData);
});
part.on('end', function () {
responseBuffer.push(Buffer.concat(tempBuffers));
});
});
multipartParser.on('finish', function () {
response.data = null;
response.multipart = responseBuffer;
finalizeRequest(resolve, reject, response);
});
respStream.pipe(multipartParser);
}
else {
respStream.on('data', function (chunk) {
responseBuffer.push(chunk);
});
respStream.on('error', function (err) {
if (req.aborted) {
return;
}
reject(enhanceError(err, config, null, req));
});
respStream.on('end', function () {
var responseData = Buffer.concat(responseBuffer).toString();
response.data = responseData;
finalizeRequest(resolve, reject, response);
});
}
});
this.once('response', function (res) {
// Forward the `response` event down the pipe-line
decoder._forwardFlowHttpResponse(res);
if (supportedEncodings.indexOf(res.headers['content-encoding']) !== -1) {
decoder._src.unpipe(decoder);
decoder._src.pipe(zlib.createUnzip()).pipe(decoder);
}
});
http.get(url, (response) => {
let output = fs.createWriteStream(out);
response.pipe(zlib.createUnzip()).pipe(output);
output.on('finish', cb)
});
return new NPromise( function ( fulfill, reject ) {
print.info( 'Unzipping uploads tarball...' );
var output = temp.createWriteStream( {suffix: '.tar'} );
output.on( 'finish', function () {
fulfill( output.path );
} );
output.on( 'error', reject );
fs.createReadStream( filePath ).pipe( zlib.createUnzip() ).pipe( output );
} );
// Look for latest region file and see if data for the requested data exists
function checkLocal(filelist, cb) {
if (filelist) {
var best = null;
for (var i = 0; i < filelist.length; i++) {
var next = filelist[i].split('_');
if (next[2] == regionID) {
if (best == null) {
best = filelist[i];
} else {
var bsplit = best.split('_');
if (parseInt(next[1]) > parseInt(bsplit[1])) {
best = filelist[i];
}
}
}
}
// Only continue if we found a candidate file. Otherwise drop through and look in the online archive.
if (best) {
var stream = fs.createReadStream(historyLocation + best);
var gunzip = zlib.createUnzip();
stream.pipe(gunzip);
var rl = readline.createInterface({
input: gunzip
});
var start = true
var found = false;
rl.on('line', function(line) {
if (found) return;
if (start) {
// First line is just a row count, skip
start = false;
} else {
// Check if line contains needed data
var data = line.split(',');
if (parseInt(data[7]) == dateTime) {
found = true;
var market = new ekmd.MarketHistory(line);
res.status(200).json(market);
}
}
});
rl.on('close', function() {
if (!found) {
// We never found a usable line, try the online archive
cb(null);
}
});
} else {
// Couldn't find a best file, try the online archive
cb(null);
}
} else {
// Couldn't get a file list, try the online archive
cb(null);
}
},
return new Promise((resolve, reject) => {
helper.downloadFileStream(downloadLink)
.pipe(zlib.createUnzip())
.pipe(tar.Extract({
path: stagingNpmDir+'/'+npmVersion.replace('v','')
, strip: 1
}))
.on('error', reject)
.on('end', resolve)
})
module.exports.unzip = function(filename, outputStream) {
var readableStream = fs.createReadStream(filename + EXT);
var unzip = zlib.createUnzip();
var deferred = q.defer();
readableStream.pipe(unzip).pipe(outputStream)
.on('end', deferred.resolve.bind(deferred))
.on('error', deferred.reject.bind(deferred));
return deferred.promise;
};
this.end().req.on('response', function(res){
if (isRedirect(res.statusCode)) {
// Don't pipe yet; wait for redirect.
return;
}
if (/^(deflate|gzip)$/.test(res.headers['content-encoding'])) {
res.pipe(zlib.createUnzip()).pipe(stream, options);
} else {
res.pipe(stream, options);
}
});
request.on('response', response => {
if (response.headers['content-encoding']) {
response = response
.pipe(zlib.createUnzip());
}
response
.pipe(fs.createWriteStream(destPath))
.on('finish', callback);
});
req.on('response', function (response) {
if ((response.headers['content-encoding']) &&
(response.headers['content-encoding'] != 'identity') &&
(response.headers['content-encoding'] != 'none'))
{
var unzip = zlib.createUnzip();
unzip.on('error', function (e) {
console.error(e, url);
});
req = req.pipe(unzip);
}
var response_headers = response.headers;
delete response_headers['content-encoding'];
delete response_headers['content-length'];
response_headers['connection'] = 'close';
res.writeHead(response.statusCode, response_headers);
if ((response.headers['content-type']) &&
(response.headers['content-type'].indexOf('text/html') == 0)) {
req.once('data', function (chunk) {
var domain = new URI(url).domain();
var stylesheet_str = elemHide.getSelectorsForDomain(domain, true).join(', ');
if (!stylesheet_str.length) {
return;
}
var head_buffer = new Buffer('<head>', 'ascii');
var head_offset = buffer_search(chunk, head_buffer);
if (head_offset != -1) {
console.warn('Modifying ' + url);
head_offset += head_buffer.length;
var start_to_head = chunk.slice(0, head_offset);
var head_to_end = chunk.slice(head_offset, chunk.length);
var stylesheet_buffer = new Buffer('<style>' + stylesheet_str +
' { display: none !important; }</style>', 'ascii');
res.write(Buffer.concat([start_to_head, stylesheet_buffer, head_to_end]));
var orig_write = res.write;
res.write = function () {
res.write = orig_write;
}
}
});
}
req = req.pipe(res);
});
function handleGZIPResponse (response) {
if (/^(deflate|gzip)$/.test(response.headers['content-encoding'])) {
var unzip = zlib.createUnzip()
var stream = new Stream()
var _on = response.on
var decoder
// Keeping node happy
stream.req = response.req
// Make sure we emit prior to processing
unzip.on('error', function (error) {
// Catch the parser error when there is no content
if (error.errno === zlib.Z_BUF_ERROR || error.errno === zlib.Z_DATA_ERROR) {
stream.emit('end')
return
}
stream.emit('error', error)
})
// Start the processing
response.pipe(unzip)
// Ensure encoding is captured
response.setEncoding = function (type) {
decoder = new StringDecoder(type)
}
// Capture decompression and decode with captured encoding
unzip.on('data', function (buffer) {
if (!decoder) return stream.emit('data', buffer)
var string = decoder.write(buffer)
if (string.length) stream.emit('data', string)
})
// Emit yoself
unzip.on('end', function () {
stream.emit('end')
})
response.on = function (type, next) {
if (type === 'data' || type === 'end') {
stream.on(type, next)
} else if (type === 'error') {
_on.call(response, type, next)
} else {
_on.call(response, type, next)
}
}
}
}