cache.refresh = function() {
var ws = fs.createWriteStream(keysFile);
s3.keys()
.pipe(es.split(','))
.pipe(es.join('\n'))
.pipe(ws);
}
var stream = through.obj(function (file, enc, callback) {
var checkAndInsert = function(buffer) {
return new Buffer(buffer.toString(enc).replace(/<\/body>/, function (w) {
return snippet + w;
}));
};
if(file.isNull()) {
//Passthrough if not file
this.push(file);
return callback();
}
if (file.contents instanceof Buffer) {
file.contents = checkAndInsert(file.contents);
this.push(file);
return callback();
} else {
file.contents = file.contents.pipe(es.split("\n")).pipe(through(function(line, enc, callback) {
line = checkAndInsert(line);
this.push(line);
callback();
})).pipe(es.join("\n"));
this.push(file);
return callback();
}
});
function (file) {
var buffered, ready = false
fs.createReadStream(file)
.on('close', function () {
ready = true; next()
})
.pipe(es.split(/(\n)/))
.pipe(es.mapSync(function (line) {
var m
if(m = /^(\s+).*/.exec(line)) {
var oldspace = m[1]
//check that the style matches what we are converting from?
//check it's all tabs, or the right number of spaces?
var newspace = oldspace.split(from).join(to)
return newspace + line.substring(oldspace.length)
} else
return line
}))
.pipe(es.join(function (err, _buf) {
buffered = _buf; next()
}))
function next() {
if(!buffered) return
if(!ready) return
fs.writeFile(
file = (opts.safe ? '/tmp/bikeshed/'+file : file),
buffered, function (err) {
if(err) throw err
console.log('wrote:' + file)
})
}
})
csv: function ( ) {
// sgvdata knows how to format sgv entries as text
res.set('Content-Type', 'text/plain');
var csvpipe = require('sgvdata/lib/text')({ format: ',', parse: /[\t,]/ });
es.pipeline(
output
, sgvdata.mapper(csvpipe.format)
, es.join('\n')
, es.writeArray(function (err, out) {
res.send(out.join(''));
})
);
},
es.map(function(entry, cb){
console.error("got data!" + entry.path);
var fileName = entry.path;
toJSON(entry).pipe(es.join(function(err, json){
if (entry.path === 'geo_mines_shp.zip'){
console.error('starting on zip file');
processMiningShapeFiles(new ZipFile(entry), cb);
}else{
console.error("Unknown entry:" + entry.path);
cb();
}
}));
});
exports ['fizz buzz replace'] = function (test) {
var split = es.split('1')
var replace = es.replace('7', 'seven')
es.connect(split,
replace,
// es.log('aeounh'),
es.join(function (err, string) {
it(string).equal(fizz7buzz)
test.done()
})
)
split.write(fizzbuzz)
split.end()
}
exports ['fizz buzz'] = function (test) {
var readThis = d.map(1, 100, function (i) {
return (
! (i % 3 || i % 5) ? "FB" :
!(i % 3) ? "F" :
!(i % 5) ? "B" :
''+i
)
}) //array of multiples of 3 < 100
var reader = es.readArray(readThis)
var join = es.join(function (err, string){
it(string).equal(fizzbuzz)
test.done()
})
reader.pipe(join)
}
module.exports = function(cb) {
var commands = {
'local.branch.current.name': ['rev-parse', '--abbrev-ref', 'HEAD'],
'local.branch.current.SHA': ['rev-parse', 'HEAD'],
'local.branch.current.shortSHA': ['rev-parse', '--short', 'HEAD'],
'local.branch.current.currentUser': ['config', '--global', 'user.name'],
'local.branch.current.lastCommitTime': ['log', '--format="%ai"', '-n1', 'HEAD'],
'local.branch.current.lastCommitMessage' : ['log', '--format="%B"', '-n1', 'HEAD'],
'local.branch.current.lastCommitAuthor': ['log', '--format="%aN"', '-n1', 'HEAD'],
'local.branch.current.tag': ['describe', '--abbrev=0', '--exact-match'],
'remote.origin.url': ['config', '--get-all', 'remote.origin.url']
};
var streams = [];
Object.keys(commands).forEach(function(cmd) {
var command = commands[cmd];
var git_cmd = "echo " + cmd + " && git " + command.join(" ");
var child = exec(git_cmd, function(err, stdout, stderr) {});
streams.push(child.stdout.pipe(es.wait()));
});
var stream = es.merge.apply(es, streams);
return stream
.pipe(es.map(function(data, cb) {
var lines = data.split(os.EOL);
lines[0] = lines[0].replace(' ', '');
lines[1] = lines[1].replace(os.EOL, '').replace('\n', '');
cb(null, lines[0] + ':' + lines[1]);
}))
.pipe(es.join(','))
.pipe(es.wait())
.pipe(es.map(function(data, cb) {
var obj = {};
var lines = data.split(',');
lines.forEach(function(line) {
var section = line.split(':');
var key = section.shift();
obj[key] = section.join(":");
});
cb(null, JSON.stringify(obj));
}))
.pipe(es.parse());
};
function build(config) {
config = config || {};
var buildStream = es.through(function write(block) {
var buildFn = block.target && config[block.target],
blockStream = new BlockStream(block);
if (buildFn) {
buildFn(blockStream);
} else {
blockStream.pipe(blockStream);
}
this.queue(blockStream.result);
});
return es.pipeline(
buildStream,
flatten(),
es.join('\n')
);
}
Stream.prototype.stream = function() {
var jStream, sStream, qStream, pStream;
// Create a stream to comma-deliminate all incoming data values:
jStream = eventStream.join( ',' );
// Create a sink stream to buffer all data values into memory:
sStream = eventStream.wait();
// Create a transform stream to calculate the quantiles:
qStream = mapStream()
.map( transform( this._quantiles ) )
.stream();
// Create a stream pipeline:
pStream = eventStream.pipeline(
jStream,
sStream,
qStream
);
// Return the pipeline:
return pStream;
}; // end METHOD stream()
module.exports = function(exportFilePath, outputFilePath, done) {
var s = fs.createReadStream(exportFilePath)
.pipe(es.replace('{"n":', '{'))
.pipe(es.replace('{"s":', '{'))
.pipe(es.replace('{"sS":[', '{['))
.pipe(es.replace(String.fromCharCode(3) + '{', ':')) // ETX
.pipe(es.replace('}' + String.fromCharCode(2), ',')) // STX
.pipe(es.replace('}' + String.fromCharCode(10), String.fromCharCode(10))) // EOL
.pipe(es.replace('["', '['))
.pipe(es.replace('"]', ']'))
.pipe(es.replace('}","{','},{'))
.pipe(es.replace('\\"', '"'))
.pipe(es.split("\n"))
.pipe(es.mapSync(function(data) {
if(data.length > 0) {
return '{' + data + '}';
}
}))
.pipe(es.mapSync(function(data) {
var changed = data.replace(/{(?!")([^{:,]+):/g, '{"$1":');
return changed;
}))
.pipe(es.mapSync(function(data) {
var changed = data.replace(/,(?!")([^{:,]+):/g, ',"$1":');
return changed;
}))
.pipe(es.join(","))
.pipe(es.wait())
.pipe(es.mapSync(function(data) {
return '[' + data + ']';
}))
.pipe(fs.createWriteStream(outputFilePath));
return s.on('close', done);
};
exports.html = function(req, res){
var keys;
var headerParser = new Transform({objectMode: true});
headerParser.header = null;
headerParser._transform = function (data, encoding, done) {
if (!this.header) {
keys = Object.keys(data);
if (keys[0] === 'Datum') {
this.header = keys;
}
}
this.push(data);
done();
}
//Data format:
//Datum,Verenigde Staten,Alabama,Alaska,Arizona,Arkansas,Californië,Colorado,Connecticut,Delaware,Washington D.C.,Florida,Georgia,Hawaï,Idaho,Illinois,Indiana,Iowa,Kansas,Kentucky,Louisiana,Maine,Maryland,Massachusetts,Michigan,Minnesota,Mississippi,Missouri,Montana,Nebraska,Nevada,New Hampshire,New Jersey,New Mexico,New York,North Carolina,North Dakota,Ohio,Oklahoma,Oregon,Pennsylvania,Rhode Island,South Carolina,South Dakota,Tennessee,Texas,Utah,Vermont,Virginia,Washington,West Virginia,Wisconsin,Wyoming,"HHS Region 1 (CT, ME, MA, NH, RI, VT)","HHS Region 2 (NJ, NY)","HHS Region 3 (DE, DC, MD, PA, VA, WV)","HHS Region 4 (AL, FL, GA, KY, MS, NC, SC, TN)","HHS Region 5 (IL, IN, MI, MN, OH, WI)","HHS Region 6 (AR, LA, NM, OK, TX)","HHS Region 7 (IA, KS, MO, NE)","HHS Region 8 (CO, MT, ND, SD, UT, WY)","HHS Region 9 (AZ, CA, HI, NV)","HHS Region 10 (AK, ID, OR, WA)","Anchorage, AK","Birmingham, AL","Little Rock, AR","Mesa, AZ","Phoenix, AZ","Scottsdale, AZ","Tempe, AZ","Tucson, AZ","Berkeley, CA","Fresno, CA","Irvine, CA","Los Angeles, CA","Oakland, CA","Sacramento, CA","San Diego, CA","San Francisco, CA","San Jose, CA","Santa Clara, CA","Sunnyvale, CA","Colorado Springs, CO","Denver, CO","Washington, DC","Gainesville, FL","Hialeah, FL","Jacksonville, FL","Miami, FL","Orlando, FL","Tampa, FL","Atlanta, GA","Roswell, GA","Honolulu, HI","Des Moines, IA","Boise, ID","Chicago, IL","Indianapolis, IN","Wichita, KS","Lexington, KY","Baton Rouge, LA","New Orleans, LA","Boston, MA","Somerville, MA","Baltimore, MD","Grand Rapids, MI","Saint Paul, MN","Kansas City, MO","Springfield, MO","Saint Louis, MO","Jackson, MS","Cary, NC","Charlotte, NC","Durham, NC","Greensboro, NC","Raleigh, NC","Lincoln, NE","Omaha, NE","Newark, NJ","Albuquerque, NM","Las Vegas, NV","Reno, NV","Albany, NY","Buffalo, NY","New York, NY","Rochester, NY","Cincinnati, OH","Cleveland, OH","Columbus, OH","Dayton, OH","Oklahoma City, OK","Tulsa, OK","Beaverton, OR","Eugene, OR","Portland, OR","Philadelphia, PA","Pittsburgh, PA","State College, PA","Providence, RI","Columbia, SC","Greenville, SC","Knoxville, TN","Memphis, TN","Nashville, TN","Austin, TX","Dallas, TX","Fort Worth, TX","Houston, TX","Irving, TX","Lubbock, TX","Plano, TX","San Antonio, TX","Salt Lake City, UT","Arlington, VA","Norfolk, VA","Reston, VA","Richmond, VA","Bellevue, WA","Seattle, WA","Spokane, WA","Madison, WI","Milwaukee, WI"
// HTTP GET Request
request("http://www.google.org/flutrends/us/data.txt")
// Read File
//fs.createReadStream("input/google-flutrends.csv")
.pipe(es.split("\n"))
// Split Strings into Array
.pipe(es.mapSync(function (data) {
if (data.length > 160) {
return data;
}
}))
// Join Strings
.pipe(es.join("\n"))
// Concat Strings
.pipe(es.wait())
// Parse CSV as Object
.pipe(csv({objectMode: true, columns: true, delimiter: ',', newline: '\n'}))
// get header
.pipe(headerParser)
// .pipe(process.stdout)
// put values in table row
.pipe(es.mapSync(function (data) {
var row = "<tr>";
for (var i=0; i < keys.length; i++) {
row += "<td>" + data[keys[i]] + "</td>";
}
row += "</tr>";
return row;
}))
// Join Strings
.pipe(es.join("\n"))
// Concat Strings
.pipe(es.wait())
// Wrap Strings
.pipe(es.mapSync(function (data) {
var tabel = "<html><head><style>td {text-align: center;} th {background: #ccc;width: 100%;min-width:200px}</style></head><body><table><tr>";
for (var i=0; i < keys.length; i++) {
tabel = tabel + "<th>" + keys[i] + "</th>";
}
tabel = tabel + "</tr>\n" + data + "\n</table><div id=\"dataviz-container\"></div><script src=\"js/d3.v3.min.js\"></script></body></html>";
return tabel;
})).pipe(res);
// // Write File
// .pipe(fs.createWriteStream("output/flutable.html"))
// .on('finish', function() {
// console.log('Einde');
// });
};
opts._.forEach(function (f) {
fs.createReadStream(f).pipe(c.createStream())
})
}
//if user is streaming input, then
else {
var c = convoy()
var manifest
process.stdin.pipe(c)
var ms = c.createStream() //manifest
.resume()
.on('data', console.log)
.pipe(es.join(function (err, str) {
console.log('>>', str)
manifest = JSON.parse(str)
}))
c.on('push', function () {
// c.createStream().on('data',console.log)
var name = manifest.files.shift()
console.log(name)
// c.createStream().on('data',console.log)
c.createStream().pipe(fs.createWriteStream(name))
})
}
}
hostname: ip,
agent: false,
timeout: 1000,
localAddress: config.getKey('local_ip')
});
req.on('response', function (resp) {
var server ="";
if (resp.headers && resp.headers.server) {
server = resp.headers.server;
}
done(JSON.stringify([ip, resp.statusCode, server]));
}).on('error',function(e) {
done(JSON.stringify([ip, 504, ""]));
}).on('socket', function (socket) {
socket.setTimeout(20000);
socket.on('timeout', function () {
req.abort();
done(JSON.stringify([ip, 504, ""]));
});
});
req.end();
}
var length = fs.statSync(inFile).size;
fs.createReadStream(inFile)
.pipe(progress({total: length}))
.pipe(es.split())
.pipe(mapConcurrent(CONCURRENT_REQUESTS, httpWorker))
.pipe(es.join('\n'))
.pipe(fs.createWriteStream(outFile));
function format ( ) {
return es.pipeline(mapper(sync.text.format), es.join('\n'));
}
var request = require("request");
var zlib = require("zlib");
var Iconv = require("iconv").Iconv;
var csv = require("csv-streamify");
var Handlebars = require("handlebars");
var es = require("event-stream");
var tmpl = Handlebars.compile("<tr><td><a href='{{URL}}'>{{Name}}</a></td><td>{{City}}</td></tr>");
// HTTP GET Request
request("http://nodestreams.com/input/people_euc-jp.csv.gz")
// Un-Gzip
.pipe(zlib.createGunzip())
// Change Encoding
.pipe(new Iconv("EUC-JP", "UTF-8"))
// Parse CSV as Object
.pipe(csv({objectMode: true, columns: true}))
// Convert Object w/ Handlebars
.pipe(es.mapSync(tmpl))
// Join Strings
.pipe(es.join("\n"))
// Concat Strings
.pipe(es.wait())
// Wrap Strings
.pipe(es.mapSync(function(data) {
return "<table><tr><th>Name</th><th>City</th></tr>\n" + data + "\n</table>";
}))
.pipe(process.stdout);
var join = require('event-stream').join
, split = require('split')
, fs = require('fs');
fs.createReadStream(__filename, { encoding: 'utf-8' })
.pipe(split())
.pipe(join('\n******\n'))
.pipe(process.stdout);