async(function () {
var bailed = false, planned = false, plan
byline.createStream(executable.stderr).on('data', function (line) {
emit('err', line)
})
byline.createStream(executable.stdout).on('data', function (line) {
if (bailed) {
emit('out', line)
} else if (tap.assertion(line)) {
emit('test', line)
} else if (!planned && (plan = tap.plan(line))) {
planned = true
emit('plan', plan.expected)
} else if (tap.bailout(line)) {
bailed = true
emit('bail', line)
} else {
emit('out', line)
}
})
delta(async()).ee(executable.stdout)
.ee(executable.stderr)
.ee(executable).on('close')
}, function (code, signal) {
module.exports = function (next) {
var stream = byline.createStream()
var i = 0
, result = {}
stream.on('data', function (line) {
var parts = line.split(',')
if (i++ === 0) {
// Header row is the first row, so initialize the graph
result.graph = {}
result.graph.title = parts[0]
result.graph.datasequences = new Array(parts.length - 1)
for (var j = 1; j < parts.length; j++) {
result.graph.datasequences[j - 1] = { title: parts[j], datapoints: [] }
}
} else {
result.graph.datasequences.forEach(function (dataset, i) {
dataset.datapoints.push({
title: parts[0],
value: i + 1 < parts.length ? parts[i + 1] : null
})
})
}
})
stream.on('end', function () {
next(null, result)
})
return stream
}
function send_serial(fp) {
/*
* 1. open gcode file
* 2. for line in file, convert to printer instructions
* 3. send to printer
*
* -- if unrecognized instruction, raise warning
*/
if (STREAM) {
log.warn("resuming gcode transmission");
} else {
log("starting gcode transmission")
STREAM = fs.createReadStream(fp);
STREAM = byline.createStream(STREAM);
// STREAM = byline(fs.createReadStream(fp));
STREAM.on('end', function() {
serial.send("end of gcode transmission");
serial.close_connection();
});
}
STREAM.on('readable', function() {
var line;
while (null !== (line = STREAM.read())) {
var gcode = parse_line(line.toString());
if (!gcode.skip) {
msg = serial.msg_pack(gcode);
serial.send(msg);
}
}
});
}
fs.stat(config.logPath('executeLog'),function(err,stats){
if(err){
res(null);
}else{
var stream = fs.createReadStream(config.logPath('executeLog'), {encoding:'utf8'});
stream = byline.createStream(stream);
var arr = [];
stream
.on('data',function(streamLineData){
var line;
try{
line = toutf.convert(streamLineData).toString();
var lineObj = JSON.parse(line);
arr.push(lineObj);
}catch(err){
res(null);
}
})
.on('end',function(){
arr.sort(function(a,b){
return ((new Date(b.executeTime))-(new Date(a.executeTime)));
});
res(null,arr);
});
}
});
var widefinder = function() {
var stream = fs.createReadStream(data);
var stream = byline.createStream(stream);
stream.on('data', function(line) { win.enqueue(line); });
stream.on('end', function() { win.tick(); });
};
function makels(filepath,online,onend) {
var rs=fs.createReadStream(filepath)
rs.setEncoding('ascii')
var ls = byline.createStream()
online && ls.on('data', online)
onend && ls.on('end', onend)
rs.pipe(ls)
return ls
}
collect.all(function(aggregate) {
fs.createReadStream(testdir+base+'.sql')
.pipe(new sql[dialect].Lex())
.on('data',function(token){ parsed.push(token) })
.on('end', aggregate());
var fh = fs.createReadStream(testdir+base+'.tokens');
fh.setEncoding('utf8');
fh.pipe(byline.createStream())
.on('data',function(line){ expected.push(JSON.parse(line)) })
.on('end', aggregate());
}, function (err) {
it('and sends back one line of log', function (done) {
var child = shell.exec(__dirname + '/../bin/logfaker --nb=1', {async: true, silent: true});
var stream = byline.createStream(child.stdout);
var nbline = 0;
stream.on('data', function (line) {
nbline++;
})
stream.on('end', function () {
should.equal(nbline, 1);
done();
});
});
var parseJournal = function() {
console.log("parse Journal..")
/*var gw = fs.watch(".abst_output", function (event, filename) {
if (filename == "goabm."+runid) {
// console.log('filename provided: ' + filename);
var jw = fs.watch(outpath, { persistent: false},function (event, filename) {
if(filename == "journal.gz") {
// dirty hack to wat for the journal to be created
// would be usefule to have someting like. child_proces.on("ready")
//console.log("journal ready")
jw.close()
gw.close()
*/
var fstream = fs.createReadStream(journal/*, { encoding: 'utf8' }*/);
var unzip = new Stream()
gzip = zlib.createGunzip()
var stream = byline.createStream();
fstream.pipe(gzip).pipe(stream)//.pipe(process.stdout)
console.log("reading journal..", journal)
/* gzip.on('data', function(chunk) {
console.log('got %d bytes of data %s', chunk.length,chunk.toString());
})*/
fstream.on('end', function() {
console.log("done with input")
})
var lines = 0;
stream.on('data', function(line) {
lines++
var content = JSON.parse(line)
Model.socketio.emit('model:journal', content);
console.log("send a state");
});
stream.on('close',function() {
console.log("done streaming journal"+ journal+" with " + lines + " states")
done(null,null)
});
}
/*
* Server
*/
function authenticate(connection, ready)
{
if (process.getuid() != 0) /* No authentication for user services */
return ready();
var authErr = function(msg)
{
console.log(msg);
connection.end();
};
var stream = byline.createStream(connection);
var onLine = function(data)
{
var line = data.toString();
if (! line.match("^auth")) return;
var cookiePath = line.split(":")[1];
var cookieToken = line.split(":")[2];
fs.readFile(cookiePath, function(err, buf)
{
if (err) console.error(err);
if (err || buf.toString() != cookieToken)
return authErr("Authentication error: cookie/token mismatch");
fs.stat(cookiePath, function(err, stat)
{
if (Date.now() - stat.mtime.getTime() > COOKIE_TTL)
return authErr("Authentication error: cookie expired");
/*
* TODO: one more security check: if the cookie is in the user's home directory
* this will happen after we read /etc/passwd to find more info about the user
*/
/* only this for now; TODO: more */
ready({ uid: stat.uid });
stream.removeListener("data", onLine);
});
});
};
stream.on("data", onLine);
};
var command = function (command, args, callback) {
if (callback === undefined) {
callback = args;
args = [];
}
args.unshift(command);
var git = spawn('git', args);
var remotes = [];
var stdout = byline.createStream(git.stdout);
stdout.on('data', function (line) {
remotes.push(line.toString('utf8'));
});
git.on('close', function (code) {
if (code !== 0) {
return callback({name: 'BadCode', message: command + ' got exit code ' + code});
}
return callback(null, remotes);
});
};
function says(t, pat) {
var watcher = byline.createStream();
var found = false;
debug('# watching for: ', pat);
watcher.on('data', function(line) {
if (!found && pat.test(line)) {
found = true;
c.stdout.unpipe(watcher);
} else {
debug('# > %s', line);
}
});
watcher.on('unpipe', function() {
t.ok(found, 'saw ' + pat);
debug('# unpiped!');
t.end();
});
c.stdout.pipe(watcher);
}
SrCreator.prototype._fileToDb = function(fileKey, cb) {
var table = mappings[fileKey];
// Running under the assumption that the top level properties of the
// mappings file are the same names as the file names
var filePath = this.srDir + fileKey + '.txt';
console.log('Parsing ' + filePath + '...');
var lineStream = byline.createStream(fs.createReadStream(filePath));
var splitStream = new SplitStream();
var insertStream = new InsertStream(this.db, table);
lineStream.pipe(splitStream).pipe(insertStream);
var numLines = 0;
var progressInterval;
var progressBar;
lineStream.on('data', function(chunk) {
++numLines;
});
lineStream.on('end', function() {
var progressString =
'and inserting '
+ ' [:bar] :percent (:current/:total) :etas';
progressBar = new ProgressBar(progressString, {
complete: '=',
incomplete: ' ',
width: 40,
total: numLines
});
progressInterval = setInterval(function() {
progressBar.update(insertStream.numInsertions/numLines);
}, 500);
});
insertStream.on('finish', function() {
progressBar.update(1);
clearInterval(progressInterval);
cb();
});
};
AndroidTracingProbe.prototype.waitForLogCat_ = function(query, timeout) {
var deferred = Q.defer(),
line = '',
commandString = "adb logcat -s chromium:W TracingControllerAndroid:I",
search = new RegExp(query, "i");
timeout = timeout || 1000 * 60 * 2;
var command = commandString.split(/\s/);
var logcat = childProcess.spawn(command[0], command.slice(1));
stream = byline.createStream(logcat.stdout);
var timerHandle = setTimeout(function() {
done(false);
}, timeout);
stream.on('data', function(data) {
var line = data.toString();
//debug('> ', line);
if (line.match(search)) {
done(line);
}
});
var completed = false;
function done(arg) {
if (timerHandle) {
clearTimeout(timerHandle);
}
if (!completed) {
completed = true;
if (arg) {
deferred.resolve(arg);
} else {
deferred.reject();
}
}
logcat.kill();
}
return deferred.promise;
};
exports.write = function (t) {
var path = '/tmp/' + uuid();
confparser.write(path, POSTGRESQL_CONF_OBJ, function (err) {
if (err) {
t.fail(err);
t.done();
}
});
var stream = fs.createReadStream(path);
stream = byline.createStream(stream);
var i = 0;
stream.on('data', function (line) {
t.equal(POSGRESQL_CONF_STR[i], line);
i++;
if (i == POSGRESQL_CONF_STR.length) {
t.done();
}
});
};
const line_filter = function(stream) {
let byline = require('byline');
let line_splitter = byline.createStream();
return stream.pipe(line_splitter).pipe(new TabSplitter());
}
si.ready(function(err,si){
console.log('ready '+err+si)
var read = fs.ReadStream(npmfile);
read.setEncoding('ascii');
var linestream = byline.createStream()
var count = 0
var done = 0
var modent = si.make('mod')
linestream.on('data', function(line){
try {
var data = JSON.parse( cutcomma(line) )
count++
modent.load$({name:data.id},function(err,mod){
if(err) {
return console.error(err)
}
mod = mod || modent.make$()
mod.name = data.id
var latest = npmentry.latest(data)
if( latest && latest.repository && 'git'==latest.repository.type ) {
mod.lastgit = mod.lastgit || 0
mod.giturl = latest.repository.url
}
else {
mod.lastgit = Number.MAX_VALUE
}
mod.lastnpm = mod.lastnpm || 0
mod.save$(function(err,mod){
done++
if( 0 == count % 100 ) {
process.stdout.write('*')
}
})
})
}
catch(e){
console.error(e)
}
})
linestream.on('end',function(){
console.log('\nmongo-size:'+count)
console.log('count:'+count+' done:'+done)
function waitfordb() {
if( done < count ) {
process.stdout.write(' '+done)
setTimeout(waitfordb,333)
}
else {
si.close()
if( argv.i ) {
insertall(depsfile,argv.h)
}
}
}
waitfordb()
})
read.pipe(linestream)
})
this.push([
(this.lastLineHi && !_lastLineHi) ? '\nhi' : 'hi',
name,
...rest
].join(' ') + '\n')
return cb()
} else if (this.lastLineComment && !_lastLineComment) {
this.push('\n' + str + '\n')
return cb()
}
this.push(str + '\n')
return cb()
}
}
const draculaSyntax = byline.createStream(
fs.createReadStream(path.join(__dirname, 'template.vim'))
)
const adventurousSyntax = fs.createWriteStream(
path.join(__dirname, '../colors/adventurous.vim')
)
draculaSyntax
.pipe(new TransformColors())
.pipe(adventurousSyntax)
var xml2js = require('xml2js');
var sprintf = require('sprintf').sprintf;
var fs = require('fs');
var byline = require('byline');
var Benchmark = require('benchmark');
var opts = {
config: {
fromDirectory: "/home/pt/melinda/data/xml"
},
input: "/home/pt/dev/deduplication/test.txt"
};
var all = [];
var input = byline.createStream( fs.createReadStream(opts.input));
input.on('data', function(data) {
var line = data.toString();
var ids = line.split("\t");
ids.forEach(function(id) {
all.push(id);
});
});
input.on('end', function() {
console.log("Testing");
var suite = new Benchmark.Suite;
#!/usr/bin/env node
var N3 = require('n3');
var N3Util = N3.Util;
var parser = N3.Parser();
var elasticsearch = require('es');
var fs = require('fs');
var byline = require('byline');
var url=require('url');
var stream = byline.createStream(process.stdin);
var docs=[];
var bulksize = 50000;
var category = require('unicode-7.0.0/categories');
var isNLS = function(s1){
var consecutive=0
if (s1.length<2) return false;
for (var i = 0, len=s1.length; i<len; i++) {
if(category[ s1.charCodeAt(i) ][0]=="L") {
if (++consecutive==2) return true;
} else {
consecutive=0;
}
if (i==len-1) return false;
}
}
var configurationFile = 'config.json';
var configuration = JSON.parse(
/**
* Created by stefansteinhart on 31.01.15.
*/
/// <reference path="byline.d.ts" />
var fs = require('fs');
var byline = require('byline');
//TODO can this be typed in an ambient way?
//var stream = byline( fs.createReadStream( 'sample.txt', {encoding: 'utf8'} ) );
var stream = byline.createStream(fs.createReadStream('sample.txt', { encoding: 'utf8' }));
stream.on('data', function (line) {
console.log(line);
});
stream = byline.createStream(stream);
stream.on('data', function (line) {
console.log(line);
});
var input = fs.createReadStream('sample.txt');
stream.pipe(fs.createWriteStream('nolines.txt'));
var lineStream = byline.createStream();
input.pipe(lineStream);
var output = fs.createWriteStream('test.txt');
lineStream.pipe(output);
stream.on('readable', function () {
var line;
while (null !== (line = stream.read())) {
console.log(line);
}
});
var LineStream = require('byline').LineStream;
var output = fs.createWriteStream('nolines.txt');
var lineStream = new LineStream();
return new Promise(function(resolve) {
let byline = require('byline');
let line_splitter = byline.createStream();
resolve(stream.pipe(line_splitter).pipe(new TabSplitter()).pipe(filter));
});
fs.readFile(path.resolve("Z:/Eve/Data/itemNameMap.json"), function(err, data)
{
itemIDMap = JSON.parse(data);
file= path.resolve(file);
console.log('Checking file: ', path.resolve(file))
var checkCount = 1;
var cnt = 0;
var lineCount = 0;
var uniqueOrders = {};
var uniqueCount = 0;
var confirmedPurchases = {};
var confirmedPurchasesByStation = {};
var confirmedSales = {};
var confirmedSalesByStation = {};
var stream = fs.createReadStream(file);
stream = byline.createStream(stream);
stream.on('data', function(line) {
if(lineCount > 0)
{
if(lineCount%10000 == 1)
console.log("At line: " + lineCount);
//get rid of any quotation marks
var delimited = line.toString().replace("\r", "").replace(/"/g, "").split(",");
var ixBuyOrder = 0;
var ixStationID = 3;
var ixName = 4;
var ixBuySell = 5;
var ixPrice = 6;
var ixRemaining = 8;
var oID = delimited[ixBuyOrder];
if(!uniqueOrders[oID])
{
uniqueCount++;
uniqueOrders[oID] = delimited;
}
else
{
//let's check if any buying or selling happened
if(uniqueOrders[oID][ixRemaining] != delimited[ixRemaining])
{
var itemID = delimited[ixName];
var stationID = delimited[ixStationID];
var pricePU = parseFloat(delimited[ixPrice]);
var quanitityDifference = parseInt(uniqueOrders[oID][ixRemaining]) - parseInt(delimited[ixRemaining]);
var itemName = itemIDMap[itemID];
var stationName = stationIDMap[stationID];
if(!stationName)
console.log("Cant find: " + stationID + "...skipping.");
//don't look at antyhign we don't know the name for
if(quanitityDifference > 0 && stationName)
{
if(uniqueOrders[oID][ixBuySell] == "1")
{
//a buy order has changed quantities!
//this confirms a PURCHASE at that price
//someone actually fulfilled part of a buy order
if(!confirmedPurchases[itemName])
confirmedPurchases[itemName] = {};
if(!confirmedPurchases[itemName][stationName])
confirmedPurchases[itemName][stationName] = [];
var purchaseData = {
item: itemName,
station: stationName,
quantity: quanitityDifference,
pricePerUnit: pricePU,
total: quanitityDifference*pricePU,
fullData: delimited
};
confirmedPurchases[itemName][stationName].push(
purchaseData
);
if(!confirmedPurchasesByStation[stationName])
confirmedPurchasesByStation[stationName] = {};
if(!confirmedPurchasesByStation[stationName][itemName])
confirmedPurchasesByStation[stationName][itemName] = [];
confirmedPurchasesByStation[stationName][itemName].push(purchaseData);
}
else
{
//otherwise, an item has been sold to someone else
//this is a confirmed sale
//someone actually offloaded some item
if(!confirmedSales[itemName])
confirmedSales[itemName] = {};
if(!confirmedSales[itemName][stationName])
confirmedSales[itemName][stationName] = [];
var saleData = {
item: itemName,
station: stationName,
quantity: quanitityDifference,
pricePerUnit: pricePU,
total: quanitityDifference*pricePU,
fullData: delimited
};
confirmedSales[itemName][stationName].push(saleData);
if(!confirmedSalesByStation[stationName])
confirmedSalesByStation[stationName] = {};
if(!confirmedSalesByStation[stationName][itemName])
confirmedSalesByStation[stationName][itemName] = [];
confirmedSalesByStation[stationName][itemName].push(saleData);
// console.log('Sell recorded!');
}
//we only update if the difference is greater than zero (otherwise, we could be updating an out of order change
uniqueOrders[oID] = delimited;
}
}
}
}
//....
//When done set the line back to empty, and resume the strem
lineCount++;
});
stream.addListener('end', function()
{
//save unique orders as json file
console.log('End stream!');
console.log("Line count: ", lineCount, " Unique: ", uniqueCount);
fs.writeFile(path.resolve("Z:/Eve/Data/" + uniqueName + "_purchaseData.json"),
JSON.stringify({purchaseData: confirmedPurchases, purchaseDataByStation: confirmedPurchasesByStation}),
function(err)
{
if(err)
console.log('Failed write purchase data');
else
console.log('Saved confirmed purchase data.');
});
fs.writeFile(path.resolve("Z:/Eve/Data/" + uniqueName + "_saleData.json"),
JSON.stringify({saleData: confirmedSales, saleDataByStation: confirmedSalesByStation}),
function(err)
{
if(err)
console.log('Failed write sale data');
else
console.log('Saved confirmed sale data.');
});
fs.writeFile(path.resolve("Z:/Eve/Data/" + uniqueName + ".json"), JSON.stringify(uniqueOrders), function(err)
{
if(err)
console.log('Failed write unique');
else
console.log('Saved Unique.');
});
});
});
function links( pathfile ) {
console.log('STAGE: links '+filepath)
var depsfile = path.dirname(filepath) +'/deps.json'
var linksfile = path.dirname(filepath) +'/links.json'
var read = fs.ReadStream(depsfile);
read.setEncoding('ascii');
//var jsr = js.parse([true])
var linestream = byline.createStream()
var index = {}
var links = {
"py/object": "__main__.web",
dangling_pages:{},
in_links:{},
number_out_links:{},
}
var count = 0
linestream.on('data',function(line){
if( ',' == line || '[' == line || ']' == line ) return;
try {
var data = JSON.parse(cutcomma(line))
data.i = ''+data.i
index[data.m]=data
links.dangling_pages[data.i]=true
count++
if( 0 == count % 100 ) {
process.stdout.write('+')
}
}
catch( e ) {
console.log(e+' bad line:<'+line+'>')
}
})
linestream.on('end',function(){
console.log('end')
var count = 0
for( var n in index ) {
count++
var m = index[n]
links.in_links[m.i] = []
}
for( var n in index ) {
var m = index[n]
//console.log(n+' -> '+m.d)
for( var dI = 0; dI < m.d.length; dI++ ) {
var depname = m.d[dI]
if( _.isUndefined(index[depname]) ) {
continue
}
var dep = index[depname].i
links.in_links[dep].push( m.i )
if( _.isUndefined(links.number_out_links[m.i]) ) {
links.number_out_links[m.i]=1
delete links.dangling_pages[m.i]
}
else {
links.number_out_links[m.i]++
}
}
}
links.size = count
fs.writeFileSync(linksfile,JSON.stringify(links))
console.log('links-size:'+count)
if( argv.r ) {
rank(depsfile)
}
})
read.pipe(linestream)
}
var request = require("request"),
fs = require("fs"),
byline = require("byline"),
mongoose = require("mongoose"),
Sighting = require('./models/sighting');
mongoose.connect(process.env.MONGODB_UFO_URI);
var id = 0;
var stream = fs.createReadStream('./ufo_awesome.json');
stream = byline.createStream(stream);
stream.on('data', function(line) {
id += 1;
console.log("Processing ID " + id);
var sighting = JSON.parse(line);
var s = new Sighting(sighting);
s.save(function (err, s) {
if (err) {
console.log(err);
throw err;
}
console.log("Sighting pushed to mongo");
});
});
.on('root',function(){
var rs = fs.createReadStream(npmfile);
rs.setEncoding('ascii')
var linestream = byline.createStream()
linestream.on('data', function(line){
try {
try {
var data = JSON.parse( cutcomma(line) )
}
catch(e){
console.log(e)
return
}
var v = data.value
var doc = {
name:v.name,
desc:v.description||'',
latest:(v['dist-tags']||{}).latest,
author:(v.author&&v.author.name)||''
}
var r = v.versions[doc.latest]
if( r ) {
if( r.homepage ) {
doc.site = ''+r.homepage
}
doc.keywords = (r.keywords||[])
if( _.isArray( doc.keywords ) ) {
doc.keywords = doc.keywords.join(' ')
}
doc.license = r.license||''
try {
doc.maints = _.isArray(r.maintainers) ?
_.map(r.maintainers||[],function(m){return m.name}).join(' ') :
''+r.maintainers
}
catch( err ) {
console.log(err)
}
doc.readme = r.readme||''
if( r.repository && r.repository.url ) {
var git = r.repository.url
.replace(/^git:/,"https:")
.replace(/git@github.com:/,"https://github.com/")
.replace(/\.git$/,"")
doc.git = git
if( !doc.site ) {
doc.site = git
}
else {
doc.site = ''+doc.site
if( doc.site.match(/^www/) ) {
doc.site = 'http://'+doc.site
}
}
}
if( !doc.site ) {
doc.site = "http://npmjs.org/package/"+v.name
}
if( doc.site && !doc.git ) {
if( doc.site.match(/github.com\//) ) {
doc.git = doc.site
}
}
}
if( prmap[v.name] ) {
doc.nr = prmap[v.name]
doc._boost = prmap[v.name] / 10
}
else {
doc._boost = 1
}
var mul = {}
if( doc.latest ) {
try {
mul.version = 1
var parts = doc.latest.split('.')
for( var pI = 0; pI < parts.length; pI++ ) {
try {
mul.version += parseInt(parts[pI].substring(0,1)) / (1000 * (parts.length-pI))
} catch(e) {}
}
doc._boost *= mul.version
}
catch(e){}
}
if( v.time ) {
try {
var now = new Date().getTime()
if( v.time.created && v.time.modified ) {
var created = new Date(v.time.created).getTime()
var modified = new Date(v.time.modified).getTime()
var age = now - created
var recent = now - modified
mul.time = 1 + Math.min( 0.001, ( age / recent ) / 10000 )
doc._boost *= mul.time
doc.created = created
doc.modified = modified
}
}
catch(e){}
}
//console.dir(doc)
count++
request({method:'POST',url:hosturl+doc.name,json:doc},function(err,res,bdy){
done++
//console.log(v.name+' '+err)
//console.dir(bdy)
if( err ) {
console.log(err)
}
})
//console.log(doc.name+' r='+doc.nr+' b='+doc._boost+' vm='+mul.version+' tm='+mul.time)
if( 0 == count % 100 ) {
process.stdout.write('.')
}
}
catch( err ) {
console.log(err)
}
})
linestream.on('end',function(){
console.log('\nindex-size:'+count)
function waitforindex() {
console.log('count: '+count+' done:'+done)
if( done < count ) {
process.stdout.write(' '+done)
setTimeout(waitforindex,333)
}
else {
cb(null,{index_size:count})
}
}
waitforindex()
})
rs.pipe(linestream)
}))