Example #1
0
  return new Promise((resolve, reject) => {
    // Use `spawn` here to process the, possibly huge, output of the file listing.

    const proc = spawn(command, args, {cwd: localDirectory});

    proc.on('error', reject);

    const filePaths = [];
    proc.stdout.pipe(split()).on('data', filePath => {
      if (transform) {
        filePath = transform(filePath);
      }

      if (filePath !== '') {
        filePaths.push(filePath);
      }
    });

    let errorString = '';
    proc.stderr.on('data', data => {
      errorString += data;
    });

    proc.on('close', code => {
      if (code === 0) {
        resolve(filePaths);
      } else {
        reject(errorString);
      }
    });
  });
Example #2
0
module.exports = function(stream) {
  var input = stream.pipe(split())
  var self = new EventEmitter();
  self.current = 0;

  // Format is each line must end in |N
  input.on('data', function(buffer) {
    if (buffer.length < 1) return;
    var idx = buffer.lastIndexOf('|');
    var number = parseInt(buffer.slice(idx + 1), 10)
    if (isNaN(number)) {
      self.emit(
        'error', 'Invalid number on sequence: ' + self.current  + '\n' + buffer
      );
      return
    }

    if (number != self.current++) {
      var msg = 'Out of order expected %d got %d';
      self.emit('error', util.format(msg, self.current, number));
      return
    }
    self.emit('sequence', number);
  });
  input.once('end', self.emit.bind(self, 'end'));
  return self;
};
Example #3
0
  skateboard({ port: 7007 }, function(stream) {

    stream.pipe(split()).on('data', function(d) {
      var d = d.trim();
      if (d.length) {
        d = d + '\n';

        if (!pending) {
          machine.write(d);
          pending = true;
        } else {
          skateLines.push(d);
        }
      }
    });

    machine.on('data', function(data) {
      if (data.trim() === 'ok') {
        if (skateLines.length) {
          machine.write(skateLines.shift());
        } else {
          pending = false;
        }
      }

      stream.write(data.toString());
    });
  });
Example #4
0
module.exports = function() {
  var tr = through(write, end),
      current;

  function write(buffer, encoding, next) {
    if (buffer.length === 0) return next();
    var data = JSON.parse(buffer);

    if (data.type === 'genre') {
      if (current) this.push(JSON.stringify(current) + '\n');
      current = { name: data.name, books: [] };
    } else if (data.type === 'book') {
      current.books.push(data.name);
    }

    next();
  }

  function end(next) {
    if (current) this.push(JSON.stringify(current) + '\n');
    next();
  }

  return combine(split(), tr, zlib.createGzip());
};
Example #5
0
module.exports = function () {
  var current;
  return combine(
    // read newline-separated json,
    split(),
    // group books into genres,
    through(function write(line) {
      if (line.length) {
        var row = JSON.parse(line);
        if (row.type === 'genre') {
          if (current) {
            this.queue(JSON.stringify(current) + "\n");
          }
          current = { 'name': row.name, 'books': [] };
        }
        else if (row.type === 'book') {
          current.books.push(row.name);
        }
      }
    }, function end () {
      if (current) {
        this.queue(JSON.stringify(current) + '\n');
      }
      this.queue(null);
    }),
    // then gzip the output
    gzip
  )
}
  return function(req, res, next) {
    //setup
    if (req._body) return next();
    var is_mime = req.header('content-type') === mime;
    if (!is_mime) return next();
    req._body = true;

    //define Readable body Stream
    req.body = new Readable({ objectMode: true });
    req.body._read = function(n) {
      req.body._paused = false;
    };

    function parseLine(line) {
      if(line) {
        var parsedLine = options.parser(line);
        if(!req.body._paused) req.body._paused = !req.body.push(parsedLine);
      }
    }
    function end() { req.body.push(null); }

    function trimNewline(line) {
      return line.replace(/\r?\n/, '');
    }

    req.pipe(split(/\b\r?\n\b/, trimNewline, null)).pipe(through(parseLine, end));

    return next();
  }
Example #7
0
function createSimulator(interval, stepsPerMM) {
  interval = interval || 0.1;
  stepsPerMM = stepsPerMM || 250;

  var sim_process = child.spawn(path.join(dir, 'grbl_sim.exe'), [interval]);

  sim_process.on('open', function() {
    [0, 1, 2].forEach(function(i) {
      sim_process.stdin.write('$' + i + '=' + stepsPerMM + '\r\n');
    });
  });

  sim_process.stdout.pipe(process.stdout);

  return duplexer(
    sim_process.stdin,
    sim_process.stderr.pipe(split()).pipe(through(function(d) {
      d = d.trim();

      var parts = d.split(', ');

      if (d[0] !== '#' && parts.length) {

        var obj = {
          time : parseFloat(parts[0]),
          x : parseInt(parts[1]) / stepsPerMM,
          y : parseInt(parts[2]) / stepsPerMM,
          z : parseInt(parts[3]) / stepsPerMM
        };
        this.push(JSON.stringify(obj) + '\n');
      }
    })));

  return stream;
};
function book_stream() {
  var parse_books = through(capture_books, end)
    , list = []

  return combine(
      split()
    , parse_books
    , zlib.createGzip()
  )

  function capture_books(str) {
    try {
      var obj = JSON.parse(str)
    } catch(e) {
      return
    }

    if (obj.type === 'genre') return list.push({genre: obj.name, books: []})
    list[list.length - 1].books.push(obj.name)
  }

  function end() {
    for (var i = 0, l = list.length; i < l; ++i) {
      parse_books.queue(JSON.stringify(list[i]) + '\n')
    }
  }
}
Example #9
0
module.exports = function () {
  return combine(
  split(), // read newline-separated json,
  group, // group books into genres,
  zlib.createGzip() // then gzip the output
  )
}
Example #10
0
list.forEach(function (index) {
    var lineCount = 0

    // create a client. Delay the execution of the periodic sending
    // so that they don't all send messages at roughly the same
    // time
    // Then get them to send a single buffer (containing a string)
    // `volume` amount of times
    var client = net.connect(TCP_PORT, function () {
        setTimeout(function () {
            setInterval(function () {
                for (var i = 0; i < volume; i++) {
                    client.write(b)
                }
            }, 1000 / rate)
        }, (1000 / clients) * index)
    })

    // the data is line seperated. So we use split to parse it
    // and then just log the lineCount when it reaches a certain
    // size
    client.pipe(split()).on("data", function () {
        ++lineCount

        if ((lineCount % volumeCheck) === 0 && index === 0) {
            console.log("lineCount", lineCount)
        }
    })
})
Example #11
0
/**
 * A git pre-receive hook, which reads from stdin and checks
 * each updated ref.
 */
function doPreReceive(check) {
    process.stdin.setEncoding("binary");

    const updates = [];
    process.stdin.pipe(split()).on("data", function(line) {
        if (line === "") {
            return;
        }
        const parts = line.split(" ");
        if (parts.length !== 3) {
            process.exit(1);
        }
        updates.push({
            "oldSha" : parts[0],
            "newSha" : parts[1],
            "ref" : parts[2]
        });
    }).on("end", function() {
        co(function *() {
            const repo = yield NodeGit.Repository.open(".");
            return yield check(repo, updates);
        }).then(function(res) {
            process.exit(+res);
        }, function(e) {
            console.error(e);
            process.exit(2);
        });
    });
}
module.exports = function () {
  var genreInfo;

  function write(buf) {
    if (buf.length === 0) {
      return;
    }

    buf = JSON.parse(buf);
    if (buf.type === 'genre') {
      if (genreInfo) {
        this.queue(JSON.stringify(genreInfo) + '\n');
      }

      genreInfo = {
        name: buf.name,
        books: []
      }
    }
    else {
      genreInfo.books.push(buf.name);
    }
  }

  function end() {
    this.queue(JSON.stringify(genreInfo) + '\n');
    this.queue(null);
  };

  return combine(
    split(),
    through(write, end),
    zlib.createGzip()
  );
};
Example #13
0
 function(cb) {
   var rowsProcessed = 0;
   var first = true;
   var columns;
   var rStream = opfile.readStream;
   var oStream = fs.createWriteStream(outputFile);
   var tStream = new Stream.Transform();
   tStream._transform = function(chunk, encoding, callback) {
     if (chunk.toString().trim() == '') {
       this.push('');
       return callback();
     }
     if (!first) rowsProcessed++;
     if (first) {
       columns = chunk.toString().replace(/_IND/g, '').split('\t');
       first = false;
     }
     var data = chunk.toString().replace(/_IND/g, '');
     this.push(data + '\n');
     callback();
   };
   oStream.on('error', function(err) {
     cb(err);
   });
   oStream.on('finish', function() {
     cb(null, rowsProcessed, columns);
   });
   rStream.pipe(split()).pipe(tStream).pipe(oStream);
 }
Example #14
0
File: node.js Project: pgte/skiff
  start (done) {
    const args = [this._address, JSON.stringify(this._options)]
    this._child = fork(path.join(__dirname, 'server.js'), args, {
      silent: true
    });

    channels.forEach(channel => {
      this._child[channel].pipe(split())
        .on('data', line => {
          line = line.trim()
          if (line) {
            process[channel].write(`${this._address} (${this._child.pid}): ${line}\n`)
          }
        })
    })

    this._child.stdout.pipe(split()).once('data', (line) => {
      if (line.match(/started/)) {
        done()
      } else if (!this._exiting) {
        done(new Error(`Could not start child, first line of output was ${line}`))
      } else {
        done()
      }
    })

    this._child.once('exit', (code, signal) => {
      if (!this._exiting) {
        throw new Error(`child exited without being asked to, code = ${code}, signal = ${signal}`)
      }
    })
  }
test('nested splicer', function (t) {
    t.plan(1);
    
    var addNewLines = through(function (buf, enc, next) {
        this.push(buf + '\n');
        next();
    });
    
    var stream = pipeline.obj([
        [ split(), addNewLines ],
        through(function (buf, enc, next) {
            this.push('> ' + buf);
            next()
        })
    ]);
    
    stream.get(0).unshift(through(function (buf, enc, next) {
        this.push(buf.toString('utf8').toUpperCase());
        next();
    }));
    
    stream.pipe(concat(function (body) {
        t.deepEqual(body.toString(), '> A\n> B\n> C\n');
    }));
    
    stream.write('a\n');
    stream.write('b\n');
    stream.write('c');
    stream.end();
});
Example #16
0
(function(){

	'use strict';

	var split = require('split');
    var through = require('through2');

    
    //var numbers =['one','two','three','four','five','six','seven','eight','nine'];
    var lineNumber = 0;
    var write = function(buffer, encoding, callback){
      var info =  "";
     if(lineNumber % 2 === 0){
       info = buffer.toString().toLocaleLowerCase() + '\n';
     }else{
     	info = buffer.toString().toLocaleUpperCase() + '\n';
     }
     this.push(info);
     lineNumber++;
     callback();
    };

    var stream = through(write);
	process.stdin.pipe(split()).pipe(stream).pipe(process.stdout);

})();
module.exports = function () {
  var genre;
  return combine(
    // Read newline-separated json,
    split(),
    // Parse and transform
    through(function(buf) {
      if (buf === '') {
        // Flush last object
        if (genre) {
          this.queue(JSON.stringify(genre) + '\n');
        }
        return;
      }
      var line = JSON.parse(buf);
      if (line.type === 'genre') {
        // If we have a genre, stringify and flush
        if (genre) {
          this.queue(JSON.stringify(genre) + '\n');
        }
        // Create a new genre for input
        genre = {
          name: line.name,
          books: []
        };
      } else if (line.type === 'book') {
        // Add to the current genre
        genre.books.push(line.name);
      }
    }),
    // Gzip output
    zlib.createGzip()
  );
};
Example #18
0
  this.stream.on('response', (function (res) {
    var self = this
    // Rate limited or temporarily unavailable
    if (res.statusCode === 420 || res.statusCode === 503) {
      var backoff = res.statusCode === 420 ? this.rateBackoff() : this.httpBackoff();
      this.abort()
      setTimeout(function () {
        self.connect()
      }, backoff)

      this.emit('reconnect', {
        type: this.errorExplanation[res.statusCode].type,
        explain: this.errorExplanation[res.statusCode]
      })
      return
    }

    // Http error
    if (res.statusCode > 200) {
      this.abort()

      this.emit('error', {
        type: 'http',
        err: new Error('Twitter connection error ' + res.statusCode),
        code: res.statusCode,
        explain: this.errorExplanation[res.statusCode]
      })
      return
    }

    // 200. Alive and well.
    this.backoffs()

    this.emit('connect')

    this.parser = split(null, function (d) {
      try {
        return JSON.parse(d)
      } catch (e) {}
    })

    this.parser = res.pipe(this.parser, {end: false})
    this.parser.pipe(this)

    // Handle this: https://dev.twitter.com/docs/streaming-apis/connecting#Stalls
    // Abort the connection and reconnect if we haven't received an update for 90 seconds
    var close = (function () {
      this.abort()
      process.nextTick(this.connect.bind(this))
      this.emit('reconnect', {type: 'stall'})
    }).bind(this)

    this.timeout = setTimeout(close, this.timeoutInterval)

    res.on('data', function () {
      clearTimeout(self.timeout)
      self.timeout = setTimeout(close, self.timeoutInterval)
    })
  }).bind(this))
Example #19
0
  files.forEach(function(file) {
    if (!/test-buffer.*/.test(file.name)) return

    hyperquest(file.download_url, httpOpts)
      .pipe(split())
      .pipe(testfixer(file.name))
      .pipe(fs.createWriteStream(__dirname + '/../test/node-' + file.name))
  })
Example #20
0
var server = http.createServer(function (req, res) {
    if (req.method !== 'POST') res.end('serious business logic\n');
    
    req.pipe(split('\n')).pipe(through(function (line) {
        var n = (Math.pow(1.102, Number(line)) + 8.92) * 1.05;
        this.emit('data', n + '\n');
    })).pipe(res);
});
Example #21
0
 function stderrPipe (readable) {
   readable
     .pipe(split())
     .on('data', function (line) {
       if (line === '') return
       errors.push(line)
     })
 }
Example #22
0
module.exports = function(a, b, cb) {
  var cutoff = 20
  var lines = 0
  var linesMatched = 0
  
  var tupler = tuple(a.stdout.pipe(split()), b.stdout.pipe(split()))
  
  console.log('Diff of first 20 lines below (', styled('red', 'not matched'), ',', styled('green', 'matched'), ')\n')
  
  // bail after 20 seconds
  var timeout = setTimeout(function() {
    console.error('ERROR: 20 second timeout. Try "run" instead of "verify"!')
    lines++
    process.exit(0)
  }, 20000)
    
  var differ = through(function(ch) {
    lines++
  
    if (ch[0] === ch[1]) linesMatched++
  
    if (lines < cutoff) {
      var diff = jsDiff.diffChars(ch[1], ch[0])
  
      diff.forEach(function(part){
        var color = 'green'
        if (part.added) color = 'brightRedBg white'
        if (part.removed) color = 'brightRedBg white'
        // var color = 'grey'
        // if (part.added) color = 'green'
        // if (part.removed) color = 'red'
        process.stdout.write(styled(color, part.value))
      })

      console.log('')
    }
  }, function() {
    differ.queue(null)
    clearTimeout(timeout)
    if (lines === linesMatched) return cb(true)
    console.log('\nOnly', linesMatched, 'of', lines, 'lines matched.\n')
    cb(false)
  })
  
  tupler.pipe(differ)
}
Example #23
0
 tcpServer.on("connection", function(clientSock) {
     // handle connections
     clientSock.setEncoding("ascii"); // probably
     
     clientSock
         .pipe(split(), { end: false })
         .on("data", self._dispatchRecord);
 });
Example #24
0
module.exports.count_rename = function (source_dir, destination_dir, extension, 
    prefix, suffix, padding, start_count, callback) {
  if (typeof source_dir !== 'string') {
    return callback('error: source_dir is invalid');
  }
  if (typeof destination_dir !== 'string') {
    return callback('error: destination_dir is invalid');
  }
  if (source_dir === destination_dir) {
    return callback('error: source_dir and destination_dir are the same'); 
  }
  if (typeof extension !== 'string') {
    return callback('error: extension is invalid');
  }
  if (typeof prefix !== 'string') {
    return callback('error: prefix is invalid');
  }
  if (typeof suffix !== 'string') {
    return callback('error: suffix is invalid');
  }
  if (typeof padding !== 'string') {
    return callback('error: padding is invalid');
  }
  if (typeof start_count !== 'number') {
    return callback('error: start_count is invalid');
  }
  if (prefix.length > 0) {
    prefix += '_';
  }
  if (suffix.length > 0) {
    suffix = '_' + suffix;
  }
  var ls = child_process.spawn('ls', [source_dir]);
  var grep = child_process.spawn('grep', [".*\\" + extension]);

  var count = start_count;

  ls.stdout.pipe(grep.stdin);
  grep.stdout.pipe(split()).on('data', function (line) {
    var filename = line.toString();
    if (filename.length === 0) {
      return;
    }
    var new_filename = prefix + String(padding + count++).slice(-(padding.length+1)) + suffix + extension;
    fs.rename(path.join(source_dir, filename), path.join(destination_dir, new_filename), function (error) {
      if (error) {
        return callback('error: failed to rename ' + filename);
      }
    });
  });

  grep.on('close', function (code) {
    if (code !== 0) {
      return callback('error: grep exited with code ' + code);
    }
  });

}
Example #25
0
function createParseStream() {
  var stream
    , queued = {}
    , lineNumber = -1
    , head = false
    , raw = ''

  function handleHost(chunk) {
    if (!head) return head = true

    if (!chunk || !Object.keys(chunk).length) return raw = ''

    chunk.raw = raw
    chunk.finish = lineNumber - 1
    stream.queue(chunk)
    raw = ''
  };

  function handleRaw(chunk) {
    raw += chunk + '\n'
    return !stream.paused
  };

  function write(data) {
    lineNumber += 1

    if (!data) return handleRaw(data)
    if (!data.match(/\S/g)) return handleRaw(data)
    if (data.match(/^\s*?\#/g)) return handleRaw(data)

    var args = shell.parse(data)
      , keyword = args.shift()

    if (keyword === 'Host') {
      handleHost(queued)
      queued = {
        type: 'host',
        start: lineNumber,
        keywords: { Host: args }
      }
    }

    raw += data + '\n'

    queued.keywords[keyword] = args
  };

  function end() {
    lineNumber += 1
    handleHost(queued)
    stream.emit('end')
    process.nextTick(function() {
      stream.emit('close')
    })
  };

  return pipeline(split('\n'), stream = through(write, end))
};
Example #26
0
 mkdirp(destPath, function (err) {
     fs.createReadStream(file, { encoding: 'utf8' })
         .pipe(split())
         .pipe(nomnomlifier(destPath, prefix))
         .pipe(gfmifier())
         .pipe(fs.createWriteStream(destination, {
             encoding: 'utf8'
         }));
 });
Example #27
0
  connect: function() {
    var self = this;

    var c = net.connect(8081);
    c.write(JSON.stringify({hello: self.listenPeerId}) + '\n');
    c.pipe(split()).on('data', function() {});

    return c;
  },
Example #28
0
module.exports.talk = function(log, format, stream) {
  stream.pipe(split())
    .on('data', function(chunk) {
      var line = chunk.toString().trim()
      if (line.length) {
        log.info(format, line)
      }
    })
}
Example #29
0
function ProcessCommandList2(ProcCmdList, str, fncPass, fncFail) {
	var socket = net.createConnection({
		port: config.options.port,
		host: config.options.host
	});
	socket.on("connect", function() {
		console.log("Connected");
		OutputBuffer = [];
	});

	socket.setEncoding('utf8');
	//socket.setEncoding('hex');
	socket.pipe(split());

	socket.on("data", function(line) {
		var buf, b = line.split("\n");
		OutputBuffer = OutputBuffer.concat(b);
		/*
		 * If processing a command that expects data coming back
		 * And if we're looking for a particular string in that response
		 * Pass the string to look for and if found return true
		 * else return false and wait for prompt
		 */
		if (!look4Prompt(socket, b, ProcCmdList, fncPass, fncFail)) {
			console.log("Looking for string -----------------");
			look4String(socket, b, line, ProcCmdList, fncPass, fncFail);
		}

		if (util.isArray(OutputBuffer)) {
			// console.log("OutputBuffer is an array");
			buf = OutputBuffer.join("<br>");
		} else {
			// console.log("OutputBuffer is NOT an Array");
			buf = OutputBuffer;
		}
	});

	socket.on("error", function(err) {
		throw err;
	});

	socket.on('end', function() {
		console.log('Connection end');
	});
	socket.on('timeout', function() {
		console.log('Connection timeout');
	});
	socket.on('drain', function() {
		console.log('Connection drain');
	});
	socket.on('error', function() {
		console.log('Connection error');
	});
	socket.on('close', function() {
		console.log('Connection closed');
	});
}
Example #30
0
File: node.js Project: pgte/skiff
 channels.forEach(channel => {
   this._child[channel].pipe(split())
     .on('data', line => {
       line = line.trim()
       if (line) {
         process[channel].write(`${this._address} (${this._child.pid}): ${line}\n`)
       }
     })
 })