function doFix(next) {
var s = linestream();
var comment_regex = /^\w*#/;
var mt = new MultiTask(next);
var scantime = Date.now(); // TODO: grab from results.txt?
s.on('data', function (line) {
if (line.match(comment_regex)) {
return;
}
var split = line.split(' ');
if (split[0] === 'i') {
return;
}
if (split[0] === 'd') {
var disk_crc = split[1];
var source = split[3];
var filename = split.slice(4).join(' ');
if (source === 'filename' && filename.indexOf('_CORRUPT.') === -1 && filename.indexOf('_CRCMISMATCH.') === -1) {
console.log(clc.red('Cannot update CRC from filename for "' + filename + '" disk crc=' + disk_crc));
return;
}
console.log(clc.blue('Updating "' + filename + '" to ' + disk_crc));
mt.dispatch();
files.updateById(filename, { crc32: disk_crc, scantime: scantime }, function (err, doc) {
if (err) { throw err; }
console.log(clc.blueBright('Updated "' + filename + '" to ' + disk_crc));
mt.done();
});
}
});
s.on('end', function () {
mt.dispatchDone();
});
fs.createReadStream('results.txt').pipe(s);
}
http.createServer(function(req, resp) {
var parsed = url.parse(req.url, true)
, query = parsed.query
if(parsed.pathname !== '/') {
return resp.end('nope')
}
resp.writeHead(200, {
'Access-Control-Allow-Origin': '*'
, 'Access-Control-Allow-Methods': 'GET'
, 'content-type': 'application/json'
})
fs.createReadStream(example)
.pipe(ls())
.pipe(through(function(line) {
if(line.indexOf(query.q) > -1) {
this.queue(line)
}
}))
.pipe(json.stringify())
.pipe(resp)
}).listen(3001)
var fs = require('fs');
var zlib = require('zlib');
var linestream = require('line-stream')
var mem = require('../lib/mem.js')
var manager = require('../index.js')();
manager.add(__dirname+'/../test/workers/c.js',2);
var rs = fs.createReadStream(__dirname+'/access.log.gz');
var gz = rs.pipe(zlib.createGunzip());
var ls = gz.pipe(linestream())
ls.pipe(manager);
statsi = setInterval(function(){
console.log(manager.getStats());
console.log(mem());
},10000);
manager.on('end',function(){
console.log('job complete! ',Date.now()-manager.stats.start);
console.log(manager.getStats());
console.log(mem());
clearInterval(statsi);
});
return ser;
}
function openKeyStream(key, done) {
var keyPath = path.join.apply(path, key);
var outPath = path.join(outBase, keyPath);
createDerpStream(outPath, {flags: 'a'}, done);
}
var keyStreams = LRU({
max: 200,
dispose: function(key, stream) {stream.close();}
});
process.stdin
.pipe(linestream())
.pipe(through2.obj(function(line, enc, done) {
var self = this;
safeParse(line, under(done, function(record) {
if (record !== null && record !== undefined) {
self.push(record);
}
}));
}))
.pipe(SprayStream(function(record, done) {
var key = getKey(record);
var stream = keyStreams.get(key);
if (stream !== undefined) return done(null, stream);
openKeyStream(key, under(done, function(stream) {
stream = wrapStream(stream);
keyStreams.set(key, stream);