module.exports = function read(filename, format, done) {
var encoding = 'utf8';
switch (format) {
case 'yaml':
case 'yml':
return fs.readFile(filename, encoding, function(error, buffer) {
if (error) {
return done(error);
}
var parsed = yaml.safeLoad(buffer.toString());
return done(null, parsed);
});
case 'js':
if (!filename.match(/^[\.\/]/)) {
filename = './' + filename;
}
return done(null, require(filename));
}
var parse = tito.formats.createReadStream(format);
var data = [];
fs.createReadStream(filename)
.pipe(parse)
.on('data', function(d) { data.push(d); })
.on('end', function() {
done(null, data);
});
};
Tables.prototype.inputParser = function() {
var type = this.options.inputType;
var options = this.options.inputOptions;
// If the type is explicitly "custom", use custom pipe
if (type === "custom") {
this.output.item("Parser: custom");
return this.options.pipe;
}
this.output.item("Parser: " + this.options.inputType);
return tito.formats.createReadStream(type, options);
};
return function(filename, done) {
var parse = tito.formats.createReadStream(type, options);
var data = [];
return fs.createReadStream(filename)
.on('error', done)
.pipe(parse)
.on('error', done)
.on('data', function(d) {
data.push(d);
})
.on('end', function() {
done(null, data);
});
};
}, function(error, sets) {
if (error) return console.error('error:', error);
console.warn('got %d sets', sets.length);
sets[0].forEach(function(d) {
d.Total = d.Value;
d.Value = 0;
});
var rows = sets.reduce(function(list, set) {
return list.concat(set);
}, []);
var keys = ['Region', 'Year'];
var result = util.group(rows, keys, function(group) {
var total = 0;
var value = 0;
group.values.forEach(function(d) {
total += coerceNumber(d.Total);
value += coerceNumber(d.Value);
});
return {
Value: value,
Share: (value / total).toFixed(4),
Total: total
};
})
.map(function(entry) {
return extend(entry.key, entry.value);
})
.filter(function(d) {
return d.Value > 0;
});
console.warn('got %d rows', result.length);
console.warn(result[0]);
streamify(result)
.pipe(tito.createWriteStream(options.of))
.pipe(fs.createWriteStream(options.o || '/dev/stdout'))
.on('end', function() {
console.warn('wrote %d rows', result.length);
});
});
'of': {
desc: 'output format (tito-compatible)',
default: 'tsv'
},
o: {
desc: 'write output to this file',
default: '/dev/stdout'
}
});
var fs = require('fs');
var tito = require('tito').formats;
var thru = require('through2').obj;
fs.createReadStream(options._[0] || '/dev/stdin')
.pipe(tito.createReadStream(options.if))
.pipe(thru(mapRow))
.pipe(tito.createWriteStream(options.of))
.pipe(fs.createWriteStream(options.o));
function mapRow(d, enc, next) {
var out = {};
out.Category = d[''].trim();
if (!out.Category || out.Category.match(/Addenda/)) {
return next();
}
Object.keys(d).filter(isYear).forEach(function(year) {
out[year] = d[year];
});
return next(null, out);
}
var parser = function() {
return tito.createReadStream('json', {
path: '.BEAAPI.Results.Data.*'
});
};