function generateCountries() {
console.log('building files for countries');
var reader = csv.createCsvFileReader('countries.csv', {
separator: ',',
quote: '"',
'escape': '"',
comment: '',
columnsFromHeader: true
});
reader.addListener('data', function(data) {
if(data.name && data.name_forma && data.name_forma.indexOf('?') < 0 && data.name_forma !== '') {
var name = simplifyUtfToAscii(data.name);
var name2 = simplifyUtfToAscii(data.name_forma);
var pageName = createCountryPage(name, name2, data.type, data.sovereignt, data.abbrev, data.postal, data.json_4326);
addPageToSitemap(pageName);
pageName = createCountryPage(name2, name, data.type, data.sovereignt, data.abbrev, data.postal, data.json_4326);
addPageToSitemap(pageName);
} else
console.log('skipping ' + data.name);
});
reader.addListener('end', function() {
finishSitemap();
generateLandingPage();
});
}
path.exists(filename, function (exists) {
if (!exists) {
callback(true);
return false;
}
var reader = csv.createCsvFileReader(filename);
reader.addListener('data', function (row) {
if (!self.fields) {
self.fields = _.map(row, function (value) { return value.toLowerCase(); });
} else {
var item = {};
_.each(self.fields, function (key, index) {
item[key.toLowerCase()] = row[index];
});
self.lines += 1;
self.outputRow(item, function () {
self.linesWritten += 1;
});
}
});
reader.addListener('end', function () {
self.outputFooter(function () {
var results = "Transformed " + self.lines + " lines.\n";
self.out.end(function () {
callback(null, results);
});
});
});
});
app.post('/machine',function(req,res) {
var x = req.files.file.path;
var csv = require('ya-csv');
var reader = csv.createCsvFileReader(x,{'columnsFromHeader':true, 'separator':','});
var dataArray = [];
reader.addListener('data',function(data){
var nameMapping = {};
nameMapping["date"] = data["Date"];
nameMapping["radiation"] = parseFloat(data["Montreal Net Radiation - CWTA (W/m2)"]);
nameMapping["humidity"] = parseFloat(data["Montreal Relative Humidity - CWTA"]);
nameMapping["temperature"] = parseFloat(data["Montreal Temperature - CWTA (C)"]);
nameMapping["wind"] = parseFloat(data["Montreal Wind Speed - CWTA (km/h)"]);
nameMapping["power"] = parseFloat(data["Real Power Demand - Downtown Main Entrance (kW)"]);
dataArray.push(nameMapping);
// console.log(data);
});
reader.addListener('end',function(){
if(dataArray.length != 0){
var forecastedData = machineLearning.csvForecast(dataArray, 'csv');
server.csvClientData(forecastedData);
//We need to convert the JSON object into csv
var csvString = 'Date,Real Power Demand - Downtown Main Entrance (kW)\n';
for(var i=0; i<forecastedData.length; i++){
if(forecastedData[i].forecast!=undefined && forecastedData[i].forecast)
csvString += forecastedData[i].date + ',' + forecastedData[i].power+'\n';
}
res.send(csvString);
dataArray = [];
}
});
});
app.post('/confidence-post',function(req,res) {
var x = req.files.file.path;
var csv = require('ya-csv');
var reader = csv.createCsvFileReader(x,{'columnsFromHeader':true, 'separator':','});
var dataArray = [];
reader.addListener('data',function(data){
var nameMapping = {};
nameMapping["date"] = data["Date"];
nameMapping["radiation"] = parseFloat(data["Montreal Net Radiation - CWTA (W/m2)"]);
nameMapping["humidity"] = parseFloat(data["Montreal Relative Humidity - CWTA"]);
nameMapping["temperature"] = parseFloat(data["Montreal Temperature - CWTA (C)"]);
nameMapping["wind"] = parseFloat(data["Montreal Wind Speed - CWTA (km/h)"]);
nameMapping["power"] = parseFloat(data["Real Power Demand - Downtown Main Entrance (kW)"]);
dataArray.push(nameMapping);
//console.log(data);
});
reader.addListener('end',function(){
if(dataArray.length != 0){
var forecastedData = conLearning.csvForecast(dataArray, 'csv');
server.confidenceClientData(forecastedData);
res.send(forecastedData);
}
});
});
function loadExternalCountryData(fileName, callback) {
console.log('loading external country data file ' + fileName);
//load HDR data
var reader = csv.createCsvFileReader(fileName, {
separator: ',',
quote: '"',
'escape': '"',
comment: '',
columnsFromHeader: true
});
reader.addListener('data', function(data) {
var name = data.name.toLowerCase().trim();
var countryData = _externalCountryData[name];
if(!countryData) {
countryData = {};
_externalCountryData[name] = countryData;
}
for(var field in data)
if(field !== 'name' && data[field] !== '')
countryData[field] = data[field];
});
reader.addListener('end', function() {
callback();
});
}
collection.count(function(err,count){
if (count == 0){
var csvIn = csv.createCsvFileReader(file, {
'separator': ',',
'quote': '"',
'comment': '#',
});
var lines = 0;
csvIn.addListener('end', function() {
console.log("added " + lines + " cities to the Db");
db.close();
});
csvIn.addListener('data', function(data) {
lines++;
data.push(1);
collection.insert({
"className" : "entity.resources.City",
"departament" : data[0],
"city" : data[1],
"latitude" : data[2],
"longitude" : data[3]
},{w:0});
});
}else{
console.log("cities already initilized, skipping process");
db.close();
}
});
storeEmployeeData : function(path, callback) {
db.collection('employeedetails');
db.bind('employeedetails');
//clear exisiting data to avoid mulitple copies
db.employeedetails.remove();
//using ya-csv reader to parse csv files
var reader = csv.createCsvFileReader(path);
reader.setColumnNames([ 'employee_id', 'birthdate','firstname','lastname','sex','start_date' ]);
reader.addListener('data', function(data) {
var employeeData = {"birthdate" : data.birthdate, "firstname" : data.firstname,
"lastname" : data.lastname,"sex" : data.sex, "start_date" : data.start_date};
var employeedetails = {};
employeedetails[data.employee_id] = employeeData;
db.employeedetails.insert(employeedetails);
});
reader.addListener('end', function() {
db.employeedetails.find({}, {"_id" : false} ,function(err, cursor) {
if(err) {
callback(err);
} else {
cursor.toArray(callback);
} });
});
reader.addListener('error', function(error) {
callback({"error" : "error occured while parsing the employee file"});
});
},
function ender(callback){
var reader = csv.createCsvFileReader('raw.csv', {columnsFromHeader:true,'separator': ','});
reader.addListener('data', function(data){
var silo = [];
var flipper = false;
silo.push(data['Asset ID'].split('|'));
loop1:
for(i=0;i<silo.length;i++){
loop2:
for(j=0;j<silo[i].length;j++){
console.log("ender "+silo[i][j]);
loop3:
for(k in combined){
if(silo[i][j] === combined[k]){
console.log("MATCH!")
break loop1;
}
}
}
flipper = true;
break loop1;
}
if(flipper){
console.log("WRITING ENDER");
fs.appendFileSync('newRaw.csv', data['Video ID']+","+data['Content Type']+","+data['Policy']+","+data['Video Title']+","+data['Video Duration (sec)']+","+data['Username']+","+data['Uploader']+","+data['Channel Display Name']+","+data['Channel ID']+","+data['Claim Type']+","+data['Claim Origin']+","+data['Total Views']+","+data['Watch Page Views']+","+data['Embedded Player Views']+","+data['Channel Page Video Views']+","+data['Live Views']+","+data['Recorded Views']+","+data['Ad-Enabled Views']+","+data['Total Earnings']+","+data['Gross YouTube-sold Revenue']+","+data['Gross Partner-sold Revenue']+","+data['Gross AdSense-sold Revenue']+","+data['Estimated RPM']+","+data['Net YouTube-sold Revenue']+","+data['Net AdSense-sold Revenue']+","+data['Multiple Claims?']+","+data['Category']+","+data['Asset ID']+","+data['Channel']+","+data['Custom ID']+","+data['ISRC']+","+data['GRid']+","+data['UPC']+","+data['Artist']+","+data['Asset Title']+","+data['Album']+","+data['Label']+","+""+"\n");
}
})
reader.addListener('end', function(){
callback();
})
}
fs.writeFile('settings.csv','datafolder,'+path+'\noptions,'+options+'\ndays,'+days+'\nformat,'+format+'\nevent,'+event+'\nyahoohr,'+yahoohr+'\nyahoomin,'+yahoomin+'\ngooglehr,'+googlehr+'\ngooglemin,'+googlemin+'\n' , function (err) {
if(err){document.getElementById("om").innerHTML+="<br/>"+err;}
var opPath= "";
var reader = csv.createCsvFileReader('settings.csv',{'separator': ','});
reader.addListener('data', function(data)
{
if(data[0]=='datafolder')
{
opPath = data[1];
}
if(data[0]=='event')
{
if(data[1].search('Delete_All_Events')>=0)
{
// delete events file
if(fs.existsSync(opPath+'/events/CA_ALL_FORTHCOMING.csv'))
{
fs.unlinkSync(opPath+'/events/CA_ALL_FORTHCOMING.csv');
}
if(fs.existsSync(opPath+'/events/BM_All_Forthcoming.csv'))
{
fs.unlinkSync(opPath+'/events/BM_All_Forthcoming.csv');
}
}
}
});
});
function pusher(callback){
var k = 0;
var reader = csv.createCsvFileReader('raw.csv', {columnsFromHeader:true,'separator': ','});
reader.addListener('data', function(data){
var silo = [];
var foo = false;
var a = setTimeout(function(){
silo.push(data['Asset ID'].split('|'));
loop1:
for(i in silo){
if(silo[i] != null){
//console.log(silo[i]);
loop2:
for(j=0;j<silo[i].length;j++){
var b = function(){
//console.log("processing asset "+silo[i][j])
db.collection('assets').findOne({assetId:silo[i][j]}, function(err,doc){
if(err){
console.log(err);
return true;
}
if(doc != null){
if(doc.notes === "TUNECORE"){
console.log("TuneCore Video ID "+data["Video ID"]);
return true;
}
else if(doc.notes === "AUDIOSOCKET"){
console.log("AudioSocket Video ID "+data["Video ID"]);
return true;
}
else{
console.log("not tc or as "+silo[i][j]);
return false;
}
}
else{
console.log("NOT IN DB "+silo[i][j]);
return false;
}
})
}
b();
if(b = true){
console.log("broken");
break loop1;
}
}
}
}
}, k+=100);
})
reader.addListener('end', function(){
console.log("end of pusher")
callback();
})
}
function tunecore(callback){
var reader = csv.createCsvFileReader('raw.csv', {columnsFromHeader:true,'separator': ','});
//var tFile = csv.createCsvFileReader('tunecore.csv', {columnsFromHeader:false,'separator': ','});
reader.addListener('data', function(data){
var silo = [];
silo.push(data['Asset ID'].split('|'));
loop1:
for(i=0;i<silo.length;i++){
loop2:
for(j=0;j<silo[i].length;j++){
console.log("tunecore "+silo[i][j]);
loop3:
for(k in tSilo){
//console.log(tSilo[k]+" vs "+silo[i][j]);
if(silo[i][j] === tSilo[k]){
console.log("found a match for TUNECORE");
fs.appendFileSync('newRaw.csv', data['Video ID']+","+data['Content Type']+","+data['Policy']+","+data['Video Title']+","+data['Video Duration (sec)']+","+data['Username']+","+data['Uploader']+","+data['Channel Display Name']+","+data['Channel ID']+","+data['Claim Type']+","+data['Claim Origin']+","+data['Total Views']+","+data['Watch Page Views']+","+data['Embedded Player Views']+","+data['Channel Page Video Views']+","+data['Live Views']+","+data['Recorded Views']+","+data['Ad-Enabled Views']+","+data['Total Earnings']+","+data['Gross YouTube-sold Revenue']+","+data['Gross Partner-sold Revenue']+","+data['Gross AdSense-sold Revenue']+","+data['Estimated RPM']+","+data['Net YouTube-sold Revenue']+","+data['Net AdSense-sold Revenue']+","+data['Multiple Claims?']+","+data['Category']+","+data['Asset ID']+","+data['Channel']+","+data['Custom ID']+","+data['ISRC']+","+data['GRid']+","+data['UPC']+","+data['Artist']+","+data['Asset Title']+","+data['Album']+","+data['Label']+","+"TUNECORE"+"\n");
break loop1;
}
}
/*
//var stream = db.collection('assets').find({"assetId":silo[i][j]}).stream();
db.collection('assets').find({"assetId":silo[i][j]}), function(err, item){
//console.log(err);
//stream.on("data", function(item){
console.log(item);
if(item.notes != null){
console.log("I JUST WROTE A LINE WITH NOTES");
fs.appendFileSync('newRaw.csv', data['Video ID']+","+data['Content Type']+","+data['Policy']+","+data['Video Title']+","+data['Video Duration (sec)']+","+data['Username']+","+data['Uploader']+","+data['Channel Display Name']+","+data['Channel ID']+","+data['Claim Type']+","+data['Claim Origin']+","+data['Total Views']+","+data['Watch Page Views']+","+data['Embedded Player Views']+","+data['Channel Page Video Views']+","+data['Live Views']+","+data['Recorded Views']+","+data['Ad-Enabled Views']+","+data['Total Earnings']+","+data['Gross YouTube-sold Revenue']+","+data['Gross Partner-sold Revenue']+","+data['Gross AdSense-sold Revenue']+","+data['Estimated RPM']+","+data['Net YouTube-sold Revenue']+","+data['Net AdSense-sold Revenue']+","+data['Multiple Claims?']+","+data['Category']+","+data['Asset ID']+","+data['Channel']+","+data['Custom ID']+","+data['ISRC']+","+data['GRid']+","+data['UPC']+","+data['Artist']+","+data['Asset Title']+","+data['Album']+","+data['Label']+","+item.notes+"\n")
}
else if(item.policy === 'TuneCore Default Policy'){
console.log("TUNECORE MATCH POLICY");
fs.appendFileSync('newRaw.csv', data['Video ID']+","+data['Content Type']+","+data['Policy']+","+data['Video Title']+","+data['Video Duration (sec)']+","+data['Username']+","+data['Uploader']+","+data['Channel Display Name']+","+data['Channel ID']+","+data['Claim Type']+","+data['Claim Origin']+","+data['Total Views']+","+data['Watch Page Views']+","+data['Embedded Player Views']+","+data['Channel Page Video Views']+","+data['Live Views']+","+data['Recorded Views']+","+data['Ad-Enabled Views']+","+data['Total Earnings']+","+data['Gross YouTube-sold Revenue']+","+data['Gross Partner-sold Revenue']+","+data['Gross AdSense-sold Revenue']+","+data['Estimated RPM']+","+data['Net YouTube-sold Revenue']+","+data['Net AdSense-sold Revenue']+","+data['Multiple Claims?']+","+data['Category']+","+data['Asset ID']+","+data['Channel']+","+data['Custom ID']+","+data['ISRC']+","+data['GRid']+","+data['UPC']+","+data['Artist']+","+data['Asset Title']+","+data['Album']+","+data['Label']+","+"TUNECORE"+"\n");
}
else{
console.log(item.assetId+" HAS NO NOTES");
fs.appendFileSync('newRaw.csv', data['Video ID']+","+data['Content Type']+","+data['Policy']+","+data['Video Title']+","+data['Video Duration (sec)']+","+data['Username']+","+data['Uploader']+","+data['Channel Display Name']+","+data['Channel ID']+","+data['Claim Type']+","+data['Claim Origin']+","+data['Total Views']+","+data['Watch Page Views']+","+data['Embedded Player Views']+","+data['Channel Page Video Views']+","+data['Live Views']+","+data['Recorded Views']+","+data['Ad-Enabled Views']+","+data['Total Earnings']+","+data['Gross YouTube-sold Revenue']+","+data['Gross Partner-sold Revenue']+","+data['Gross AdSense-sold Revenue']+","+data['Estimated RPM']+","+data['Net YouTube-sold Revenue']+","+data['Net AdSense-sold Revenue']+","+data['Multiple Claims?']+","+data['Category']+","+data['Asset ID']+","+data['Channel']+","+data['Custom ID']+","+data['ISRC']+","+data['GRid']+","+data['UPC']+","+data['Artist']+","+data['Asset Title']+","+data['Album']+","+data['Label']+","+"\n");
}
}
}
}
})
reader.addListener('end', function(data){
callback();
}) */
}
}
})
reader.addListener('end', function(){
callback();
})
}
function init(){
console.log("Loading mock data set");
var isFirstRow = true;
var records = 0;
var reader = csv.createCsvFileReader('server/components/mock/Random-Points-250k.csv');
reader.addListener('data', function(data) {
var i;
if (isFirstRow){
isFirstRow = !isFirstRow;
for (i = 0; i < data.length; i++) {
indexToKeyName[i] = data[i];
}
} else {
var obj = {};
// Convert data to GeoJSON
obj["type"] = "Feature";
obj["geometry"] = {
"type": "Point",
"coordinates": []
};
obj["properties"] = {
"classification": "U"
};
for (i = 0; i < data.length; i++) {
// why am i doing this pls halp
if(indexToKeyName[i] === "count"){
obj.properties[indexToKeyName[i]] = parseInt(data[i]);
} else if (indexToKeyName[i] === "lat" ||
indexToKeyName[i] === "lon" ||
indexToKeyName[i] === "percent"){
obj.properties[indexToKeyName[i]] = parseFloat(data[i]);
} else {
obj.properties[indexToKeyName[i]] = data[i];
}
}
obj.properties.featureId = uuid.v4();
// Coordinates are lon,lat per GeoJSON spec
obj.geometry.coordinates = [parseFloat(data[LON_INDEX]), parseFloat(data[LAT_INDEX])];
db[obj.properties.featureId] = obj;
records += 1;
}
});
reader.addListener('end', function(){
console.log("Mock set loaded with " + records + " records");
});
}
function init(){
console.log('Loading mock data set');
var isFirstRow = true,
records = 0,
reader = csv.createCsvFileReader('server/components/mock/Random-Points-250k.csv');
reader.addListener('data', function(data) {
var i,
obj;
if (isFirstRow){
isFirstRow = !isFirstRow;
for (i = 0; i < data.length; i++) {
indexToKeyName[i] = data[i];
}
} else {
obj = {};
// Convert data to GeoJSON
obj['type'] = 'Feature';
obj['geometry'] = {
type: 'Point',
coordinates: []
};
obj['properties'] = {
classification: 'U'
};
for (i = 0; i < data.length; i++) {
// why am i doing this pls halp
if(indexToKeyName[i] === 'count'){
obj.properties[indexToKeyName[i]] = parseInt(data[i]);
} else if (indexToKeyName[i] === 'lat' ||
indexToKeyName[i] === 'lon' ||
indexToKeyName[i] === 'percent'){
obj.properties[indexToKeyName[i]] = parseFloat(data[i]);
} else {
obj.properties[indexToKeyName[i]] = data[i];
}
}
obj.properties.featureId = generateUUID();
// Coordinates are lon,lat per GeoJSON spec
obj.geometry.coordinates = [parseFloat(data[LON_INDEX]), parseFloat(data[LAT_INDEX])];
db[obj.properties.featureId] = obj;
records += 1;
}
});
reader.addListener('end', function(){
console.log('Mock set loaded with ' + records + ' records');
});
}
app.get("/score", function(request, response) {
var reader = csv.createCsvFileReader("scores.csv");
reader.setColumnNames(['name', 'email', 'score']);
var scores = [];
reader.addListener('data', function(data) {
scores.push(data);
});
reader.addListener('end', function(){
response.send(scores);
})
});
function pusher3(callback){
var reader = csv.createCsvFileReader('audiosocket2.csv', {columnsFromHeader:false,'separator': ','});
reader.addListener('data', function(data){
console.log(data[0]);
aSilo.push(data[0]);
combined.push(data[0]);
})
reader.addListener('end', function(){
callback();
})
}
function generate_month_entry(file) {
if (path.extname(file) != '.csv') {
return;
}
var filename = path.basename(file);
var datestring = filename.substr(0, filename.lastIndexOf('.'));
moment.locale('ro');
var d = new Date(datestring);
var year = moment(d).format('YYYY');
var month = moment(d).format('MM');
var monthname = moment(d).format('MMMM');
console.log(monthname);
monthEntries = {total:{debit:0, credit:0}};
i = 0;
// Read the entries for a month.
if (fs.existsSync(file)) {
var reader = csv.createCsvFileReader(file);
var entry = {date:'', details:'', debit:0, credit:0};
header = ['Details', 'Credit', 'Debit'];
reader.addListener('data', function(data) {
var date = data[0];
var debit = (typeof data[2] != 'undefined') ? parseFloat(data[2].replace('.','').replace(',','.')): data[2];
var credit = (typeof data[3] != 'undefined') ? parseFloat(data[3].replace('.','').replace(',','.')): data[3];
if (date.length && i > 0) {
if (typeof transaction != 'undefined') {
generate_transaction_entry(transaction);
}
transaction = {
day: parseInt(date.substring(0, 2)),
date: date,
details: data[1],
debit: (isNaN(debit) ? 0: debit),
credit: (isNaN(credit) ? 0: credit),
};
}
else if (typeof transaction != 'undefined') {
transaction.details += "<br>" + data[1];
}
i++;
});
}
else {
error = {message:"No file found"};
}
}
exports.searchName = function(req, res){
var searchName = req.query.filename;
if (searchName.split("imagesDB/")[1])
searchName=searchName.split("imagesDB/")[1];
var reader = csv.createCsvFileReader('../faces.csv');
reader.addListener('data', function(data) {
if(data[0])
{ var fileName=data[0].split("imagesDB/");
fileName=fileName[1];
if(fileName == searchName)
res.send(data[1]);
}
});
}
function parseCsv() {
var rows = [];
var reader = csv.createCsvFileReader(csvFileName, { columnsFromHeader: true });
reader.addListener('data', function(data) {
rows.push(data);
});
reader.addListener('end', function(data) {
console.log(rows.length.toString() + ' row(s) extracted from file.');
//continue to next step
proc.next(rows);
});
}
app.post('/upload', function(req, res){
//io.sockets.on('anything',function(socket){
//console.log(req.sessionStore);
//var socket = req.sessionStore.Adrian;
var uploadRead = csv.createCsvFileReader(req.files.csv.path, {columnsFromHeader:false,'separator': ','});
var foo = [];
uploadRead.addListener('data', function(data){
//console.log(data['Asset ID']);
//push an objects to foo
//foo.push({'videoId':data['Video ID']},{'assetId': data['Asset ID']})
//foo.push({'videoId': data['Video ID'], 'assetId': data['Asset ID']});
foo.push(data[0]);
console.log(data[0]);
//console.log(data['Asset ID']);
//return(foo)
//console.log(foo)
})
uploadRead.addListener('end', function(){
/*for(i=0;i<foo.length;i++){
io.sockets.emit('query', {assetId:foo[i]});
if(i = foo.length){
//res.redirect('/cms');
}
}*/
async.series([
function(callback){
var k = 0;
res.redirect('/cms');
var a = setTimeout(function(){
callback();
},k+=5000)
},
function(callback){
pre(callback);
},
function(callback){
emiter(foo,req);
callback();
}
])
//res.redirect('/cms');
//emiter(foo);
})
//})
})
tasks.push( function(callback){
var temp = [],
reader = csv.createCsvFileReader(src, {
// dirty fix, with default quote " gives error, I do not know why
quote:'$',
columnsFromHeader: true
});
reader.addListener('data', function(data) {
temp.push(data);
});
reader.addListener('end', function() {
fs.appendFileSync(dest, JSON.stringify(temp, null, '\t'));
callback(null, true);
});
});
var readCSV = function(path, out, finished) {
var reader = csv.createCsvFileReader(path, {
'separator': '\t'
// 'quote': '"',
// 'escape': '"',
// 'comment': '',
});
reader.addListener('data', function(data) {
out(getEntry(data));
});
if (finished) {
reader.addListener('end', function() {
finished();
});
}
}
DB.prototype.load = function (csvfile) {
var that = this;
var items = [];
var reader = csv.createCsvFileReader(csvfile, {
'separator': ',',
'quote': '"',
'escapechar': '"',
'comment': '#',
});
reader.on('data', function(line) {
var current = {};
current.start = parseInt(line[0]);
current.end = parseInt(line[1]);
current.code = line[4];
current.country = line[6];
items.push(current);
});
reader.on('error', function (err) {
var error = new(Error)('csv parsing error, did you specify the database correctly?');
var req;
while (req = that.queue.pop()) {
req.callback(error);
}
that.emit('ready', error);
});
reader.on('end', function () {
that.ready = true;
that.items = items;
that.emit('ready', null, that);
var req;
while (req = that.queue.pop()) {
that.lookup(req.ip, req.callback);
}
});
};
function cvs2json(filename){
filename =filename || 'data';
var csvIn = csv.createCsvFileReader('./'+filename+'.csv', {
'separator': ',',
'quote': '"',
'comment': '#',
columnsFromHeader:true
}),csvArray = [];
console.log('start to read '+filename+'.cvs ...');
csvIn.addListener('data',function(data){
csvArray.push(data);
});
csvIn.addListener('end',function(){
console.log('read '+filename+'.cvs over!');
fs.writeFileSync('./'+filename+'.js', 'var data='+JSON.stringify(csvArray) );
});
}
handler.upload = function (req, res) {
console.log(req.files.fileupload.path);
console.log(req.files.fileupload.size);
//Read and parse csv data
var reader = yacsv.createCsvFileReader(req.files.fileupload.path, {
'separator': ',',
'quote': '"',
'escape': '"',
'comment': 'H'
});
reader.setColumnNames([ 'measureDate','temperature','humidity','interval','transfertTime','remark' ]);
//Insertion des data
reader.addListener('data', function(data) {
insertTempoData(data);
});
//Redirection vers accueil
res.redirect('/');
};
function loadStaticBlurbs(callback) {
console.log('loading static blurbs');
var reader = csv.createCsvFileReader('static-blurbs.csv', {
separator: ',',
quote: '"',
'escape': '"',
comment: '',
columnsFromHeader: true
});
reader.addListener('data', function(data) {
_staticBlurbs[data.name.toLowerCase()] = data.blurb;
});
reader.addListener('end', function() {
callback();
});
}
function processFile(file) {
var reader = csv.createCsvFileReader(file, { separator: ';', quote: '"', 'escape': '"', columnsFromHeader: true });
reader.on('error', function (err) {
console.error('file: ' + file + ' ' + JSON.stringify(err));
});
reader.on('data', function (record) {
// Rekeningnummer;Rubrieknaam;Naam;Munt;Afschriftnummer;Datum;Omschrijving;Valuta;Bedrag;Saldo;
var trans = new Transaction({
accountNumber: record.Rekeningnummer,
accountName: record.Naam,
currency: record.Munt,
date: parseDate(record.Datum),
valueDate: parseDate(record.Valuta),
message: record.Omschrijving,
amount: parseFloat(record.Bedrag.replace(',', '.')),
balance: parseFloat(record.Saldo.replace(',', '.'))
});
for (var i = 0; i < regex.length; i++) {
var r = regex[i];
var match = record.Omschrijving.match(r.regex);
if (match) {
trans.info = r.parse(match);
break;
}
}
trans.save(function (err) {
if (err) {
console.error(err);
}
});
});
reader.on('end', function (file) {
console.log('done processing: ' + file);
});
}
function generateStates() {
console.log('generating states and provinces');
var reader = csv.createCsvFileReader('states-and-provinces.csv', {
separator: ',',
quote: '"',
'escape': '"',
comment: '',
columnsFromHeader: true
});
reader.addListener('data', function(data) {
//data.alias
var names = [simplifyUtfToAscii(data.name)];
if(data.alias.trim() !== '') {
var aliases = data.alias.split(/[|,]/);
for(var i in aliases)
names.push(simplifyUtfToAscii(aliases[i]));
}
for(var i in names) {
var name = names[i].trim();
//skip place names that have unknown characters
if(name.indexOf('?') >= 0 || name === '')
continue;
var pageName = createProvincePage(name, data.engtype_1, data.country, data.region, data.area_sqkm, data.abbrev, data.postal, data.json, i > 0);
addPageToSitemap(pageName);
//don't want to add aliases
if(name === data.name)
addLinkToLandingPage(name, pageName, data.country);
}
});
reader.addListener('end', function() {
generateCountries();
});
}
app.post('/api/weather', multipartyMiddleware, function (req, res) {
var dataset_name = req.body.name;
var dataset_csv = req.files.file;
var reader = csv.createCsvFileReader(dataset_csv.path, {
'separator': ',',
'quote': '',
'escape': '',
'comment': ''
});
var dataset = Dataset({
_id: mongoose.Types.ObjectId(),
name: dataset_name
});
dataset.save(function (err) {
if (err) throw err;
console.log("Dataset created with object id: " + dataset._id)
});
reader.addListener('data', function (data) {
var weather = Weather({
_id: mongoose.Types.ObjectId(),
dataset_id: dataset._id,
datetime: new Date(data[0]),
pressure: {data:parseFloat(data[1])},
rainfall: {data:parseFloat(data[2])},
wind: {data:parseFloat(data[3])},
wind_direction: {data:parseFloat(data[4])},
surface_temperature: {data:parseFloat(data[5])},
relative_humidity: {data:parseFloat(data[6])},
solar_flux: {data:parseFloat(data[7])},
battery: {data:parseFloat(data[8])}
});
weather.save(function (err) {
if (err) throw err;
console.log("Weather save for dataset: " + weather.dataset_id);
})
});
res.json({dataset_id: dataset._id});
});
var parseCsv = function parseCsv(cb) {
var cvsOptions = {};
var ops = [];
if (!options.cols) {
cvsOptions.columnsFromHeader = true;
}
var reader = csv.createCsvFileReader(csvFile, cvsOptions);
if (options.cols) {
reader.setColumnNames(options.cols);
}
reader.addListener('data', function(data) {
var keys = getKeys(options.keys, reader.columnNames, data);
keys.forEach(function onKey(key) {
ops.push({
type: 'put',
key: key,
value: data,
valueEncoding: 'json',
keyEncoding: 'json'
});
if (ops.length === 100) {
executeBatch(ops.slice(0));
ops = [];
}
});
});
reader.addListener('end', function() {
executeBatch(ops, function(err) {
if (err) {
cb(err);
} else {
cb(reader.columnNames);
}
});
});
};
storeSalaryData : function(path, callback) {
db.collection('salarydetails');
db.bind('salarydetails');
//clear exisiting data to avoid mulitple copies
db.salarydetails.remove();
//using ya-csv reader to parse csv files
var reader = csv.createCsvFileReader(path);
reader.setColumnNames([ 'employee_id', 'salary','start_salary','end_salary' ]);
reader.addListener('data', function(data) {
var salaryData = {"empid" : data.employee_id, "salary" : data.salary, "start_salary" : data.start_salary, "end_salary" : data.end_salary};
db.salarydetails.insert(salaryData);
});
reader.addListener('end', function() {
callback(null, {"msg" : "Success"});
});
reader.addListener('error', function(error) {
callback({"error" : "error occured while parsing the salary file"});
});
},