MongoClient.connect(url, function(err, db) {
if (err) throw err;
var tabletop = Tabletop.init({
key: spreadsheet_key,
callback: spreadsheetLoaded,
simpleSheet: true
});
function spreadsheetLoaded(data) {
console.log("HERE", data);
var collection = db.collection('matrix');
var nfcmap = db.collection('nfcmap');
// remove all existing elements
collection.remove({}, function(err, removed){});
//nfcmap.remove({}, function(err, removed){});
console.log("REMOVED ALL");
// run through the elements, only deal with updates, no deletions
// entries are only valid if there is a title present
_.each(data, function(row, index) {
console.log(row, index);
collection.insert(row, {safe: true}, function(err, docs) {
if (err) throw err;
});
// for the nfc we insert the rowNumber
nfcmap.update({rowNumber: row.rowNumber}, {$set: {rowNumber: row.rowNumber}}, {upsert: true}, function(err, docs) {
if (err) throw err;
});
});
process.exit();
}
});
export default function(spreadsheetUrl, updateState){
let output = {};
tabletop.init({
key: spreadsheetUrl, // copy of live spreadsheet
callback: success
});
function success(data) {
formatData(data);
updateState(true, output.filters, output.contactList);
}
function formatData(data){
let
filterNames = [],
contactList = [];
for(var key in data){
let
sheet = data[key],
contacts = sheet.elements;
filterNames.push(sheet.name);
for(var row in contacts){
contactList.push(contacts[row]);
}
}
output.filters = filterNames;
output.contactList = contactList;
}
}
function init() {
Tabletop.init({
key: '0AkRR3zKqdlUHdDNRTHpoQ1E1S08waXVpV1RUWVVIb3c',
callback: parseResponse,
simpleSheet: true
});
}
function bakeProviders(spreadsheetId, callback) {
Tabletop.init({
key: spreadsheetId,
callback: function(data, tabletop) {
var err = null;
var warnings = null;
for (var i = 0; i < data.length; i++) {
d = data[i];
if (d.nearbus !== undefined) { d.nearbus = d.nearbus.toLowerCase() === 'true' ? true : false;}
if (d.nearbybuslines !==undefined) { d.nearbybuslines = splitValues(d.nearbybuslines);}
if (d.lowincome !==undefined) { d.lowincome = d.lowincome.toLowerCase() === 'true' ? true : false;}
if (d.completedculturalcompetencytraining !==undefined) { d.completedculturalcompetencytraining = d.completedculturalcompetencytraining.toLowerCase() === 'true' ? true : false;}
if (d.type !==undefined) { d.type = splitValues(d.type);}
if (d.specialties !==undefined) { d.specialties = splitValues(d.specialties);}
if (d.languages !==undefined) { d.languages = splitValues(d.languages);}
if (d.county !==undefined) { d.county = splitValues(d.county);}
if (d.orientation !==undefined) { d.orientation = splitValues(d.orientation);}
if (d.sexgenderidentity !==undefined) { d.sexgenderidentity = splitValues(d.sexgenderidentity);}
if (d.race !==undefined) { d.race = splitValues(d.race);}
if (d.affirmationstrainings !==undefined) { d.affirmationstrainings = splitValues(d.affirmationstrainings);}
}
callback(err, warnings, data);
},
simpleSheet: true
});
}
componentWillMount () {
Tabletop.init({
key: '1cWg1D5fmG-Y8IFCRm-2CWQt0UZixreH8OS4wu90M_A8',
orderby: 'querimpeachment',
reverse: true,
callback: (data, tabletop) => { this.props.actions.saveDeputados(data, tabletop); },
simpleSheet: true }
);
}
sheet.exists(function(notFound) {
if (notFound) {
plotErrorMessage(notFound);
return;
}
Tabletop.init({
key: sheet.id,
callback: createBlips
});
});
//Load data from google spreadsheet and write it to JSON files.
function fetchData(){
if (!offlineMode){
console.log('loading spreadsheet data.')
//var spreadsheet_URL = 'https://docs.google.com/spreadsheets/d/123DWrahipU6XOVjnVdTd0kdOBFBlzXuxButFymJ-OmA/pubhtml';//original
var spreadsheet_URL = '1oMDOK6jxU-um8T1SP3CwxGbNTDRxeuHSSuiU7owaY78';//revised
var myData;
function onLoad(data, tabletop) {
console.log("loading, updating and saving data from spreadsheet");
for (var i = 0; i<sections.length; i++){
//create JSON branch for each language
//global.overview[language] = global.overview.sitewide[i];
for (var j = 0; j<global.overview.sitewide.length; j++){
var language = global.overview.sitewide[j].language;
global[sections[i]][language] = global[sections[i]].sitewide[j];
}
}
//Write updated data to .JSON files and update global variables.
var currentNumber=0;
function writeJSON(){
if(currentNumber<sections.length){
//currently outputting these to a folder in the parent directory
//for integration with Jekyll template
var filename = '../../_data/' + sections[currentNumber] + '.json'
jf.writeFile(filename, data[sections[currentNumber]].elements, function(err) {
global[sections[currentNumber]].sitewide = readJSONFile(filename);
currentNumber++;
writeJSON();
})
} else {
process.exit();
}
}
writeJSON();
console.log("done");
};
var options = {
key: spreadsheet_URL,
callback: onLoad
};
Tabletop.init(options);
}
}
grunt.task.registerTask('fetch-data', 'Fetch data from Google spreadsheet.', function() {
var done = this.async();
console.log('Fetching Google Spreadsheet...');
var Tabletop = require('tabletop');
Tabletop.init( {
key: '0AjNAJ9Njg5YTdGtEZVdreHpBN3ZFOFJVVDdLUXhEcmc',
callback: _handleData,
simpleSheet: false
});
function _handleData(data) {
console.log('Saving Google Spreadsheet...');
grunt.file.write('src/js/app/models/fetchedData.js', 'define([],function() { return ' + JSON.stringify(data, null, " ") + '; });');
done();
}
});
// if online
function freshData(req, res) {
function tabletopCb(data, tabletop){
loadSheet(data, tabletop)
buildPage(req, res, data)
}
var options = {key: KEY, callback: tabletopCb, simpleSheet: true}
if (!sheetData.length || (Date.now() - lastFetch) > 6000000000000000000) {
console.log("you are online with old data, fetching new")
Tabletop.init(options)
}
else {
console.log("you are online with fresh data")
localData(req, res, buildPage)
}
}
async.forEach(sheets, function (item, callback){
function writeData(dataSet, tabletop) {
var obj = {
columns: tabletop.sheets(item.sheetName)['column_names'],
rows: tabletop.sheets(item.sheetName).toArray()
}
var file = 'src/api/'+item.api;
jsonfile.writeFileSync(file, obj)
console.log('Saved: '+item.name);
callback();
}
Tabletop.init( { key: item.url,
callback: writeData,
wanted: [item.sheetName],
simpleSheet: false } )
}, function(err) {
return function () {
if (typeof tabletop === 'undefined'){
tabletop = Tabletop.init({
key: args.key,
wanted:[args.sheet],
callback:storeData,
prettyColumnNames :false,
parseNumbers: false,
proxy: (args.isProxy && typeof args.bucket !== 'undefined')
? 'http://'+args.bucket+'.s3.amazonaws.com'
: undefined
})
window.Tabletop = window.Tabletop || Tabletop
} else {
tabletop.addWanted(args.sheet)
}
waitForData(callback)
}
vmscript.on(['Item'], function() {
console.log('Getting spreadsheet from Google Docs.');
// Item Translation
// https://docs.google.com/spreadsheets/d/1yhC0JLjJ68tkcFoXzC8lRla6bE7qcSGrX6FUxlPYKbA/edit#gid=894866128
// Publish URL:
// https://docs.google.com/spreadsheets/d/1yhC0JLjJ68tkcFoXzC8lRla6bE7qcSGrX6FUxlPYKbA/pubhtml?gid=894866128&single=true
Tabletop.init( { key: 'https://docs.google.com/spreadsheets/d/1yhC0JLjJ68tkcFoXzC8lRla6bE7qcSGrX6FUxlPYKbA/pubhtml?gid=894866128&single=true',
callback: function(data, tabletop) {
for (var i=0;i<data.length;i++) {
console.log('Updating: '+data[i].ID+' '+data[i].Name);
db.Item.update({ _id: data[i].ID }, {
Name: data[i].Name,
Description1: data[i].Description1,
Description2: data[i].Description2,
Description3: data[i].Description3
}).exec();
}
done(true);
},
simpleSheet: true });
});
vmscript.on(['SkillInfo'], function() {
console.log('Getting spreadsheet from Google Docs.');
// Item Translation
// https://docs.google.com/spreadsheets/d/1rL1HPMZi8Yxt8XPqAm15wkJc0Ak1hzqZB1yD864QO4w/edit#gid=1823029235
// Publish URL:
// https://docs.google.com/spreadsheets/d/1rL1HPMZi8Yxt8XPqAm15wkJc0Ak1hzqZB1yD864QO4w/pubhtml?gid=1823029235&single=true
Tabletop.init( { key: 'https://docs.google.com/spreadsheets/d/14hWHyB7KMg1V2uJVOx3zSp44OTZLy8085c63zSs3vME/pubhtml?gid=1602464661&single=true',
callback: function(data, tabletop) {
for (var i=0;i<data.length;i++) {
//console.log(data[i]);
//console.log('Updating: '+data[i].ID+' '+data[i].Name);
//console.log(data[i].Description1);
db.Skill.update({ ID: data[i].ID }, {
Name: data[i].Name,
Description1: data[i].Description1,
Description2: data[i].Description2,
Description3: data[i].Description3
}).exec();
}
done(true);
},
simpleSheet: true });
});
function getData(reply) {
tabletop = Tabletop.init({
key: KEY,
callback: function(data, tabletop) {
sheetData = data;
if(sheetData) {
if(sheetData.Texts){
texts = sheetData.Texts.elements;
console.log("TEXTS", texts);
}
if(sheetData.Team){
team = sheetData.Team.elements;
console.log("TEAM", team);
}
if(sheetData.Faq){
faq = sheetData.Faq.elements;
console.log("FAQ", faq);
}
if(sheetData.Partners){
partners = sheetData.Partners.elements;
console.log("PARTNERS", partners);
}
if(sheetData.Speakers){
speakers = sheetData.Speakers.elements;
console.log("SPEAKERS", speakers);
}
if(sheetData.Covers){
covers = sheetData.Covers.elements;
console.log("COVERS", covers);
}
}
},
simpleSheet: false
});
if(reply) reply("HELLLO!");
}
var Tabletop = require('tabletop');
var fs = require('fs');
var util = require('util');
var doc = process.argv[2];
if (!doc) {
console.error('Please supply a doc to transfer');
}
else {
Tabletop.init({
key: doc,
simpleSheet: true,
callback: function(data) {
fs.writeFile('./json/' + doc + '.json', JSON.stringify(data), 'utf8', function(e, d) {
if (e) {
throw(e);
}
console.log(util.format('Doc %s was transferred sucessfully', doc));
});
}
});
}
function init () {
var result = Tabletop.init({ key: '153rVXCt4nlQn8Fj80GmWOLO5_aF4phW0MZrNIZfpUc8',
callback: cb,
simpleSheet: true })
return result
}
function init() {
Tabletop.init( { key: '1O6BCyMFwOeQetLlk4WAjs04cEtoxwQvgRAjGNm-FEVo',
callback: saveData,
simpleSheet: true } )
}
grunt.registerMultiTask('preloadSpreadsheet', 'Update spreadsheet data', function () {
var Tabletop = require('tabletop');
var stringify = require('json-stable-stringify');
var _ = require('lodash');
var directory = 'app/scripts/spreadsheetpreload/';
var url = this.data;
var key = this.target;
var done = this.async();
Tabletop.init({
key: url,
parseNumbers: true,
debug: true,
callback: function (data) {
data = _.pick(data, ['unittypes', 'upgrades', 'achievements', 'tutorial']);
data = _.mapValues(data, function(sheet) {
return _.omit(sheet, ['raw']);
});
//var text = JSON.stringify(data, null, 2);
// built-in stringify puts sheets in a random order. Use a consistent
// order with json-stable-stringify for cleaner diffs.
var text = stringify(data, {space:' '});
text = '// This is an automatically generated file! Do not edit!\n// Edit the source at: '+url+'\n// Generated by Gruntfile.js:preloadSpreadsheet\n// key: '+key+'\n\'use strict\';\n\ntry {\n angular.module(\'swarmSpreadsheetPreload\');\n //console.log(\'second'+key+'\');\n}\ncatch (e) {\n // module not yet initialized by some other module, we\'re the first\n angular.module(\'swarmSpreadsheetPreload\', []);\n //console.log(\'first'+key+'\');\n}\nangular.module(\'swarmSpreadsheetPreload\').value(\'spreadsheetPreload-'+key+'\', '+text+');';
var filename = directory + key + '.js';
grunt.file.write(filename, text);
console.log('Wrote '+filename);
done();
}
});
});
request(domain, function(error, response, html) {
if (!error && response.statusCode == 200) {
var $ = cheerio.load(html);
$('ul').each(function(n, element) {
var bookFolder = $(this).attr('id');
//console.log('*********'+ bookFolder)
var imageFolder = bookFolder + "/images"
var stylesheetsFolder = bookFolder + "/stylesheets"
var fontsFolder = bookFolder + "/fonts"
var metaFolder = bookFolder + "/META-INF"
//Create book folder and subdirectories for /images, stylesheets etc ========================================
if (!fs.existsSync(bookFolder)) fs.mkdirSync(bookFolder);
if (!fs.existsSync(imageFolder)) fs.mkdirSync(imageFolder);
if (!fs.existsSync(stylesheetsFolder)) fs.mkdirSync(stylesheetsFolder);
if (!fs.existsSync(fontsFolder)) fs.mkdirSync(fontsFolder);
if (!fs.existsSync(metaFolder)) fs.mkdirSync(metaFolder);
//Add bookfolder to an array so that you can add files to it.
books.push(bookFolder);
});
//console.log(books);
//ADD STYLESHEETS================================================================================
console.log(' ')
var files_array = ['stylesheet.css', 'page_styles.css'];
var download = function(uri, filename, callback) {
request.head(uri, function(err, res, body) {
var r = request(uri).pipe(fs.createWriteStream(filename));
r.on('close', callback);
});
};
var counter = 0;
var fileNumber = 0;
function loop() {
if (counter < books.length && fileNumber < files_array.length) {
var source = domain + '/stylesheets/' + files_array[fileNumber];
var output = books[counter] + '/stylesheets/' + files_array[fileNumber];
//console.log('Saving ' + files_array[fileNumber] +' files in '+ books[counter]);
download(source, output, function() {
loop();
});
counter++;
} else if (fileNumber < files_array.length) {
counter = 0;
fileNumber++;
loop();
} else {
console.log(' ')
}
}
loop();
//Generalizing a function for downloading files.
function callLoop(arrayInput, folderInput, bookNumberChapter) {
var loop_array = arrayInput;
var folder = folderInput; //optional variable for subdirectories e.g. 'fonts/'
var book = 0;
var fileNumber = 0;
var inputFolder = '';
function looper() {
if (bookNumberChapter) {
folder = '';
inputFolder = 'book/' + book + '/';
//console.log('-----------FOLDER: ' + inputFolder);
}
if (book < books.length && fileNumber < loop_array.length) {
var source = domain + '/' + inputFolder + folder + loop_array[fileNumber] + "?mode=export";
//console.log('Downloading: '+ loop_array[fileNumber]);
//console.log('From: '+ source);
var output = books[book] + '/' + folder + loop_array[fileNumber];
//console.log('Saving ' + loop_array[fileNumber] +' in ' + books[book] + folder);
//console.log(' ');
download(source, output, function() {
looper();
});
book++;
} else if (fileNumber < loop_array.length) {
book = 0;
fileNumber++;
looper();
} else {
console.log('Done with looper().');
}
}
looper();
}
var fonts_array = config.fonts;
if (config.downloadFonts) {
callLoop(fonts_array, 'fonts/', false);
}
var mainFiles_array = ['backmatter.xhtml', 'bodymatter.xhtml', 'content.opf', 'cover.xhtml', 'introduction.xhtml', 'titlepage.xhtml', 'toc.xhtml', 'toc.ncx'];
callLoop(mainFiles_array, '', true);
var meta_array = ['container.xml'];
callLoop(meta_array, 'META-INF/', false);
//Manually creating mimetype file (instead of scraping) because scraping added a \n that broke validation.
function addMime() {
for (var s = 0; s < books.length; s++) {
var targetMime = "book" + s + '/mimetype'
fs.writeFile(targetMime, "application/epub+zip", function(err) {
if (err) {
return console.log(err);
}
});
}
}
addMime();
//LOADING PHOTOS BASED ON THE SPREADSHEET================================================================================
var Tabletop = require('tabletop');
var testURL = config.spreadsheet;
var myData;
function onLoad(data, tabletop) {
console.log("loading spreadsheet");
console.log('-----------------------');
myData = data.photos.elements;
//console.log(myData);
loopPhotos();
};
var options = {
key: testURL,
callback: onLoad
};
Tabletop.init(options);
var currentBook = 0;
//SET PHOTO NUMBER TO 2 IN ORDER TO SKIP THE INTRO PHOTOS LISTED AT TOP OF SPREADSHEET
var photo_number = 0;
if (config.subfolders) {
photo_number = 2;
}
//BEGIN DOWNLOADING PHOTOS ================================================================================
function loopPhotos() {
var subfolder = '';
if (config.subfolders) {
subfolder = 'v2_' + (currentBook + 1) + '/';
}
if (currentBook < (books.length + 1) && photo_number < myData.length) {
if (currentBook == myData[photo_number].book) {
//CREATE IMAGE SUBFOLDERS IF THEY DON'T EXIST
var imageSubfolder = books[currentBook - 1] + '/images/' + subfolder;
if (!fs.existsSync(imageSubfolder)) fs.mkdirSync(imageSubfolder);
var source = domain + '/images/' + subfolder + encodeURIComponent(myData[photo_number].filename);
var output = books[currentBook - 1] + '/images/' + subfolder + myData[photo_number].filename;
//DOWNLOAD CHAPTER PHOTOS INTO SUBFOLDERS
download(source, output, function() {
//console.log('Saving ' + myData[photo_number].filename + ' files in ' + imageSubfolder);
photo_number++;
loopPhotos();
});
} else {
currentBook++;
loopPhotos();
}
} else {
console.log('Looping through intro photos');
if (config.subfolders) {
loopIntroPhotos();
}
//DOWNLOAD INTRO PHOTOS TO SUBFOLDERS
function loopIntroPhotos() {
//COUNT DOWN BOOK NUMBER FROM PREVIOUS FUNCTION
currentBook--;
//RESET PHOTO NUMBER TO FIRST ROW IN SPREADSHEET
photo_number = 0;
//LOOP THROUGH FIRST TWO PHOTOS IN THE SPREADSHEET AND THROUGH THE SUBFOLDERS
for (photo_number; photo_number < 3; photo_number++) {
//
if (photo_number < 2 && currentBook > -1) {
var source = domain + '/images/' + subfolder + encodeURIComponent(myData[photo_number].filename);
var output = books[currentBook] + '/images/' + subfolder + myData[photo_number].filename;
console.log(output);
download(source, output, function() {
//console.log('Saved an intro photo...');
});
} else if (photo_number >= 2 && currentBook > -1) {
loopIntroPhotos();
}
};
};
console.log('Done saving photos');
}
}
}
});