var dropDatabase = function (testDbName) {
var db = mongo(testDbName);
return db.dropDatabase()
.then(function () {
db.close();
});
};
module.exports = function () {
debug('retrieving scrape history')
// mongo defined in hosts file by docker when linking mongo container
var db = pmongo('mongodb://mongo:27017/youtube-comment-scraper', ['scrapes'])
return db.scrapes.find().sort({timestamp: -1}).toArray()
}
var loadFixture = function (testDbName, testCollectionName, data) {
var db = mongo(testDbName);
return db.createCollection(testCollectionName)
.then(function (collection) {
return Q.all(data.map(function (dataItem) {
return saveDocument(collection, dataItem);
}));
})
.then(function () {
db.close();
});
};
module.exports = function(config) {
var coll = pmongo(config.get('connexionURI')).collection(config.get('collectionName'));
return datamodel()
.declare('template', function(req, fill) {
fill(basename + '.html');
})
.declare('headers', function(req, fill) {
var headers = {};
headers['Content-Type'] = require('../helpers/format.js')(req.params.format);
if (req.params.format === 'zip') {
headers['Content-Disposition'] = 'attachment; filename="export.zip"';
}
fill(headers);
})
.declare('site', function(req, fill) {
fill({
title : config.get('title'),
description : config.get('description')
});
})
.declare('page', function(req, fill) {
fill({
title : 'Display doc',
description : null,
types : ['text/html', 'application/json', 'application/zip']
});
})
.declare('user', function(req, fill) {
fill(req.user ? req.user : {});
})
.declare('config', function(req, fill) {
fill(config.get());
})
.declare('url', function(req, fill) {
fill(require('url').parse(req.protocol + '://' + req.get('host') + req.originalUrl));
})
.declare('selector', function(req, fill) {
fill({ wid : req.params.doc});
})
.append('item', function(req, fill) {
coll.findOne(this.selector).then(fill).catch(fill);
})
.send(function(res, next) {
res.set(this.headers);
debug('locals', this);
render(res, this, next);
})
.takeout();
}
function LocalPeople() {
var db = pmongo('history', ['people']);
var descriptor = 'local';
function getPersonById(id) {
return db.people.findOne({ id: id }).then(function(rawPerson) {
return buildWithTransitions(rawPerson);
});
}
function buildWithTransitions(rawPerson) {
var transitions = history.getCurrentTransitions(rawPerson.xid);
return factory.build(rawPerson, descriptor, transitions);
}
function findByQuery(query) {
var pattern = new RegExp(query);
var asId = parseInt(query.replace(/l(ocal)?-/, ''));
return db.people.find({ $or: [
{ email: pattern },
{ id: asId },
]}).toArray();
}
this.create = function(email) {
return db.people.count().then(function(count) {
var nextId = count + 1;
var xid = descriptor + '-' + nextId;
var now = (new Date()).toISOString();
return db.people.insert({ email: email, when: now, id: nextId, xid: xid }).then(function() { return nextId; });
});
};
this.get = function(id) {
return getPersonById(id).then(getPayableTicket).then(getIssuableCertificate);
};
this.search = function(query) {
return findByQuery(query).then(function(results) {
var promises = _.chain(results)
.map(buildWithTransitions)
.map(Q.promised(getPayableTicket))
.map(Q.promised(getIssuableCertificate))
.value();
return Q.all(promises);
});
};
this.listing = function() {
return Q.when({});
};
}
module.exports = function(config) {
var coll = pmongo(config.get('connexionURI')).collection(config.get('collectionName'));
return function (req, res, next) {
var id = req.params.doc
, schema = { "xml" : {
"transform" : {
"type" : "text",
"required" : false
}
},
"json" : {
}
}
;
if (!schema[req.params.format]) {
return res.status(400).send('Bad Request').end();
}
var form = require('formatik').parse(req.query || {}, schema[req.params.format], 'fr');
if (form.isValid()) {
var data = {}
, fields = form.mget('value')
, selector = { wid : req.params.doc };
coll.findOne(selector)
.then(function(o) {
res.set('Content-Type', require('../helpers/format.js')(req.params.format));
res.status(200);
res.send(o.content[req.params.format]);
res.end();
})
.catch(function(e) {
res.status(500).send('Internal Server Error').end();
});
}
else {
res.status(400).send('Bad Request').end();
}
};
};
module.exports = function(config) {
var coll = pmongo(config.get('connexionURI')).collection(config.get('collectionName'));
return function (req, res, next) {
var form = require('formatik').parse(req.body, {
"key" : {
"type" : "text",
"required" : true,
"pattern" : "[a-z][a-z0-9. _-]+"
},
"val" : {
"type" : "text",
"required" : true
}
}, 'fr');
if (form.isValid()) {
var data = {}
, fields = form.mget('value')
, selector = { wid : req.params.doc };
data[fields.key] = fields.val;
debug('save', selector, data);
coll.update(selector, { $set : data })
.then(function() {
res.status(200).send('OK').end();
})
.catch(function() {
res.status(500).send('Internal Server Error').end();
});
res.status(200).send('OK').end();
}
else {
res.status(400).send('Bad Request').end();
}
};
};
var Promise = require('bluebird');
var request = Promise.promisifyAll(require('request'));
var _ = require('underscore');
var pmongo = require('promised-mongo');
var db = pmongo('mongodb://localhost:27017/feedme', ["facebook", "facebookTokens"]);
var googleApiKey = process.env.GOOGLEAPIKEY || "123FAKEKEY";
var radius = "7000"; //in meters
var targetAddress = "San Francisco";
var tokens = [];
var insertCount = 0;
var locationNames = [];
var eventIds = [];
var events = [];
//need to manually terminate program when scraping complete, since db connection is always open
var terminateProgram = function(){
console.log("Program finished,", insertCount, "entries added / updated");
process.exit(1);
};
setInterval(function(){
db.runCommand({ping:1})
.then(function(res) {
if(res.ok){
console.log("We're still up!");
}
});
}, 3000);
var getAllTokens = function(){
return db.facebookTokens.find().toArray()
.then(function(items){
tokens = _.pluck(items, "token");
// var MemoryStore = require('connect/middleware/session/memory');
var express = require('express');
var app = express();
var server = require('http').Server(app);
var io = require('socket.io')(server);
var cookieParser = require('cookie-parser');
var cookie = require('cookie');
var bodyParser = require('body-parser');
var session = require('express-session');
var sessionStore = new session.MemoryStore();
var pmongo = require('promised-mongo');
var _ = require('lodash');
var config = require('./config');
var db = pmongo(config.dbURL, ['users']);
var passport = require('passport');
var GoogleStrategy = require('passport-google').Strategy;
app.use(cookieParser()); // read cookies (needed for auth)
app.use(bodyParser()); // get information from html forms
app.use(session({ store: sessionStore, secret: config.sessionSecret, key:config.sidKey})); // session secret
app.use(passport.initialize());
app.use(passport.session()); // persistent login sessions
passport.use(new GoogleStrategy({
returnURL: 'http://'+config.host+':5000/auth/google/return',
realm: 'http://'+config.host+':5000/'
},
function(identifier, profile, done) {
profile.openId = identifier;
db.users.findOne({
var pmongo = require('promised-mongo');
var Promise = require('promise');
var db = pmongo('codeopticon');
var collections = {};
function getCollection(name) {
if (!(name in collections)) {
collections[name] = db.collection(name);
}
return collections[name];
}
function saveRepo(repo) {
var repos = getCollection('repos');
if (typeof repo === 'string') {
repo = JSON.parse(repo);
}
return repos.findOne({id: repo.id}).then(function(data) {
if (data) {
return;
}
return repos.save(repo);
});
}
function getRepo(obj) {
MongoDB.prototype.connect = function () {
var connectionStr = util.format('%s:%d/%s', this.config.host, this.config.port, this.config.database);
this.db = pmongo(connectionStr, [this.config.collection]);
this.collection = this.db.collection(this.config.collection);
};
var Promise = require('bluebird');
var http = require('http');
var request = Promise.promisifyAll(require('request'));
var _ = require('underscore');
/************************************************************************
* Other dependencies
***********************************************************************/
var foodPhrases = require('../shared/foodPhrases');
var excludedPhrases = require('../shared/excludedPhrases');
/************************************************************************
* Database access
***********************************************************************/
var pmongo = require('promised-mongo');
var db = Promise.promisifyAll(pmongo(process.env.MONGOURL, ['meetup','eventbrite','funcheap','facebook','coordsTable']));
/************************************************************************
* Main Function
***********************************************************************/
module.exports = function(req, res){
res.header("Access-Control-Allow-Origin", "*");
// Below changed the default address for testing purposes
var address = req.query.address || '94108';
console.log('address: ',address);
var time = req.query.time || new Date().getTime();
var radius = req.query.radius || 5;
var googleApiKey = process.env.GOOGLEAPIKEY || "123FAKEKEY";
// console.log("address:", address, "time:", time);
var pmongo = require('promised-mongo');
if ( process.env.NODE_ENV === 'production' ) {
var db = pmongo(process.env.MONGODB_URI, {
authMechanism: 'ScramSHA1'
});
}
else {
var db = pmongo('goodnewsdb');
}
module.exports = db;
// clears database
db.deleteEverything = function () {
return Promise.all([
db.collection('articles').remove({})
])
};
'use strict';
var express = require('express');
var app = express();
var pmongo = require('promised-mongo');
var db = pmongo('contactlist', ['contactlist']);
app.disable('X-Powered-By');
app.post('/negative', function(req, res) {
var x = req.body;
eval(x + 3);
res.send('Hello World');
});
app.post('/positive', function(req, res) {
eval('console.log("Hello World")');
res.send('Hello again');
});
var server = app.listen(3000, function () {
var port = server.address().port;
console.log('Your app listening at http://localhost:%s', port);
});
module.exports = function(cfg){
var mongoUrl = cfg.mongodb && cfg.mongodb.url || 'mongodb://localhost/default';
var items = mongo(mongoUrl).collection('items');
function getTrio(activeQuery,onSuccess,cb){
items.findOne(activeQuery).then(function(active){
if(!active) cb(null);
else {
//this is pretty bad
var prevQuery = active.type == 'achewood'?
{ $or: [
{published: {$lt: active.published}},
{published: active.published, type: 'raysplace'}]} :
{published: {$lt: active.published}};
var nextQuery = active.type == 'raysplace'?
{ $or: [
{published: {$gt: active.published}},
{published: active.published, type: 'achewood'}]} :
{published: {$gt: active.published}};
Promise.all([
items.findOne(prevQuery, {limit: 1,
sort: {published: -1}, fields: edgeFields }),
items.findOne(nextQuery, {limit: 1,
sort: {published: 1}, fields: edgeFields })
]).then(function(values){
var trio = {
active: fortifyItem(active),
prev: fortifyItem(values[0]),
next: fortifyItem(values[1])
};
cb(trio);
});
}
});
}
function renderPage(reqCobbler) {
return function(req,res,next) {
var onSuccess = errHandlingCb(next);
var cobbled = reqCobbler(req);
getTrio(cobbled.query,onSuccess,function(trio){
if(!trio){
next();
} else {
if (trio.active.type == 'blog') {
trio.blogName = blogInfo[trio.active.blog].title;
}
trio.pageTitle = getPageTitle(trio.active);
trio.originalUrl = req.originalUrl;
res.render(cobbled.viewName,trio);
}
});
};
}
var app = express();
app.set('views', __dirname + '/views');
app.set('view engine', 'jade');
app.get("/achewood/date/:date",renderPage(function(req){
return { query: {type: 'achewood', mdydate: req.params.date},
viewName: 'achewood' };
}));
app.get("/raysplace/date/:date",renderPage(function(req){
return { query: {type: 'raysplace', mdydate: req.params.date},
viewName: 'raysplace' };
}));
app.get("/blogs/:blog*",renderPage(function(req){
return { query: {type: 'blog', blog: req.params.blog,
path: req.params[0] },
viewName: 'blogs/' + req.params.blog };
}));
app.get("/",function(req,res){
res.render('index');
});
app.get("/list",function(req,res,next){
return next();
});
function getLocationForDocument(doc){
if(doc.type == 'achewood'){
return '/achewood/date/'+doc.mdydate;
} else if(doc.type == 'raysplace') {
return '/raysplace/date/'+doc.mdydate;
} else if(doc.type == 'blog') {
return '/blogs/'+doc.blog+doc.path;
} else return null;
}
function redirectToDocLocation(doc,res){
if(doc){
res.redirect(getLocationForDocument(doc));
} else {
res.render('notfound');
}
}
function redirectToLatest(type,res,next){
var query = {};
if (type) query.type = type;
items.findOne(query, {limit: 1,
sort: {published: -1},
fields: {
type: 1,
blog: 1,
mdydate: 1,
path: 1}}).then(function(doc){
redirectToDocLocation(doc,res);
});
}
app.get("/latest",function(req,res,next){
redirectToLatest(req.query.type,res,next);
});
app.get("/go",function(req,res){
res.redirect(getLocationForSource(req.param('q')));
});
//error handlers like this are too good to save for just errors
app.get("/404",function(req,res){
res.render('notfound');
});
app.get("/500",function(req,res){
res.render('error',{error: req.query.e || "ENTERING DEATH MODE"});
});
app.use(express.static(__dirname+'/static'));
app.use('/gh-pages', express.static(__dirname+'/docs'));
app.use(function respondNotFound(req,res){
res.status(404).render('notfound');
});
app.use(function respondError(err,req,res,next){
console.error(err);
res.status(500).render('error',{error:err});
});
return app;
};
var pmongo = require('promised-mongo');
var uri = 'mongodb://*****:*****@ds021895.mlab.com:21895/pear';
var db = pmongo(uri, {
authMechanism: 'ScramSHA1'
});
module.exports = db
var Promise = require('bluebird');
var mongo = require('promised-mongo');
var db = mongo('test', ['a'])
db.a.insert({a:'this is a test'})
.then(db.a.find().forEach(function(doc) {
console.log(doc);
})
.then(function() {
console.log('finished');
}))
const _ = require('lodash');
import {
GraphQLList,
GraphQLObjectType,
GraphQLSchema,
GraphQLString,
GraphQLInt,
GraphQLFloat,
GraphQLEnumType,
GraphQLNonNull,
GraphQLInterfaceType
} from 'graphql';
const mongo = require('promised-mongo');
const db = mongo('mongodb://localhost/mydb');
const postsCollection = db.collection('posts');
const firebase = require("firebase");
const authorsRef = new firebase("https://tz-graphql.firebaseio.com/").child("authors");
const Author = new GraphQLObjectType({
name: "Author",
description: "Represent the type of an author of a blog post or a comment",
fields: () => ({
_id: {type: GraphQLString},
name: {type: GraphQLString},
twitterHandle: {type: GraphQLString}
})
});
const Post = new GraphQLObjectType({
module.exports = function(__dirname) {
/**
* Imports and Initializations
**/
var path = require("path");
var config = require("../config")
var pm = require("promised-mongo")
var db = pm("mongodb://localhost/bydesign", ["authors", "posts"])
var handlebars = require("handlebars")
handlebars = require("./stachehelper")(handlebars, config.paths.client)
var insert = require("./insertPost")(db, config.paths.client)
var Renderer = require("./renderer")
var api = require("./api")(db)
var express = require("express")
var app = express()
var cookie = require("cookie-parser")
var body = require("body-parser")
var session = require("express-session")
var MongoStore = require("connect-mongo")(session)
var Google = require('passport-google-oauth').OAuth2Strategy
var passport = require("passport")
var logger = require("./logger")
var morgan = require("morgan")
var utils = require("./utils")
var renderer = new Renderer(__dirname, db, handlebars);
/**
* Middleware Initialization
**/
app.use(morgan("dev"))
app.use(cookie())
app.use(body.json())
app.use(session({ secret: 'temporarysecret', store: new MongoStore({
url: "mongodb://localhost/bydesign"
}),
resave: true,
saveUninitialized: true,
}))
app.use(passport.initialize())
app.use(passport.session())
app.use(renderer.handle.bind(renderer))
app.use(api)
/**
* App routing
**/
app.use(express.static("build"))
app.post("/visible", function(req, res) {
if (req.user) {
handleVisibility(req.body.state, req.body.page).
then(function() {
res.status(200)
res.send({
"state": req.body.state ? "visible" : "hidden",
"visible": req.body.state
})
renderer.reload()
}).
catch(function(err) {
res.status(500)
res.send({
"state": !req.body.state ? "visible" : "hidden",
"visible": !req.body.state
})
logger.error(err)
})
} else {
req.status(403)
res.send("Please Log In first")
}
})
app.post("/editor/", function(req, res) {
if (req.user) {
uploadPost(null, req.body, req.user.id)
.then(function (id) {
renderer.reload()
res.send({msg: "Ok", id: id})
})
.catch(logger.error)
} else {
req.status(403)
res.send("Please Log In first")
}
})
app.post("/editor/:MOD", function(req, res) {
if (req.user) {
uploadPost(req.params.MOD, req.body, req.user.id)
.then(renderer.reload.bind(renderer))
.catch(logger.error)
res.send({msg: "Ok"})
} else {
req.status(403)
res.send("Please Log In first")
}
})
/**
* Upload Handling
**/
var uploadPost = function(guid, body, author) {
if (body.tags.length == 0 && body.title.match(/^\s*$/) && body.content.match(/^\s*$/) && guid) {
return db.posts.remove({ guid });
}
if (!guid) {
return utils.generateId(db.posts)
.then((id) =>
db.posts.insert({
tags: body.tags,
visible: body.visible === true,
author: author,
timestamp: Date.now(),
guid: id,
title: {
text: body.title,
url: `/posts/${id}`
},
slug: utils.slugify(body.title),
content: body.content
})
.then(() => id)
);
} else {
return db.posts.update({ guid }, {
$set: {
title: {
text: body.title
},
content: body.content,
tags: body.tags
}
});
}
}
var handleVisibility = function(visible, guid) {
return db.posts.update({ guid }, { $set: { visible }});
}
/**
* MongoDB access functions
**/
var getAuthors = function(gid) {
return db.authors.findOne({ gid });
}
/**
* Authentication using Passport OAuth
**/
passport.serializeUser(function(user, done) {
db.authors.findOne({"id": user.id})
.then((data) => {
if (data !== undefined) {
user._json.image.url = user._json.image.url.replace(/sz=50$/, "sz=576");
return db.authors.update({"id": user.id}, user).then(() => done(undefined, JSON.stringify(user)));
} else {
return db.authors.insert(user, () => done(undefined, JSON.stringify(user)))
}
})
.catch((err) => done(err, JSON.stringify(user)))
})
passport.deserializeUser(function(user, done) {
done(null, JSON.parse(user))
})
if (process.env.AUTH_SECRET) {
passport.use(new Google({
clientID: "477715393921-ft3c5717cv685qomofqhksgtg2sk6ciu.apps.googleusercontent.com",
clientSecret: process.env.AUTH_SECRET,
callbackURL: process.env.REMOTE ? "http://dttw.tech/google/auth" : "http://127.0.0.1:3000/google/auth"
},
function(accessToken, refreshToken, profile, done) {
for (var i = 0; i < profile.emails.length; i++) {
if (config.admins.indexOf(profile.emails[i].value) >= 0) {
done(null, profile)
return
}
}
done("Non Admin")
}
))
app.get('/google/login',
passport.authenticate('google', { scope: 'https://www.googleapis.com/auth/userinfo.email' }))
app.get(/^\/google\/auth/,
passport.authenticate('google', { failureRedirect: '/google/login' }),
function(req, res) {
// Successful authentication, redirect home.
res.redirect('/admin')
})
} else {
logger.warn("Missing authentication variable. Authentication will be unavailable.")
}
// Default Case: 404
app.use(renderer.fourohfour.bind(renderer))
/**
* API Access
**/
/**
* Start Server
**/
app.listen(3000)
return app
}
var pmongo = require('promised-mongo');
var db;
/*module.exports.Register = function(err, db, uri, config){
MongoClient.connect(uri, config, function (err, database) {
if(err) return next(err);
db = database;
module.exports.db = db;
module.exports.ObjectID = ObjectID;
});
};*/
db = pmongo('test');
module.exports.db = db;
module.exports.ObjectId = pmongo.ObjectId;
const mongo = require('promised-mongo')
const db = mongo('mongodb://localhost:27017/app')
// TODO if the mongo db is unreachable on startup, the app will never connect
// if mongodb fails during execution, it will also never reconnect
/*
not supported by promised-mongo
db.on('error', (err) => {
console.warn('database error', err) // eslint-disable-line no-console
})
db.on('connect', () => {
console.warn('database connected') // eslint-disable-line no-console
})
*/
module.exports = {
db,
todosCollection: db.collection('todos'),
usersCollection: db.collection('users'),
// postsCollection: db.collection('posts'),
// commentsCollection: db.collection('comments'),
}
var path = require('path');
var React = require('react/addons');
var ReactApp = React.createFactory(require('./site').ReactApp);
var bodyParser = require('body-parser');
var expressValidator = require('express-validator');
var util = require("util");
var Promise = require('bluebird');
app.use(bodyParser.json());
app.use(expressValidator({
customValidators: {
isObjectId: isValidObjectId
}
}));
app.use(express.static(path.join(__dirname, 'static')));
var pmongo = require('promised-mongo');
var db = pmongo(process.env.MONGOLAB_URI || 'near');
app.get('/', function (req, res) {
Promise.all([getCities(), getPlaces()]).then(function(results) {
var cities = results[0];
var places = results[1];
var state = {cities: cities, places: places};
var reactHtml = React.renderToString(ReactApp({initialState: state}));
res.render('index.ejs', {reactOutput: reactHtml, state: JSON.stringify(state)});
}).done()
});
app.use(logger('dev'));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.use(cookieParser());
app.use(session({
secret: 'keyboard catz',
resave: false,
saveUninitialized: true
}));
app.use(flash());
app.use(express.static(path.join(__dirname, 'public')));
var db;
if (process.env.MONGOHQ_URL) {
console.log("Connecting to MongoHQ")
db = pmongo(process.env.MONGOHQ_URL, ['matches']);
} else {
db = pmongo('match-audio', ['matches']);
}
app.use(function(req, res, next) {
req.db = res.db = db;
next();
})
app.get('*', function(req,res,next) {
// force SSL
if (req.headers['cf-visitor'] && req.headers['cf-visitor'] != '{"scheme":"https"}') {
return res.redirect("https://" + req.headers['host'] + req.url);
} else if (req.headers['cf-visitor']) {
var restify = require('restify');
var fs = require('fs');
var validator = require('is-my-json-valid/require');
var pmongo = require('promised-mongo');
var db = pmongo(process.env.MONGODB || 'mongodb://localhost/aleksandria');
var creds = {
user: process.env.USER || 'admin',
passwd: process.env.PASSWORD || 'password'
};
var schemas = fs.readdirSync(__dirname+'/schemas')
.filter(function (filename) {
return /^.*\.json$/.test(filename);
}).map(function (filename) {
var name = filename.replace('.json', '');
var schema = validator('/schemas/'+filename,
{
greedy:true,
verbose:true
});
return {
name: name,
schema: schema
};
}).reduce(function (schemas, schema) {
schemas[schema.name] = schema.schema;
return schemas;
}, {});
connect: function(url) {
this.db = mongo(url, [this.settings.treeCollection, this.settings.nodeCollection]);
},
var Promise = require('bluebird');
var request = Promise.promisifyAll(require('request'));
var cheerio = require('cheerio');
var _ = require('underscore');
var pmongo = require('promised-mongo');
var db = pmongo('mongodb://localhost:27017/feedme', ["eventbrite"]);
var googleApiKey = process.env.GOOGLEAPIKEY || "123FAKEKEY";
var eventUrls = [];
var pageCount = 0;
var insertCount = 0;
var getSearchPageUrl = function(pageNum){
return "http://www.eventbrite.com/directory?loc=San+Francisco%2C+CA&is_miles=True&vp_ne_lat=37.812&price=1&vp_sw_lng=-122.527&page="+pageNum+"&slng=-122.42&vp_sw_lat=37.7034&radius=60.0&vp_ne_lng=-122.3482&slat=37.77";
};
//need to manually terminate program when scraping complete, since db connection is always open
var terminateProgram = function(){
console.log("Program finished,", insertCount, "entries added / updated");
process.exit(1);
};
//recursively get urls of all event pages from search results
var getEventLinks = function(pageNum, recursiveCount, finishCallback){
var url = getSearchPageUrl(pageNum);
var $;
console.log("GET Search Page:", pageNum);
pageCount++;
request.getAsync(url)
.then(function(args){
$ = cheerio.load(args[1]);
return $(".js-search-result-click-action").toArray();
})
function serve () {
console.log(kuler('castor@' + pck.version, 'orange'));
//
// Data path :
// Check and fix a data source directory
//
var dataPath = config.get('dataPath') ;
var dateConfig;
debug('dataPath', dataPath);
var confile = path.normalize(dataPath) + '.json';
if (fs.existsSync(confile)) {
console.log(kuler('Configuration :', 'olive'), kuler(confile, 'limegreen'));
config.load(confile);
dateConfig = fs.statSync(confile).mtime;
}
config.set('dataPath', dataPath);
//
// View path :
// Find and Check the directory's templates
//
var viewPath = require('./helpers/view.js')(config);
//
// Check & Fix required config parameters
//
config.fix('connexionURI', 'mongodb://localhost:27017/castor/');
config.fix('collectionName', undefined); // auto
config.fix('debug', false);
config.fix('port', '3000');
config.fix('logFormat', 'combined');
config.fix('title', 'Castor');
config.fix('description', '');
config.fix('theme', 'default');
config.fix('middlewares', {});
config.fix('filters', {});
config.fix('asynchronousFilters', {});
config.fix('operators', {});
config.fix('loaders', {});
config.fix('routes', {});
config.fix('browserifyModules', []);
config.fix('itemsPerPage', 30);
config.fix('concurrency', require('os').cpus().length);
config.fix('turnoffAll', false);
config.fix('turnoffSync', false);
config.fix('turnoffPrimus', false);
config.fix('turnoffRoutes', false);
config.fix('turnoffWebdav', false);
config.fix('filesToIgnore', [ "**/.*", "~*", "*~", "*.sw?", "*.old", "*.bak", "**/node_modules" ]);
config.fix('customFields', {});
config.fix('loader:csv:separator', undefined); // auto
config.fix('loader:csv:encoding', 'utf8');
if (config.get('turnoffAll') === true) {
config.set('turnoffSync', true);
config.set('turnoffPrimus', true);
config.set('turnoffRoutes', true);
config.set('turnoffWebdav', true);
}
console.log(kuler('Theme :', 'olive'), kuler(viewPath, 'limegreen'));
//
// add some statements when loading files to MongoDB
//
if (fs.existsSync(dataPath)) {
console.log(kuler('Source :', 'olive'), kuler(dataPath, 'limegreen'));
var opts = {
"connexionURI" : config.get('connexionURI'),
"collectionName": config.get('collectionName'),
"concurrency" : config.get('concurrency'),
"ignore" : config.get('filesToIgnore'),
"dateConfig" : dateConfig
};
var fr = new Loader(dataPath, opts);
fr.use('**/*', require('./loaders/initialize-tags.js')(config));
fr.use('**/*.csv', require('castor-load-csv')(config.get('loader:csv')));
fr.use('**/*.xml', require('castor-load-xml')(config.get('loader:xml')));
// fr.use('**/*.pdf', require('./loaders/append-yaml.js')());
hook('loaders')
.from(viewPath, __dirname)
.over(config.get('loaders'))
.apply(function(hash, func) {
fr.use(hash, func(config.get('loaders:' + hash)));
});
fr.use('**/*', require('castor-load-custom')({
fieldname : 'fields',
schema: config.get('customFields')
}));
if (config.get('turnoffSync') === false) {
fr.sync(function(err) {
console.log(kuler('Files and Database are synchronised.', 'green'));
});
}
config.set('collectionName', fr.options.collectionName);
}
//
// add some indexes
//
var coll = pmongo(config.get('connexionURI')).collection(config.get('collectionName'))
, usfs = config.get('customFields')
, idx = Object.keys(usfs).map(function(i) {var j = {}; j['fields.' + i] = 1; return j});
idx.push({ 'text': 'text' });
idx.forEach(function(i) {
coll.ensureIndex(i, { w: 1 }, function(err, indexName) {
console.log(kuler('Index field :', 'olive'), kuler(Object.keys(i)[0] + '/' + indexName, 'limegreen'));
});
});
var ops = require('./helpers/operators.js');
ops.use('count', require('./operators/count.js'));
ops.use('catalog', require('./operators/catalog.js'));
ops.use('distinct', require('./operators/distinct.js'));
ops.use('ventilate', require('./operators/ventilate.js'));
ops.use('total', require('./operators/total.js'));
hook('operators')
.from(viewPath, __dirname)
.over(config.get('operators'))
.apply(function(hash, func) {
ops.use(hash, func);
});
//
// Middlewares :
// add middlewares to Express
//
var app = express();
var env = nunjucks.configure(viewPath, {
autoescape: true,
express: app
});
env.addFilter('nl2br', require('./filters/nl2br.js')(config));
env.addFilter('hash', require('./filters/hash.js')(config));
env.addFilter('stack', require('./filters/stack.js')(config));
hook('filters')
.from(viewPath, __dirname)
.over(config.get('filters'))
.apply(function(hash, func) {
env.addFilter(hash, func(config));
});
hook('filters')
.from(viewPath, __dirname)
.over(config.get('asynchronousFilters'))
.apply(function(hash, func) {
env.addFilter(hash, func(config), true);
});
app.use(morgan(config.get('logFormat'), {
stream : process.stderr
}));
hook('middlewares')
.from(viewPath, __dirname)
.over(config.get('middlewares'))
.apply(function(hash, func) {
app.use(hash, func(config));
});
if (config.get('turnoffRoutes') === false) {
//
// add routes to send data on the Web
//
hook('routes')
.from(viewPath, __dirname)
.over(config.get('routes'))
.apply(function(hash, func) {
app.route(hash).all(func(config));
});
app.route('/robots.txt').get(require('./routes/inform-robots.js')(config));
app.route('/sitemap.xml').get(require('./routes/inform-searchengines.js')(config));
app.route('/browse.:format').all(require('./routes/browse.js')(config));
app.route('/compute.:format').all(require('./routes/compute.js')(config));
app.route('/display/:doc.:format').all(require('./routes/display.js')(config));
app.route('/dump/:doc.:format').all(require('./routes/dump.js')(config));
app.route('/save/:doc').all(bodyParser.urlencoded({ extended: false })).post(require('./routes/save.js')(config));
app.route('/export.:format').all(require('./routes/export-docs.js')(config));
app.route('/:name.:format').all(require('./routes/serve.js')(config));
var modules = config.get('browserifyModules');
if (Array.isArray(modules) && modules.length > 0) {
app.get('/bundle.js', browserify(modules));
}
if (config.get('turnoffWebdav') === false) {
app.route('/webdav*').all(require('./helpers/webdav.js')({
debug: config.get('debug')
}));
}
app.route('/assets/*').all(require('ecstatic')({
root : path.join(viewPath, 'assets'),
baseDir : '/assets',
cache : 3600,
showDir : true,
autoIndex : true,
humanReadable : true,
si : false,
defaultExt : 'html',
gzip : false
}));
app.route('/').all(function(req, res) { res.redirect('index.html'); });
app.use(function(req, res, next) {
res.status(404).send('Not Found').end();
});
}
else {
app.use(function(req, res, next) {
res.status(503).send('Service Unavailable').end();
});
}
//
// HTTP Server :
// initialize the HTTP server and the "realtime" server
//
var server = require('http').createServer(app);
if (config.get('turnoffPrimus') === false) {
var primus = new Primus(server, {});
primus.use('emitter', require('primus-emitter'));
primus.on('connection', function (spark) {
fr.on('changed', function(err, doc) {
if (!err) {
debug('changed', err, doc);
spark.send('changed', doc);
}
});
fr.on('cancelled', function(err, doc) {
if (!err) {
debug('cancelled', err, doc);
spark.send('cancelled', doc);
}
});
fr.on('dropped', function(err, doc) {
if (!err) {
debug('dropped', err, doc);
spark.send('dropped', doc);
}
});
fr.on('added', function(err, doc) {
if (!err) {
debug('added', err, doc);
spark.send('added', doc);
}
});
});
}
//
// Listen :
// get or find a port number and launche the server.
//
portfinder.basePort = config.get('port');
portfinder.getPort(function (err, newport) {
if (err) {
throw err;
}
config.set('port', newport);
server.listen(newport, function() {
console.log(kuler('Server is listening on port ' + server.address().port + '.', 'green'));
});
});
return server;
}
var bodyParser = require('body-parser');
var pmongo = require('promised-mongo');
var debug = require('debug')('server');
var app = express();
// schemas setup
var schema = {};
fs.readdirSync(path.join(__dirname, 'schema')).forEach(function (file) {
var s = require(path.join(__dirname, 'schema', file));
schema[s.title] = s;
});
app.set('schema', schema);
// mongo setup
var db = pmongo(process.env.MONGO_CONN_STRING, ['manager', 'channel', 'listener']);
db.manager.ensureIndex({ rdioUser: 1 }, { unique: true });
app.set('db', db);
// express setup
app.use(favicon());
app.use(logger('dev'));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded());
app.use(cookieParser());
app.all('*', function(req, res, next) {
res.header('Access-Control-Allow-Origin', '*');
res.header('Access-Control-Allow-Headers', 'Content-Type,Authorization,User,RdioUser,Listener');
var pmongo = require('promised-mongo');
var dbName = process.env.NODE_ENV === 'production' ? process.env.MONGOLAB_URI : 'tutoriality_dev';
var db = pmongo(dbName);
var _ = require('underscore');
db.deleteEverything = function () {
return Promise.all([
db.collection('tutorials').remove(),
db.collection('users').remove()
])
}
db.loadFixtures = function() {
// Get the fixture from a file, this just an object
// where the key is the collection name and the
// value is an array of records to be inserted
var fixtures = require('./fixtures.js');
_.each(fixtures, function(records, collectionName){
db.collection(collectionName).find().then(function(items){
// only insert if the collection is empty
if(items.length === 0) {
db.collection(collectionName).insert(records);
}
})
});
}
db.collection('users').createIndex( { "alias": 1 }, { unique: true } )
db.getMongoID = function(idString) {
return pmongo.ObjectId(idString);
}
function Processor(schema, options) {
options = options || {};
if (!(this instanceof Processor)) {
return new Processor(schema, options);
}
events.EventEmitter.call(this);
var self = this;
self.options = {};
self.options.collectionName = options.collectionName || '';
self.options.connexionURI = options.connexionURI || process.env.MONGO_URL;
self.options.concurrency = options.concurrency || 1;
self.schema = schema;
self.enrich = new Enricher(self.schema);
self.coll1 = pmongo(self.options.connexionURI).collection(self.options.collectionName);
self.coll2 = pmongo(self.options.connexionURI).collection(self.options.collectionName + '_corpus');
self.iterator1 = function iterator1(key, callback) {
//
// .operator
//
if (self.schema[key].operator) {
var ope = self.operators[self.schema[key].operator];
if (ope && typeof ope.map === 'function' && typeof ope.reduce === 'function') {
var map = ope.map;
var reduce = ope.reduce;
var finalize = ope.finalize || function(o) { return o;};
var fields = self.schema[key].fields || {};
var opts = {
out : {
replace : self.options.collectionName + '_computer_' + key
},
query: self.schema[key].selector || {},
scope: {
exp : fields
}
};
self.coll1.mapReduce(map, reduce, opts).then(function(coll) {
coll.find().toArray(function (err, res) {
debug('mapreduce('+key+')', res);
return callback(null, { _id : key, value: err ? err : finalize(res) });
});
}).catch(function (err) {
return callback(null, { _id : key, value: err });
});
return;
}
}
callback(null, { _id : key, value: self.schema[key].default ? self.schema[key].default : ''})
}
self.iterator2 = function iterator2(item, index, values) {
var ctx = _.chain(values).map(function(kv) { return [kv._id, kv.value]; }).object().value();
var ret = self.enrich.process(item._id, ctx, item.value);
if (ret !== undefined) {
item.value = ret;
}
return item;
}
}