Example #1
0
var executeTask = function(t){

	isTaskExecuting = true;
	
	
	Step(
		function() {
			logger.debug("=======Task Begin==========");
			logger.debug("The task is: ");
			logger.debug(t);
			downloadFile(t, this);
		},
		function(err, t) {
			if (err) {
				logger.debug("Skip doTranslation");
				throw err;
			}
			doTranslation(t, this);
		},
		function(err, t) {
			if (err) {
				logger.debug("Skip uploadFile");
				throw err;
			}
			uploadFile(t, this);
		},
		function(err, t) {
			sendMailNotification(t, this);
		},
		function(err, t) {
			cleanupTempFiles(t, this);
			
			//this(err);
		},
		function (err) {			
			isTaskExecuting = false;
			
			logger.debug("=======Task Complete==========");
		}
	);
};
Example #2
0
var dialback = function(req, res, next) {

    var auth,
        now = Date.now(),
        fields,
        unauthorized = function(msg) {
            res.status(401);
            res.setHeader("WWW-Authentication", "Dialback");
            res.setHeader("Content-Type", "text/plain");
            res.send(msg);
        },
        parseFields = function(str) {
            var fstr = str.substr(9); // everything after "Dialback "
            var pairs = fstr.split(/,\s+/); // XXX: won't handle blanks inside values well
            var fields = {};
            pairs.forEach(function(pair) {
                var kv = pair.split("="),
                    key = kv[0],
                    value = kv[1].replace(/^"|"$/g, "");
                fields[key] = value;
            });
            return fields;
        };

    if (!req.headers.hasOwnProperty("authorization")) {
        unauthorized("No Authorization header");
        return;
    }

    auth = req.headers.authorization;

    if (auth.substr(0, 9) != "Dialback ") {
        unauthorized("Authorization scheme is not 'Dialback'");
        return;
    }

    fields = parseFields(auth);

    // must have a token

    if (!fields.hasOwnProperty("token")) {
        unauthorized("Authorization header has no 'token' field");
        return;
    }

    // must have a webfinger or host field

    if (!fields.hasOwnProperty("host") && !fields.hasOwnProperty("webfinger")) {
        unauthorized("Authorization header has neither 'host' nor 'webfinger' fields");
        return;
    }

    fields.url = URLMaker.makeURL(req.originalUrl);

    if (!req.headers.hasOwnProperty("date")) {
        unauthorized("No 'Date' header");
        return;
    }

    fields.date = req.headers.date;

    if (Math.abs(Date.parse(fields.date) - now) > 300000) { // 5-minute window
        unauthorized("'Date' header is outside our 5-minute window");
        return;
    }

    Step(
        function() {
            seenRequest(fields.host || fields.webfinger,
                        fields.url,
                        fields.date,
                        fields.token,
                        this);
        },
        function(err, seen) {
            if (err) throw err;
            if (seen) {
                unauthorized("We've already seen this request.");
                return;
            } else {
                saveRequest(fields.host || fields.webfinger,
                            fields.url,
                            fields.date,
                            fields.token,
                            this);
            }
        },
        function(err) {
            if (err) throw err;
            discoverEndpoint(fields, this);
        },
        function(err, endpoint) {
            if (err) throw err;
            postToEndpoint(endpoint, fields, this);
        },
        function(err, body, res) {
            if (err) {
                unauthorized(err.message);
            } else if (fields.hasOwnProperty("host")) {
                req.remoteHost = fields.host;
                next();
            } else if (fields.hasOwnProperty("webfinger")) {
                req.remoteUser = fields.webfinger;
                next();
            }
        }
    );
};
Example #3
0
    me.init = function(callback, allow_broken_css){
        var that = this;
        var store_key = extended_store_key || base_store_key;
        var redis_client;
        var style;
        var style_version;

        Step(
            function getRedisClient(){
                redis_pool.acquire(grainstore_styles.db, this);
            },
            function getStyle(err, data){
                if (err) throw err;
                redis_client = data;
                redis_client.GET(store_key, this);
            },
            function initCheck(err, data){
                if (err) throw err;

                do { 

                  if (_.isNull(data)) {
                    // no redis record
                    break;
                  }

                  var record = JSON.parse(data);

                  // save pre-recorded interactivity and sql
                  if ( record.interactivity ) interactivity = record.interactivity;
                  if ( record.sql ) opts.sql = record.sql;

                  // All checks passed, nothing more to do here
                  if (!_.isUndefined(redis_client))
                      redis_pool.release(grainstore_styles.db, redis_client);
                  me.knownByRedis = true;
                  callback(null, data);
                  return;

                } while (0);

                // XML needs to be re-generated, go on
                if ( !_.isNull(style_override) ) return null;

                redis_client.GET(base_store_key, this);
            },
            function renderBaseStyleOrDefaultOrOverride(err, data){
                if (err) throw err;
                if ( ! _.isNull(data)){
                    var parsed = JSON.parse(data);
                    style = parsed.style;
                    style_version = parsed.version || default_style_version;
                    //interactivity = parsed.interactivity;
                } else if ( ! _.isNull(style_override) ) {
                    style = style_override;
                    style_version = style_version_override || default_style_version;
                } else {
                    if ( ! grainstore_styles.hasOwnProperty(geom_type) ) {
                      throw new Error("No style available for geometry of type '" + geom_type + "'"); 
                    }
                    style = grainstore_styles[geom_type];
                    style_version = grainstore_styles['version'];
                }
                var next = this;

                // We only render in order to check the XML
                if ( allow_broken_css ) return null;

                that.render(style, function(err, compiled_XML) {
                  if ( err ) {
                    if ( allow_broken_css ) {
                      console.log("Invalid style: " + style + " -- " + err);
                      err = compiled_XML = null;
                    }
                  } 
                  next(err);
                }, style_version);
            },
            function setStore(err){
                if (err) throw err;

                if ( store_key == base_store_key ) {
                  if ( ! opts.table ) return null;
                  var tostore = {};
                  tostore.sql = opts.sql;
                  if ( opts.gcols ) tostore.gcols = opts.gcols;
                  if ( opts.gcoltypes ) tostore.gcoltypes = opts.gcoltypes;
                  if ( opts.extra_ds_opts ) tostore.extra_ds_opts = opts.extra_ds_opts;
                  tostore.style = style;
                  tostore.version = style_version;
                  tostore.interactivity = interactivity;
                  redis_client.SET(store_key, JSON.stringify(tostore), this);
                }
                else if ( _.isNull(style_override) ) { 
                  var tostore = {style: style, version: style_version};
                  var payload = JSON.stringify(tostore);
                  redis_client.SET(base_store_key, payload, this);
                }
                else return null;
            },
            function callbackExit(err){
                if (!_.isUndefined(redis_client))
                    redis_pool.release(grainstore_styles.db, redis_client);
                callback(err, JSON.stringify({style: style, version: style_version}));
            }
        );
    };
Example #4
0
MBTiles.prototype.info = function(callback) {
    var that = this;
    var info = {};
    info.basename = path.basename(that.filename);
    info.id = info.basename.replace(path.extname(that.filename), '');
    Step(
        // Load metadata table
        function() {
            var end = this;
            that.db.all("SELECT name, value FROM metadata", function(err, rows) {
                if (rows) for (var i = 0; i < rows.length; i++) {
                    info[rows[i].name] = rows[i].value;
                }
                end(err);
            });
        },
        // Determine min/max zoom if needed
        function(err) {
            if (err) return callback(err);
            if (typeof info.maxzoom !== 'undefined'
                && typeof info.minzoom !== 'undefined') return this(null);

            var group = this.group();

            var zoomquery = that.db.prepare('SELECT zoom_level FROM tiles ' +
                                            'WHERE zoom_level = ? LIMIT 1');
            for (var i = 0; i < 30; i++) {
                zoomquery.get(i, group());
            }
            zoomquery.finalize();
        },
        function(err, rows) {
            if (err) throw err;
            if (!err && rows) {
                var zooms = _.filter(rows, function(row) { return row; }),
                zooms = _.pluck(zooms, 'zoom_level');
                info.minzoom = zooms.shift();
                info.maxzoom = zooms.length ? zooms.pop() : info.minzoom;
            }
            this();
        },
        // Determine bounds if needed
        function(err) {
            if (err) return callback(err);
            if (info.bounds) return this();
            if (typeof info.minzoom === 'undefined') return this();

            var next = this;
            Step(
                function() {
                    that.db.get(
                        'SELECT MAX(tile_column) AS maxx, ' +
                        'MIN(tile_column) AS minx, MAX(tile_row) AS maxy, ' +
                        'MIN(tile_row) AS miny FROM tiles ' +
                        'WHERE zoom_level = ?',
                        info.minzoom,
                        this
                    );
                },
                function(err, row) {
                    if (!err && row) {
                        // @TODO this breaks a little at zoom level zero
                        var mercator = new SphericalMercator({});
                        var urTile = mercator.xyz_to_bbox(row.maxx,
                            row.maxy, info.minzoom, true);
                        var llTile = mercator.xyz_to_bbox(row.minx,
                            row.miny, info.minzoom, true);
                        info.bounds = [llTile[0], llTile[1],
                            urTile[2], urTile[3]].join(',');
                    }
                    next();
                }
            );
        },
        // Return info
        function(err) {
            if (err) return callback(err);
            info.minzoom = parseInt(info.minzoom);
            info.maxzoom = parseInt(info.maxzoom);
            info.bounds = _.map(info.bounds.split(','), function(val) { return parseFloat(val) });
            info.center = {
                lat: (info.bounds[3] - info.bounds[1]) / 2 + info.bounds[1],
                lon: (info.bounds[2] - info.bounds[0]) / 2 + info.bounds[0]
            }
            callback(null, info);
        }
    );
};
Example #5
0
module.exports = function assetManager (assets, options) {
	var self = this;

	settings = assets || settings;
	if (!settings) {
		throw new Exception('No asset groups found');
	}

	options = options || {};
	var jsmin = getJsMinifier(options.jsMinifier);

	if (!settings.forEach) {
		settings.forEach = function(callback) {
			Object.keys(this).forEach(function(key) {
				if (key !== 'forEach') {
					callback(settings[key], key);
				}
			});
		};
	}

	Step(function() {
		var grouping = this.group();
		settings.forEach(function(group, groupName) {
			var patterns = []
			, insertions = []
			, matchInsertionCount = {};

			group.files.forEach(function(fileName, index) {
				var pattern = null;
				if (fileName.exec) { // Got a RegEx
					pattern = fileName;
				} else if (fileName.trim() === '*') {
					pattern = /\.[a-z]+$/i; // Anything with a extension
				}

				if (pattern) {
					patterns.push({
						pattern: pattern,
						index: index
					});
					matchInsertionCount['insert-'+index] = 0;
				}
			});

			var fileFetchCallback = grouping();
			fs.readdir(group.path, function(err, files) {
				if (err) {
					throw err;
				}
				files.forEach(function(fileName, index) {
					var alreadyIncluded = false,
						matchedPattern = false;

					group.files.forEach(function(includedFile) {
						if (alreadyIncluded || includedFile.trim && (includedFile.trim() === fileName.trim())) {
							alreadyIncluded = true;
						}
					});

					if (!alreadyIncluded) {
						patterns.forEach(function(pattern) {
							if (!matchedPattern && pattern.pattern.exec(fileName)) {
								matchedPattern = pattern;
							}
						});
					}
					if (matchedPattern) {
						insertions.push({
							file: fileName,
							index: matchedPattern.index
						});
					}
				});
				insertions.forEach(function(insertion, index) {
					if (!matchInsertionCount['insert-'+insertion.index]) {
						group.files.splice(insertion.index, 1, insertion.file);
					} else {
						group.files.splice(insertion.index+matchInsertionCount['insert-'+insertion.index], 0, insertion.file);
					}
					matchInsertionCount['insert-'+insertion.index] += 1;
				});
				fileFetchCallback(null, true);
			});
		});
	}, function(err, contents) {
		settings.forEach(function (group, groupName) {
			if (!group.stale) {
				group.files.forEach(function (file, index) {
					if (!file.match) {
						console.log('No match for: '+file);
						group.files.splice(index, 1);
						return;
					}
					if (file.match(/^https?:\/\//)) {
						return;
					}
					fs.watchFile(group.path + file, function (old, newFile) {
						if (old.mtime.toString() != newFile.mtime.toString()) {
							self.generateCache(groupName);
						}
					});
				});
			}
		});
		self.generateCache();
	});

	this.generateCache = function (generateGroup) {
		var self = this;
		settings.forEach(function (group, groupName) {
			var userAgentMatches = {};
			if (group.preManipulate) {
				Object.keys(group.preManipulate).forEach(function(key) {
					userAgentMatches[key] = true;
				});
			}
			if (group.postManipulate) {
				Object.keys(group.postManipulate).forEach(function(key) {
					userAgentMatches[key] = true;
				});
			}
			if (!Object.keys(userAgentMatches).length) {
				userAgentMatches = ['^'];
			} else {
				userAgentMatches = Object.keys(userAgentMatches);
			}

			userAgentMatches.forEach(function(match) {
				var path = group.path;

				// Concatenate the contents of a group.
				var concatContents = function(contents) {
					if (group.dataType === 'javascript') {
						return contents.join(';\n');
					} else {
						return contents.join('\n');
					}
				}

				Step(function () {
					var grouping = this.group();
					group.files.forEach(function (file) {
						if (!generateGroup || generateGroup && groupName === generateGroup) {
							self.getFile(file, path, groupName, grouping());
						}
					});
				}, function (err, contents) {
					if (err) throw err;

					var grouping = this.group();
					var lastModified = null;

					for (var i = 0, l = contents.length; i < l; i++) {
						var file = contents[i];
						if (typeof file == "string"){
							continue;
						}
						if (typeof file.modified != "undefined"){
							file.modified = new Date();
						}
						if (Object.prototype.toString.call(file.modified) === "[object Date]" && !isNaN(file.modified)){
							
						} else {
							file.modified = new Date();
						}
						if (!lastModified || lastModified.getTime() < file.modified.getTime()) {
							lastModified = file.modified;
						}
						if (!group.preManipulate) {
							group.preManipulate = {};
						}

						self.manipulate(group.preManipulate[match], file.content, file.filePath, i, i === l - 1, grouping());
					};
					if (!lastModified && !contents.length) {
						grouping();
						return;
					}
					cacheTimestamps[groupName] = lastModified.getTime();
					if (!cache[groupName]) {
						cache[groupName] = {};
					}
					cache[groupName][match] = {
						'modified': lastModified.toUTCString()
					};
				}, function (err, contents) {
					if (err) throw err;
					var grouping = this.group();

					var content = concatContents(contents);
					var dataTypeLowerCase = group.dataType.toLowerCase();
					if (!group.debug) {
						if (dataTypeLowerCase === 'javascript' || dataTypeLowerCase === 'js') {
							(function (callback){callback(null, jsmin(content));})(grouping());
						} else if (dataTypeLowerCase === 'html') {
							(function (callback){callback(null, htmlmin(content));})(grouping());
						} else if (dataTypeLowerCase === 'css') {
							(function (callback){callback(null, cssmin(content));})(grouping());
						}
					} else {
						grouping()(null, content);
					}
				}, function (err, contents) {
					if (err) throw err;

					var grouping = this.group();
					var content = concatContents(contents);
					if (!group.postManipulate) {
						group.postManipulate = {};
					}
					self.manipulate(group.postManipulate[match], content, null, 0, true, grouping());

				}, function (err, contents) {
					if (err) throw err;

					var content = concatContents(contents);
					cacheHashes[groupName] = crypto.createHash('md5').update(content).digest('hex');

					cache[groupName][match].encodings = {};
					var encodings = cache[groupName][match].encodings;

					var utf8Buffer = new Buffer(content, 'utf8');
					encodings.utf8 = {
						'buffer': utf8Buffer,
						'length': utf8Buffer.length,
						'encoding': false
					};

					if(zlib) {
						var gzipBuffer = zlib.gzip(utf8Buffer, function(error, result) {
							encodings.gzip = {
								'buffer': result,
								'length': result.length,
								'encoding': 'gzip'
							};
						});
					} 
				});
			});
		});
	};

	this.manipulate = function (manipulateInstructions, fileContent, path, index, last, callback) {
		if (manipulateInstructions && Array.isArray(manipulateInstructions)) {
				var callIndex = 0;
				(function modify(content, path, index, last) {
					if (callIndex < manipulateInstructions.length) {
						callIndex++;
						manipulateInstructions[callIndex-1](content, path, index, last, function (content) {
							modify(content, path, index, last);
						});
					} else {
						callback(null, content);
					}
				})(fileContent, path, index, last);
		} else if (manipulateInstructions && typeof manipulateInstructions === 'function') {
			manipulateInstructions(fileContent, path, index, last, callback);
		} else {
			callback(null, fileContent);
		}
	};

	this.getFile = function (file, path, groupName, callback) {
		var isExternal = false;
		if (file && file.match(/^https?:\/\//)) {
			isExternal = true;
		}

		var fileInfo = {
			'filePath': isExternal ? file: path+file
		};

		if (isExternal) {
			request({uri: file}, function(err, res, body) {
				fileInfo.content = body;
				fileInfo.external = true;
				if (typeof res != "undefined" && res != null){
					fileInfo.modified = new Date(res.headers['last-modified']);
				}
				callback(null, fileInfo);
			});
		} else {
			setTimeout(function() {
				fs.readFile(path+file, function (err, data) {
					if (err) {
						console.log('Could not find: '+file);
						callback(null, '');
						return;
					}
					fileInfo.content = data.toString();

					fs.stat(path+file, function (err, stat) {
						fileInfo.modified = stat.mtime;
						callback(null, fileInfo);
					});
				});
			}, 100);
		}
	};

	this.acceptsGzip = function(req) {
		var accept = req.headers["accept-encoding"];
		return accept && accept.toLowerCase().indexOf('gzip') !== -1;
	}

	function assetManager (req, res, next) {
		var self = this;
		var found = false;
		var response = {};
		var mimeType = 'text/plain';
		var groupServed;
		settings.forEach(function (group, groupName) {
			if (group.route.test(req.url)) {
				var userAgent = req.headers['user-agent'] || '';
				groupServed = group;
				if (group.dataType === 'javascript') {
					mimeType = 'application/javascript';
				}
				else if (group.dataType === 'html') {
					mimeType = 'text/html';
				}
				else if (group.dataType === 'css') {
					mimeType = 'text/css';
				}
				if (cache[groupName]) {
					Object.keys(cache[groupName]).forEach(function(match) {
						if (!found && userAgent.match(new RegExp(match, 'i'))) {
							found = true;
							var item = cache[groupName][match];

							var content = item.encodings.utf8;
							if(zlib && item.encodings.gzip && this.acceptsGzip(req)) {
								content = item.encodings.gzip;
							}
							
							response = {
								contentLength: content.length
								, modified: item.modified
								, contentBuffer: content.buffer
								, encoding: content.encoding
							};
						}
					});
				}
			}
		});
		
		if (!found) {
			next();
		} else {
			if (groupServed.serveModify) {
				groupServed.serveModify(req, res, response, function(response) {
					serveContent(response);
				});
			} else {
				serveContent(response);
			}
			function serveContent(response) {
				var headers = {
					'Content-Type': mimeType,
					'Content-Length': response.contentLength,
					'Last-Modified': response.modified,
					'Date': (new Date).toUTCString(),
					'Cache-Control': 'public,max-age=' + 31536000,
					'Expires': response.expires || (new Date(new Date().getTime()+63113852000)).toUTCString(),
					'Vary': 'Accept-Encoding'
				};

				if(response.encoding) {
					headers['Content-Encoding'] = response.encoding
				}

				if (req.headers['if-modified-since'] &&
					Date.parse(req.headers['if-modified-since']) >= Date.parse(response.modified)) {
					res.writeHead(304, headers);
					res.end();
				} else {
					res.writeHead(200, headers);
					res.end(response.contentBuffer);
				}
			}
			return;
		}
	};

	assetManager.cacheTimestamps = cacheTimestamps;
	assetManager.cacheHashes = cacheHashes;

	return assetManager;
};
                topic: function(cl) {
                    var cb = this.callback,
                        users = {jack: {}, jill: {}},
                        postnote;

                    Step(
                        function() {
                            register(cl, "jack", "up|the|hill", this.parallel());
                            register(cl, "jill", "pail/of/water", this.parallel());
                        },
                        function(err, user1, user2) {
                            if (err) throw err;
                            users.jack.profile = user1.profile;
                            users.jill.profile = user2.profile;
                            accessToken(cl, {nickname: "jack", password: "******"}, this.parallel());
                            accessToken(cl, {nickname: "jill", password: "******"}, this.parallel());
                        },
                        function(err, pair1, pair2) {
                            if (err) throw err;
                            users.jack.pair = pair1;
                            users.jill.pair = pair2;

                            var act = {
                                verb: "follow",
                                object: {
                                    objectType: "person",
                                    id: users.jill.profile.id
                                }
                            },
                                url = "http://localhost:4815/api/user/jack/feed",
                                cred = makeCred(cl, users.jack.pair);

                            httputil.postJSON(url, cred, act, this);
                        },
                        function(err, posted, result) {
                            if (err) throw err;
                            var act = {
                                verb: "post",
                                to: [{
                                    id: "http://localhost:4815/api/user/jill/followers",
                                    objectType: "collection"
                                }],
                                object: {
                                    objectType: "note",
                                    content: "Hello, world."
                                }
                            },
                                url = "http://localhost:4815/api/user/jill/feed",
                                cred = makeCred(cl, users.jill.pair);
                            httputil.postJSON(url, cred, act, this);
                        },
                        function(err, posted, result) {
                            if (err) throw err;
                            postnote = posted;
                            var url = "http://localhost:4815/api/user/jack/inbox",
                                cred = makeCred(cl, users.jack.pair),
                                callback = this;

                            // Need non-zero time for async distribution
                            // to work. 2s seems reasonable for unit test.

                            setTimeout(function() {
                                httputil.getJSON(url, cred, callback);
                            }, 2000);
                        },
                        function(err, doc, result) {
                            if (err) {
                                cb(err, null, null);
                            } else {
                                cb(null, doc, postnote);
                            }
                        }
                    );
                },
Example #7
0
 'twitter/callback': function (request, response, store) {
 
   var service = _getService('twitter');
   step(
     function () {
     
       _log.trace (request.url);
       
       var twitterCallback = url.parse(request.url, true).query;
       
       _log.trace (twitterCallback);
       
       var token = twitterCallback.oauth_token;
       var verifier = twitterCallback.oauth_verifier;
       
       var secret = _tokens[token];
       
       _log.trace (token + ', ' + secret + ', ' + verifier);
       service.getOAuthAccessToken(token, secret, verifier, this);
       
       delete _tokens[token];
     
     },
     function (error, oauth_token, oauth_token_secret, results) {
       _log.trace (error);
       _log.trace (results);
       
       if (!error) {
       
         service.getProtectedResource(
           'https://api.twitter.com/1/account/verify_credentials.json',
           'GET', oauth_token, oauth_token_secret, this);
         
       
       } else {
         response.writeHead(302, {
           'Location': '/'
         });
         response.end();
       }
     },
     function (error, data, serviceResponse) {
     
       if (!error) {
       
         var user = JSON.parse(data);
         _log.trace(user);
                                    
         var users = model.get('user', store.getClient());      
         users.find({username: user.screen_name}, 
           function (items) {
                                    
             if (items.length == 0) {
               _log.debug ('Create user: '******'Location': '/index.html#user/login/' + user._id });
                   response.end('');                               
                 });
             } else {
               // Get first user and return to index
               _log.debug (items[0]);
                                          
               response.writeHead(302, {
                 'Location': '/index.html#user/login/' + items[0]._id });
               response.end('');                               
             }
                                      
           });
           // End verify credentials          
       
       } else {
         response.writeHead(302, {
           'Location': '/'
         });
         response.end();
       }
     
     }
   );
     
 },
Example #8
0
var step=require('step');
var fs = require('fs');

step(
  // Loads two files in parallel
  function loadStuff() {
    fs.readFile(__filename, this.parallel());
   fs.readFile("F:/weixin/pa-wxs/app.js", this.parallel());
  },
  // Show the result when done
  function showStuff(err, code, users) {
    if (err) throw err;
    console.log(code);
    console.log(users);
  }
)
Example #9
0
    return function(context, principal, args, callback) {
        var base = relmaker(context),
            url = URLMaker.makeURL(base),
            collection = {
                displayName: titlemaker(context),
                objectTypes: ["collection"],
                url: url,
                links: {
                    first: {
                        href: url
                    },
                    self: {
                        href: url
                    }
                },
                items: []
            };

        if (!callback) {
            callback = args;
            args = {start: 0, end: 20};
        }

        var lists, stream;

        Step(
            function() {
                streammaker(context, this);
            },
            function(err, result) {
                if (err) throw err;
                stream = result;
                stream.count(this);
            },
            function(err, totalItems) {
                var filtered;
                if (err) throw err;
                collection.totalItems = totalItems;
                if (totalItems === 0) {
                    if (_.has(collection, "author")) {
                        collection.author.sanitize();
                    }
                    callback(null, collection);
                    return;
                }
                if (!principal) {
                    filtered = new FilteredStream(stream, idPublicOnly(Collection.type));
                } else {
                    filtered = new FilteredStream(stream, idRecipientsOnly(principal, Collection.type));
                }

                if (_(args).has("before")) {
                    filtered.getIDsGreaterThan(args.before, args.count, this);
                } else if (_(args).has("since")) {
                    filtered.getIDsLessThan(args.since, args.count, this);
                } else {
                    filtered.getIDs(args.start, args.end, this);
                }
            },
            function(err, ids) {
                if (err) {
                    if (err.name == "NotInStreamError") {
                        throw new HTTPError(err.message, 400);
                    } else {
                        throw err;
                    }
                }
                Collection.readArray(ids, this);
            },
            function(err, results) {
                var group = this.group();
                if (err) throw err;
                lists = results;
                _.each(lists, function(list) {
                    list.expandFeeds(group());
                });
            },
            function(err) {
                if (err) throw err;
                finisher(context, principal, collection, this);
            },
            function(err) {
                if (err) {
                    callback(err, null);
                } else {
                    _.each(lists, function(item) {
                        item.sanitize();
                    });
                    collection.items = lists;
                    if (lists.length > 0) {
                        collection.links.prev = {
                            href: collection.url + "?since=" + encodeURIComponent(lists[0].id)
                        };
                        if ((_(args).has("start") && args.start + lists.length < collection.totalItems) ||
                            (_(args).has("before") && lists.length >= args.count) ||
                            (_(args).has("since"))) {
                            collection.links.next = {
                                href: collection.url + "?before=" + encodeURIComponent(lists[lists.length-1].id)
                            };
                        }
                    }
                    if (_.has(collection, "author")) {
                        collection.author.sanitize();
                    }
                    callback(null, collection);
                }
            }
        );
    };
Example #10
0
    return function(context, principal, args, callback) {

        var str,
            base = relmaker(context),
            url = URLMaker.makeURL(base),
            collection = {
                displayName: titlemaker(context),
                url: url,
                items: [],
                links: {
                    first: {
                        href: url
                    },
                    self: {
                        href: url
                    }
                }
            };

        if (!callback) {
            callback = args;
            args = {start: 0, end: 20};
        }

        Step(
            function() {
                streammaker(context, this);
            },
            function(err, results) {
                if (err) throw err;
                if (!principal) {
                    // XXX: keep a separate stream instead of filtering
                    str = new FilteredStream(results, objectPublicOnly);
                } else if (context.author && context.author.id == principal.id) {
                    str = results;
                } else {
                    str = new FilteredStream(results, objectRecipientsOnly(principal));
                }
                str.count(this.parallel());
            },
            function(err, count) {
                var type;
                if (err) throw err;
                collection.totalItems = count;
                if (count === 0) {
                    this(null, []);
                } else {
                    type = context.type;
                    if (type && _.has(args, "before")) {
                        str.getObjectsGreaterThan({id: args.before, objectType: type}, args.count, this.parallel());
                    } else if (type && _.has(args, "since")) {
                        str.getObjectsLessThan({id: args.since, objectType: type}, args.count, this.parallel());
                    } else {
                        str.getObjects(args.start, args.end, this.parallel());
                    }
                }
            },
            function(err, refs) {
                var group = this.group();
                if (err) throw err;
                _.each(refs, function(ref) {
                    ActivityObject.getObject(ref.objectType, ref.id, group());
                });
            },
            function(err, objs) {
                var group = this.group();
                if (err) throw err;
                collection.items = objs;
                _.each(collection.items, function(obj) {
                    obj.expandFeeds(group());
                });
            },
            function(err) {
                if (err) throw err;
                finisher(context, principal, collection, this);
            },
            function(err) {
                if (err) {
                    callback(err, null);
                } else {
                    callback(null, collection);
                }
            }
        );
    };
Example #11
0
    return function(context, principal, args, callback) {

        var str,
            base = relmaker(context),
            url = URLMaker.makeURL(base),
            collection = {
                displayName: titlemaker(context),
                url: url,
                objectTypes: ["person"],
                items: []
            };

        if (!callback) {
            callback = args;
            args = {start: 0, end: 20};
        }

        Step(
            function() {
                streammaker(context, this);
            },
            function(err, results) {
                if (err) throw err;
                if (args.q) {
                    str = new FilteredStream(results, Person.match(args.q));
                } else {
                    str = results;
                }
                str.count(this);
            },
            function(err, count) {
                if (err) {
                    if (err.name == "NoSuchThingError") {
                        collection.totalItems = 0;
                        this(null, []);
                    } else {
                        throw err;
                    }
                } else {
                    collection.totalItems = count;
                    if (_(args).has("before")) {
                        str.getIDsGreaterThan(args.before, args.count, this);
                    } else if (_(args).has("since")) {
                        str.getIDsLessThan(args.since, args.count, this);
                    } else {
                        str.getIDs(args.start, args.end, this);
                    }
                }
            },
            function(err, ids) {
                if (err) throw err;
                if (ids.length === 0) {
                    this(null, []);
                } else {
                    Person.readArray(ids, this);
                }
            },
            function(err, people) {
                if (err) throw err;

                collection.items = people;

                finisher(context, principal, collection, this);
            },
            function(err) {

                if (err) {

                    callback(err, null);

                } else {

                    _.each(collection.items, function(person) {
                        person.sanitize();
                    });

                    collection.links = {
                        self: {
                            href: URLMaker.makeURL(base, {offset: args.start, count: args.count})
                        },
                        current: {
                            href: URLMaker.makeURL(base)
                        }
                    };

                    if (collection.items.length > 0)  {
                        collection.links.prev = {
                            href: URLMaker.makeURL(base,
                                                   {since: collection.items[0].id})
                        };
                        if (collection.totalItems > collection.items.length &&
                            (!_.has(args, "start") || collection.totalItems > (args.start + collection.items.length))) {
                            collection.links.next = {
                                href: URLMaker.makeURL(base,
                                                       {before: collection.items[collection.items.length - 1].id})
                            };
                        }
                    }

                    if (_.has(collection, "author")) {
                        collection.author.sanitize();
                    }

                    callback(null, collection);
                }
            }
        );
    };
Example #12
0
    function sendResult(err, data) {
      if (err) {
        err.type = 'ServerError';
        callback(err);
        return;
      }

      var top = data.chain.length ?
        data.chain[data.chain.length-1].chainHash :
        Util.NULL_HASH;

      var start = 0;
      loop:
      if (txLocator.length) {
        for (var i = data.chain.length-1; i >= 0; i--) {
          for (var j = 0; j < txLocator.length; j++) {
            if (data.chain[i].hash.compare(txLocator[j]) == 0) {
              start = i;
              break loop;
            }
          }
        }
      }

      var txs = data.chain.slice(i, MAX_PER_REQUEST);

      var txsHashes = txs.map(function (val) {
        return val.hash;
      });
      Step(
        function loadTransactions() {
          console.log("loading transactions");
          storage.Transaction.find({_id: {"$in": txsHashes}}, this);
        },
        function loadBlocks(err, txData) {
          if (err) throw err;

          var txIndex = {};
          txData.forEach(function (tx) {
            txIndex[tx.hash.toString('base64')] = tx;
          });

          if (!txs.length) {
            this(null);
          }

          for (var i = 0, l = txs.length; i < l; i++) {
            var cb = this.parallel();

            storage.Block.findOne(
              {txs: txs[i].hash, active: 1},
              function (i, err, block) {
                if (err) {
                  // Query error
                  txs[i] = null;
                  cb(err);
                  return;
                } else if (!block) {
                  // No result (means corrupted db, ignore transaction)
                  txs[i] = null;
                  cb(null);
                  return;
                } else {
                  // Success

                  // Create final tx metadata object
                  var dbObj = txIndex[txs[i].hash.toString('base64')];
                  txs[i] = self.createOutTx(dbObj, txs[i], block);
                  cb(null);
                }
              }.bind(this, i)
            );
          }
        },
        function finalize(err) {
          if (err) {
            err.type = 'ServerError';
            callback(err);
            return;
          }

          callback(null, {
            top: top.toString('base64'),
            txs: txs,
            height: +blockChain.getTopBlock().height
          });
        }
      );
    };
Example #13
0
        function (err, txs) {
        try {
          if (err) throw err;
          console.log("txcoint", txs.length);

          var data = new PubkeysData();

          pubKeyHashes.forEach(function (pubKeyHash) {
            // Set up events for new transactions
            var hash64 = pubKeyHash.toString('base64');
            blockChain.addListener('txAdd:'+hash64, addTxToChain.bind(global, data));
            blockChain.addListener('txRevoke:'+hash64, revokeTxFromChain.bind(global, data));
          });

          data.accounts = pubKeyHashes.map(function (pubKeyHash) {
            return {pubKeyHash: pubKeyHash};
          });
          data.chain = [];

          var txsArray = [];
          var accIndex = {};

          Step(
            function loadBlockMetainfo() {
              // If there are no transactions, skip this step. This is
              // needed because Step hangs if no this.parallel() is called.
              if (!txs.length) {
                this(null);
                return;
              }

              var parallel = this.parallel;
              txs.forEach(function (tx) {
                storage.Block.findOne({txs: tx.hash, active: 1}, parallel());
              });
            },
            function processBlockMetaInfo() {
              var blocks = Array.prototype.slice.apply(arguments);
              var err = blocks.shift();

              if (err) throw err;

              txs = txs.map(function (tx, i) {
                for (var j = 0, l = blocks[i].txs.length; j < l; j++) {
                  if (blocks[i].txs[j].compare(tx.hash) == 0) {
                    break;
                  }
                }
                return {
                  tx: tx,
                  height: blocks[i].height,
                  index: j
                };
              });
              this(null);
            },
            function generateChain() {
              if (err) throw err;

              // Sort transactions by height, then index
              txs.sort(function (a,b) {
                if (a.height == b.height) {
                  return a.index - b.index;
                } else {
                  return a.height - b.height;
                }
              });

              // Create a chain with only unique values
              var curHash, lastHash, chainHash;
              for (var i = 0; i < txs.length; i++) {
                curHash = txs[i].tx.hash;
                // Add first tx
                if (i == 0) {
                  chainHash = curHash;

                  // Add only unique txs
                  // (we only need to check against the last one as they are sorted)
                } else if (lastHash.compare(curHash) != 0) {
                  chainHash = Util.sha256(chainHash.concat(curHash));
                } else {
                  continue;
                }
                data.chain.push({
                  hash: curHash,
                  chainHash: chainHash,
                  height: txs[i].height,
                  index: txs[i].index
                });
                lastHash = curHash;
              }

              console.log(data);

              this(null, data);
            },
            callback
          );
        } catch (err) {
          callback(err);
        }
      });
Example #14
0
// npm install step
var fs = require('fs'),
Step = require('step');
try {
    Step (
        function readData() {
            fs.readFile('./data/data1.txt', 'utf8', this);
        },
        function modify(err, text) {
            if (err) throw err;
            return text.replace(/somecompany\.com/g,'burningbird.net');
        },
        function writeData(err, text) {
            if (err) throw err;
            fs.writeFile('./data/data1.txt', text, this);
        }
    );
} catch(err) {
    console.error(err);
}
                topic: function(cl) {
                    var callback = this.callback,
                        pairs,
                        list,
                        act,
                        q = new Queue(25);
                    
                    Step(
                        function() {
                            var i, group = this.group();
                            for (i = 0; i < 150; i++) {
                                q.enqueue(newPair, [cl, "robot"+i, "bad*password*"+i], group());
                            }
                        },
                        function(err, results) {
                            var cred;
                            if (err) throw err;
                            pairs = results;
                            cred = makeCred(cl, pairs[0]);
                            httputil.postJSON("http://localhost:4815/api/user/robot0/feed",
                                              cred,
                                              {
                                                  verb: "create",
                                                  object: {
                                                      objectType: "collection",
                                                      objectTypes: ["person"], // robots are people, too
                                                      displayName: "Robots"
                                                  }
                                              },
                                              this);
                        },
                        function(err, act) {
                            var group = this.group(),
                                cred;
                            if (err) throw err;
                            list = act.object;
                            cred = makeCred(cl, pairs[0]);
                            _.each(_.pluck(pairs.slice(1), "user"), function(user) {
                                q.enqueue(httputil.postJSON,
                                          ["http://localhost:4815/api/user/robot0/feed",
                                           cred,
                                           {
                                               verb: "add",
                                               object: user.profile,
                                               target: list
                                           }
                                          ],
                                          group());
                                
                            });
                        },
                        function(err, responses) {
                            var cred = makeCred(cl, pairs[0]);
                            if (err) throw err;
                            httputil.postJSON("http://localhost:4815/api/user/robot0/feed",
                                              cred,
                                              {
                                                  verb: "post",
                                                  to: [list],
                                                  object: {
                                                      objectType: "note",
                                                      content: "Cigars are evil; you won't miss 'em."
                                                  }
                                              },
                                              this
                                             );
                        },
                        function(err, body, resp) {
                            var cb = this;
                            if (err) throw err;
                            act = body;
                            setTimeout(function() { cb(null); }, 5000);
                        },
                        function(err) {
                            var group = this.group();
                            _.each(pairs.slice(1), function(pair) {
                                var user = pair.user,
                                    cred = makeCred(cl, pair);

                                q.enqueue(httputil.getJSON,
                                          ["http://localhost:4815/api/user/"+user.nickname+"/inbox",
                                           cred],
                                          group());
                            });
                        },
                        function(err, feeds) {
                            callback(err, feeds, act);
                        }
                    );
                },
Example #16
0
        function(err, results) {
            if(err) {
                throw "Error inserting posts into DB: " + err.toString();;
            }

            debug("Done inserting posts");

            step(
                function() {
                    client.query("BEGIN", this);
                },
                function(err, results) {
                    client.query(
                        "select count(*) from post_ranks",
                        this
                    );
                },
                function(err, results) {
                    if(results.rows[0].count == 0) {
                        debug("Backfilling");
                        backfill_data(this);
                    } else {
                        debug("Not backfilling");
                        setTimeout(this, 0);
                    }
                },
                function(err, results) {
                    debug("Inserting new ranks into the DB");

                    // Store post ranks for this time period
                    var db_timestamp = get_db_timestamp(new Date());

                    var group = this.group();
                    for(var rank = 0; rank < exports.max_rank; rank++) {
                        (function(rank, group) {
                            debug(_s.sprintf("Inserting: Rank: %d, day: %d, hour: %d", rank, db_timestamp.day, db_timestamp.hour));
                            client.query(
                                "insert into post_ranks (" +
                                    "select " +
                                        "posts.post_id, " +
                                        "row_number() over (order by points desc) as rank, " +
                                        "$1 as use_date, " +
                                        "$4 as max_rank " +
                                    "from posts " +
                                    "left outer join (" +
                                        "select * from post_ranks " +
                                        "where " +
                                            "to_char(use_date, 'D') = $2 " +
                                            "and to_char(use_date, 'HH24') = $3 " +
                                    ") as post_ranks " +
                                    "on posts.post_id=post_ranks.post_id " +
                                    "where post_ranks.post_id is null " +
                                    "order by posts.points desc " +
                                    "limit $4" +
                                ")",
                                [db_timestamp.date, db_timestamp.day, db_timestamp.hour, rank],
                                group
                            );
                        })(rank, group());
                    }
                },
                function(err, results) {
                    if(err) {
                        console.log("Error inserting new ranks into the database: " + err.toString());
                    } else {
                        debug("Success");
                    }

                    client.query("END", this);
                }
            );
        }
Example #17
0
exports.refresh_data = function() {
    var start_date = new Date();
    start_date.setTime(start_date.getTime() - 1000 * 60 * 60 * 24 * 14);
    var max_posts = 1000;
    var limit = 100;
    var query_str = "http://api.thriftdb.com/api.hnsearch.com/items/_search?filter[fields][create_ts]=[%s TO *]&filter[fields][type]=submission&pretty_print=true&sortby=points desc&limit=%d&start=%d";
    var start_index = 0;

    step(
        function() {
            // Get posts from HN Search
            var group = this.group();
            while(start_index + limit <= max_posts) {
                (function(group) {
                    requester.get(
                        _s.sprintf(
                            query_str, start_date.toISOString(),
                            limit,
                            start_index
                        ), 
                        function(body) {
                            group(null, body);
                        }
                    );
                })(group());
                start_index += limit;
            }
        },
        function(err, posts_arrays) {
            for(var posts_array in posts_arrays) {
                var posts = posts_arrays[posts_array];
                var resp = JSON.parse(posts);
                posts = resp.results;

                var group = this.group();
                for(var post in posts) {
                    if(!posts.hasOwnProperty(post)) continue;

                    (function(post, group) {
                        // Upsert posts
                        step(
                            function() {
                                debug(_s.sprintf("Inserting into posts: %d - %s", post.id, post.title));

                                client.query(
                                    "update posts set points=$1, title=$2, num_comments=$3 where post_id=$4",
                                    [post.points, post.title, post.num_comments, post.id],
                                    this
                                );
                            },
                            function(err, results) {
                                if(err) {
                                    console.log("Error inserting new posts into the database: " + err.toString());
                                }

                                client.query(
                                    "insert into posts (" +
                                        "post_id, " +
                                        "points, " +
                                        "title, " +
                                        "domain, " +
                                        "username, " +
                                        "url, " +
                                        "num_comments, " +
                                        "creation_date" +
                                    ") select $1, $2, $3, $4, $5, $6, $7, $8 where not exists (select 1 from posts where post_id=$1)",
                                    [
                                        post.id,
                                        post.points,
                                        post.title,
                                        post.domain,
                                        post.username,
                                        post.url,
                                        post.num_comments,
                                        new Date(post.create_ts)
                                    ],
                                    group
                                );
                            }
                        );
                    })(posts[post].item, group());
                }
            }
        },
        function(err, results) {
            if(err) {
                throw "Error inserting posts into DB: " + err.toString();;
            }

            debug("Done inserting posts");

            step(
                function() {
                    client.query("BEGIN", this);
                },
                function(err, results) {
                    client.query(
                        "select count(*) from post_ranks",
                        this
                    );
                },
                function(err, results) {
                    if(results.rows[0].count == 0) {
                        debug("Backfilling");
                        backfill_data(this);
                    } else {
                        debug("Not backfilling");
                        setTimeout(this, 0);
                    }
                },
                function(err, results) {
                    debug("Inserting new ranks into the DB");

                    // Store post ranks for this time period
                    var db_timestamp = get_db_timestamp(new Date());

                    var group = this.group();
                    for(var rank = 0; rank < exports.max_rank; rank++) {
                        (function(rank, group) {
                            debug(_s.sprintf("Inserting: Rank: %d, day: %d, hour: %d", rank, db_timestamp.day, db_timestamp.hour));
                            client.query(
                                "insert into post_ranks (" +
                                    "select " +
                                        "posts.post_id, " +
                                        "row_number() over (order by points desc) as rank, " +
                                        "$1 as use_date, " +
                                        "$4 as max_rank " +
                                    "from posts " +
                                    "left outer join (" +
                                        "select * from post_ranks " +
                                        "where " +
                                            "to_char(use_date, 'D') = $2 " +
                                            "and to_char(use_date, 'HH24') = $3 " +
                                    ") as post_ranks " +
                                    "on posts.post_id=post_ranks.post_id " +
                                    "where post_ranks.post_id is null " +
                                    "order by posts.points desc " +
                                    "limit $4" +
                                ")",
                                [db_timestamp.date, db_timestamp.day, db_timestamp.hour, rank],
                                group
                            );
                        })(rank, group());
                    }
                },
                function(err, results) {
                    if(err) {
                        console.log("Error inserting new ranks into the database: " + err.toString());
                    } else {
                        debug("Success");
                    }

                    client.query("END", this);
                }
            );
        }
    );

};
Example #18
0
    return function(context, principal, args, callback) {

        var base = relmaker(context),
            url = URLMaker.makeURL(base),
            collection = {
                displayName: titlemaker(context),
                objectTypes: ["activity"],
                url: url,
                links: {
                    first: {
                        href: url
                    },
                    self: {
                        href: url
                    }
                },
                items: []
            };

        var str, ids;

        // XXX: making assumptions about the context is probably bad

        if (context.user) {
            collection.author = context.user.profile;
        }

        // args are optional

        if (!callback) {
            callback = args;
            args = {start: 0, end: 20};
        }

        Step(
            function() {
                streammaker(context, this);
            },
            function(err, results) {
                if (err) {
                    if (err.name == "NoSuchThingError") {
                        collection.totalItems = 0;
                        collection.items      = [];
                        this(null, 0);
                    } else {
                        throw err;
                    }
                } else {
                    // Skip filtering if remote user == author
                    if (principal && collection.author && principal.id == collection.author.id) {
                        str = results;
                    } else if (!principal) {
                        // XXX: keep a separate stream instead of filtering
                        str = new FilteredStream(results, publicOnly);
                    } else {
                        str = new FilteredStream(results, recipientsOnly(principal));
                    }
                    str.count(this);
                }
            },
            function(err, totalItems) {
                if (err) throw err;
                collection.totalItems = totalItems;
                if (totalItems === 0) {
                    this(null, []);
                    return;
                }
                if (_(args).has("before")) {
                    str.getIDsGreaterThan(args.before, args.count, this);
                } else if (_(args).has("since")) {
                    str.getIDsLessThan(args.since, args.count, this);
                } else {
                    str.getIDs(args.start, args.end, this);
                }
            },
            function(err, ids) {
                if (err) {
                    if (err.name == "NotInStreamError") {
                        throw new HTTPError(err.message, 400);
                    } else {
                        throw err;
                    }
                }
                if (ids.length === 0) {
                    this(null, []);
                } else {
                    Activity.readArray(ids, this);
                }
            },
            function(err, activities) {
                if (err) throw err;
                activities = activities.filter(function(n){return n});
                activities.forEach(function(act) {
                    act.sanitize(principal);
                });
                collection.items = activities;
                if (activities.length > 0) {
                    collection.links.prev = {
                        href: collection.url + "?since=" + encodeURIComponent(activities[0].id)
                    };
                    if ((_(args).has("start") && args.start + activities.length < collection.totalItems) ||
                        (_(args).has("before") && activities.length >= args.count) ||
                        (_(args).has("since"))) {
                        collection.links.next = {
                            href: collection.url + "?before=" + encodeURIComponent(activities[activities.length-1].id)
                        };
                    }
                }
                if (finisher) {
                    finisher(principal, collection, this);
                } else {
                    this(null);
                }
            },
            function(err) {
                if (err) {
                    callback(err, null);
                } else {
                    if (_.has(collection, "author")) {
                        collection.author.sanitize();
                    }
                    callback(null, collection);
                }
            }
        );
    };
Example #19
0
ActivityObject.beforeCreate = function(props, callback) {

    var type = this.type;

    if (!_.has(props, "objectType")) {
        props.objectType = type;
    }

    var now = Stamper.stamp();

    props.published = props.updated = now;


    Step(
        function() {
            var User = require("./user").User;
            if (!_(props).has("id")) {
                props._uuid = IDMaker.makeID();
                props.id   = ActivityObject.makeURI(type, props._uuid);

                if (!_.has(props, "links")) {
                    props.links = {};
                }

                props.links.self = {
                    href: URLMaker.makeURL("api/" + type + "/" + props._uuid)
                };

                props.likes = {
                    url: URLMaker.makeURL("api/" + type + "/" + props._uuid + "/likes")
                };

                props.replies = {
                    url: URLMaker.makeURL("api/" + type + "/" + props._uuid + "/replies")
                };

                props.shares = {
                    url: URLMaker.makeURL("api/" + type + "/" + props._uuid + "/shares")
                };

                if (_.has(props, "author") &&
                    _.isObject(props.author)) {
                    if (_.has(props.author, "preferredUsername") &&
                        _.isString(props.author.preferredUsername)) {
                        props.url = URLMaker.makeURL([props.author.preferredUsername, type, props._uuid].join("/"));
                        this(null, null);
                    } else {
                        User.fromPerson(props.author.id, this);
                    }
                } else {
                    this(null, null);
                }

            } else {
                // For non-new stuff, clear out volatile data
                ActivityObject.trimCollection(props, "likes");
                ActivityObject.trimCollection(props, "replies");
                ActivityObject.trimCollection(props, "shares");
                if (_(props).has("replies")) {
                    if (_(props.replies).isObject()) {
                        delete props.replies.totalItems;
                        delete props.replies.items;
                    } else {
                        delete props.replies;
                    }
                }
                this(null, null);
            }
        },
        function(err, user) {
            if (err) throw err;
            if (user) {
                props.url = URLMaker.makeURL([user.nickname, type, props._uuid].join("/"));
            }
            // Save the author by reference; don't save the whole thing
            ActivityObject.compressProperty(props, "author", this.parallel());
            ActivityObject.compressProperty(props, "inReplyTo", this.parallel());
        },
        function(err) {
            if (err) {
                callback(err, null);
            } else {
                callback(null, props);
            }
        }
    );
};
Example #20
0
    function(context, principal, collection, callback) {

        var coll = context.collection,
            base = "/api/collection/"+coll._uuid+"/members",
            prevParams,
            nextParams,
            first,
            last;

        collection.author = coll.author;

        if (collection.author) {
            collection.author.sanitize();
        }

        if (collection.items.length > 0) {

            if (!collection.links) {
                collection.links = {};
            }

            first = collection.items[0];

            prevParams = {since: first.id};

            if (!collection.objectTypes ||
                collection.objectTypes.length != 1 ||
                first.objectType != collection.objectTypes[0]) {
                prevParams.type = first.objectType;
            }

            collection.links.prev = {
                href: URLMaker.makeURL(base, prevParams)
            };

            if (collection.items.length < collection.totalItems) {

                last = collection.items[collection.items.length - 1];

                nextParams = {before: last.id};

                if (!collection.objectTypes ||
                    collection.objectTypes.length != 1 ||
                    last.objectType != collection.objectTypes[0]) {
                    nextParams.type = last.objectType;
                }

                collection.links.next = {
                    href: URLMaker.makeURL(base, nextParams)
                };
            }
        }

        Step(
            function() {
                var group = this.group();
                _.each(collection.items, function(item) {
                    item.expand(group());
                });
            },
            function(err, expanded) {
                var followable;

                if (err) throw err;

                // Add the first few replies for each object

                firstFewReplies(principal, collection.items, this.parallel());

                // Add the first few shares for each object

                firstFewShares(principal, collection.items, this.parallel());

                // Add the first few "likers" for each object

                addLikers(principal, collection.items, this.parallel());

                // Add whether the current user likes the items

                addLiked(principal, collection.items, this.parallel());

                // Add the followed flag to applicable objects

                followable = _.filter(collection.items, function(obj) {
                    return obj.isFollowable();
                });

                addFollowed(principal, followable, this.parallel());

                // Add proxy URLs to applicable objects
                addProxyObjects(principal, collection.items, this.parallel());
            },
            callback
        );
    }
Example #21
0
  'menome/callback': function (request, response, store) {
    var code = url.parse(request.url, true).query.code;

    var service = _getService('menome');

    step(
      function init() {
        service.getOAuthAccessToken(code, null, this);
      },
      function gotAccessToken(error, access_token, refresh_token) {
        if (!error) {
          service.get('https://menomeapi.sunburn.in.th/1/user/user.json', access_token, this);    
        } else {
          response.writeHead(302, {
            'Location': '/'
          });
          response.end();
        }
      },
      function gotUser(error, result) {

        if (!error) {
          var user = JSON.parse(result);
          _log.trace(user);

          var users = model.get('user', store.getClient());      
          users.find({username: user.id}, 
            function (items) {

              if (items.length == 0) {
                _log.debug ('Create user: '******'Location': '/index.html#user/login/' + user._id });
                    response.end('');                               
                  });
              } else {
                // Get first user and return to index
                _log.debug (items[0]);

                response.writeHead(302, {
                  'Location': '/index.html#user/login/' + items[0]._id });
                response.end('');                               
              }

            });
            // End verify credentials   
        } else {
          response.writeHead(302, {
            'Location': '/'
          });
          response.end();
        }


      });

  },
Example #22
0
var redeemCode = function(req, res, next) {

    var code = req.body.code,
        recovery,
        user;

    Step(
        function() {
            Recovery.get(code, this);
        },
        function(err, results) {
            if (err) throw err;
            recovery = results;

            if (recovery.recovered) {
                throw new Error("This recovery code was already used.");
            }

            if (Date.now() - Date.parse(recovery.timestamp) > Recovery.TIMEOUT) {
                throw new Error("This recovery code is too old.");
            }

            User.get(recovery.nickname, this);
        },
        function(err, results) {
            if (err) throw err;
            user = results;
            setPrincipal(req.session, user.profile, this);
        },
        function(err) {
            if (err) throw err;
            user.expand(this);
        },
        function(err) {
            if (err) throw err;
            user.profile.expandFeeds(this);
        },
        function(err) {
            if (err) throw err;
            req.app.provider.newTokenPair(req.client, user, this);
        },
        function(err, pair) {
            if (err) throw err;

            user.token = pair.access_token;
            user.secret = pair.token_secret;

            // Now that we're done, mark this recovery code as done

            recovery.recovered = true;
            recovery.save(this);
        },
        function(err) {
            if (err) {
                req.log.error(err);
                res.status(400);
                res.json({error: err.message});
            } else {
                res.json(user);
            }
        }
    );
};
Example #23
0
			userAgentMatches.forEach(function(match) {
				var path = group.path;

				// Concatenate the contents of a group.
				var concatContents = function(contents) {
					if (group.dataType === 'javascript') {
						return contents.join(';\n');
					} else {
						return contents.join('\n');
					}
				}

				Step(function () {
					var grouping = this.group();
					group.files.forEach(function (file) {
						if (!generateGroup || generateGroup && groupName === generateGroup) {
							self.getFile(file, path, groupName, grouping());
						}
					});
				}, function (err, contents) {
					if (err) throw err;

					var grouping = this.group();
					var lastModified = null;

					for (var i = 0, l = contents.length; i < l; i++) {
						var file = contents[i];
						if (typeof file == "string"){
							continue;
						}
						if (typeof file.modified != "undefined"){
							file.modified = new Date();
						}
						if (Object.prototype.toString.call(file.modified) === "[object Date]" && !isNaN(file.modified)){
							
						} else {
							file.modified = new Date();
						}
						if (!lastModified || lastModified.getTime() < file.modified.getTime()) {
							lastModified = file.modified;
						}
						if (!group.preManipulate) {
							group.preManipulate = {};
						}

						self.manipulate(group.preManipulate[match], file.content, file.filePath, i, i === l - 1, grouping());
					};
					if (!lastModified && !contents.length) {
						grouping();
						return;
					}
					cacheTimestamps[groupName] = lastModified.getTime();
					if (!cache[groupName]) {
						cache[groupName] = {};
					}
					cache[groupName][match] = {
						'modified': lastModified.toUTCString()
					};
				}, function (err, contents) {
					if (err) throw err;
					var grouping = this.group();

					var content = concatContents(contents);
					var dataTypeLowerCase = group.dataType.toLowerCase();
					if (!group.debug) {
						if (dataTypeLowerCase === 'javascript' || dataTypeLowerCase === 'js') {
							(function (callback){callback(null, jsmin(content));})(grouping());
						} else if (dataTypeLowerCase === 'html') {
							(function (callback){callback(null, htmlmin(content));})(grouping());
						} else if (dataTypeLowerCase === 'css') {
							(function (callback){callback(null, cssmin(content));})(grouping());
						}
					} else {
						grouping()(null, content);
					}
				}, function (err, contents) {
					if (err) throw err;

					var grouping = this.group();
					var content = concatContents(contents);
					if (!group.postManipulate) {
						group.postManipulate = {};
					}
					self.manipulate(group.postManipulate[match], content, null, 0, true, grouping());

				}, function (err, contents) {
					if (err) throw err;

					var content = concatContents(contents);
					cacheHashes[groupName] = crypto.createHash('md5').update(content).digest('hex');

					cache[groupName][match].encodings = {};
					var encodings = cache[groupName][match].encodings;

					var utf8Buffer = new Buffer(content, 'utf8');
					encodings.utf8 = {
						'buffer': utf8Buffer,
						'length': utf8Buffer.length,
						'encoding': false
					};

					if(zlib) {
						var gzipBuffer = zlib.gzip(utf8Buffer, function(error, result) {
							encodings.gzip = {
								'buffer': result,
								'length': result.length,
								'encoding': 'gzip'
							};
						});
					} 
				});
			});
Example #24
0
var handleRecover = function(req, res, next) {

    var user = null,
        recovery,
        nickname = req.body.nickname,
        force = req.body.force;

    Step( 
        function () { 
            req.log.info({nickname: nickname}, "checking for user to recover");
            User.get(nickname, this);
        },
        function(err, result) {
            if (err) {
                if (err.name == "NoSuchThingError") {
                    req.log.info({nickname: nickname}, "No such user, can't recover");
                    res.status(400);
                    res.json({sent: false, noSuchUser: true, error: "There is no user with that nickname."});
                    return;
                } else {
                    throw err;
                }
            }
            user = result;
            if (!user.email) {
                req.log.info({nickname: nickname}, "User has no email address; can't recover.");
                // Done
                res.status(400);
                res.json({sent: false, noEmail: true, error: "This user account has no email address."});
                return;
            }
            if (force) {
                req.log.info({nickname: nickname}, "Forcing recovery regardless of existing recovery records.");
                this(null, []);
            } else {
                req.log.info({nickname: nickname}, "Checking for existing recovery records.");
                // Do they have any outstanding recovery requests?
                Recovery.search({nickname: nickname, recovered: false}, this);
            }
        },
        function(err, recoveries) {
            var stillValid;
            if (err) throw err;
            if (!recoveries || recoveries.length === 0) {
                req.log.info({nickname: nickname}, "No existing recovery records; continuing.");
                this(null);
                return;
            } 
            stillValid = _.filter(recoveries, function(reco) { return Date.now() - Date.parse(reco.timestamp) < Recovery.TIMEOUT; });
            if (stillValid.length > 0) {
                req.log.info({nickname: nickname, count: stillValid.length}, "Have an existing, valid recovery record.");
                // Done
                res.status(409);
                res.json({sent: false, existing: true, error: "You already requested a password recovery."});
            } else {
                req.log.info({nickname: nickname}, "Have old recovery records but they're timed out.");
                this(null);
            }
        },
        function(err) {
            if (err) throw err;
            req.log.info({nickname: nickname}, "Creating a new recovery record.");
            Recovery.create({nickname: nickname}, this);
        },
        function(err, recovery) {
            var recoveryURL;
            if (err) throw err;
            req.log.info({nickname: nickname}, "Generating recovery email output.");
            recoveryURL = URLMaker.makeURL("/main/recover/" + recovery.code);
            res.render("recovery-email-html",
                       {principal: user.profile,
                        principalUser: user,
                        recovery: recovery,
                        recoveryURL: recoveryURL,
                        layout: false},
                       this.parallel());
            res.render("recovery-email-text",
                       {principal: user.profile,
                        principalUser: user,
                        recovery: recovery,
                        recoveryURL: recoveryURL,
                        layout: false},
                       this.parallel());
        },
        function(err, html, text) {
            if (err) throw err;
            req.log.info({nickname: nickname}, "Sending recovery email.");
            Mailer.sendEmail({to: user.email,
                              subject: "Recover password for " + req.app.config.site,
                              text: text,
                              attachment: {data: html,
                                           type: "text/html",
                                           alternative: true}},
                             this);
        },
        function(err) {
            if (err) {
                next(err);
            } else {
                req.log.info({nickname: nickname}, "Finished with recovery");
                res.json({sent: true});
            }
        }
    );
};
Example #25
0
function processVersions( repo, fn ) {
	Step(
		// get all tags for the repo
		function() {
			pluginsDb.getTags( repo.id, this.parallel() );
			repo.getVersionTags( this.parallel() );
		},

		// filter to new versions
		function( error, processedTags, tags ) {
			if ( error ) {
				retry.log( "processVersions", repo.id );
				return fn( error );
			}

			return tags.filter(function( tag ) {
				return !(tag in processedTags);
			// only process up to 10 tags per run
			// this keeps the number of open file descriptors lower
			// it's unlikely that any update will have more than 10 tags
			}).slice( -10 );
		},

		// get releases
		function( error, tags ) {
			if ( error ) {
				retry.log( "processVersions", repo.id );
				return fn( error );
			}

			if ( !tags.length ) {
				logger.log( "No tags to process for " + repo.id );
				return fn( null );
			}

			logger.log( "Processing", repo.id, tags );
			this.parallel()( null, tags );
			var group = this.group();
			tags.forEach(function( tag ) {
				repo.getRelease( tag, group() );
			});
		},

		// filter to valid releases
		function( error, tags, releases ) {
			if ( error ) {
				retry.log( "processVersions", repo.id );
				return fn( error );
			}

			var releasesCb = this.parallel(),
				invalidGroup = this.group();
			releasesCb( null, releases.filter(function( release, i ) {
				if ( release ) {
					return true;
				}

				// track invalid tags so we don't process them on each update
				pluginsDb.addTag( repo.id, tags[ i ], invalidGroup() );
				return false;
			}));
		},

		// process the releases
		function( error, releases ) {
			if ( error ) {
				retry.log( "processVersions", repo.id );
				return fn( error );
			}

			if ( !releases.length ) {
				logger.log( "No valid releases for " + repo.id );
				return fn( null );
			}

			var group = this.group();
			releases.forEach(function( release ) {
				for ( var file in release.manifests ) {
					processRelease( repo, release.tag, file, release.manifests[ file ], group() );
				}
			});
		},

		function( error ) {
			fn( error );
		}
	);
}
Example #26
0
function _parse(content, options, callback) {

	if(typeof options == 'function') {

		callback     = options;
		options.test = false;

	}


	var on = outcome.error(callback),
	warnings,
	window;

	step(

		/**
		 * load it.
		 */

		function() {

			jsdom.env({
				html: content,
				scripts: [
			  		__dirname + "/jquery-1.5.js"
				],
				done: this
			});

		},


		/**
		 * set it.
		 */

		on.success(function(win) {


			window = win;
			_copyStyles(window);
			this();

		}),

		/**
		 * test it.
		 */

		on.success(function() {

			if(!options.test) {

				return this();

			}

			test(window, {}, this);

		}),

		/**
		 * clean it.
		 */

		on.success(function(warn) {

			warnings = warn;

			//strip stuff that cannot be processed
			window.$('script, link, style').remove();

            if (!options.comments) {
                //strip comments - sometimes gets rendered
                window.$('*').contents().each(function(node) {
                    if(this.nodeType == 8) {
                        window.$(this).remove();
                    }
                });
            }

			this();

		}),


		/**
		 * finish it.
		 */

		function(html) {

			callback(null, window.document.innerHTML, warnings || []);

		}
	);


}
Example #27
0
function checkMentions() {
	Step(
		function getUnreadCount() {
			weibo.getUnread(this);
		},
		function getUnreadMentions(unreadCount) {
			if (!unreadCount || unreadCount.mention_status === 0) {
				return;
			}
			weibo.getMentions(unreadCount.mention_status, this);
		},
		function createPieceFromMention(mentions) {
			if (!mentions || mentions.length === 0) {
				return;
			}
			for (var i = mentions.length - 1; i >= 0; i--) {
				var mention = mentions[i];
				var content;
				// check retweet
				if (mention.retweeted_status) {
					content = mention.retweeted_status.text;
				} else {
					content = truncateContent(mention.text);
				}
				var userWeiboId = mention.user.id;
				if (content) {
					createPiece(userWeiboId, mention.id, content, mention.retweeted_status);
				}
			}
			weibo.clearUnread();
		});

	function truncateContent(mention) {
		var result = /@\S+ (.+)/.exec(mention);
		if (result && result[1]) {
			return result[1];
		}
		result = /(.+)@\S+/.exec(mention);
		if (result && result[1]) {
			return result[1];
		}
		return null;
	}

	function createPiece(weiboId, mentionId, content, retweet) {
		checkUser(insertPiece);

		function checkUser(cb) {
			var connection = mysql.createConnection(config.mysql);
			connection.query(
				'SELECT * FROM user ' +
				'where weibo_id=' + weiboId,
				function selectCb(err, users, fields) {
					if (err) {
						throw err;
					}
					if (!users.length) {
						weibo.comment(mentionId, "请注册^^");
						return;
					}
					connection.end();
					cb(users[0]);
				}
			);
		}

		function insertPiece(user) {
			if (user) {
				var connection = mysql.createConnection(config.mysql);
				console.log("insert piece into db");
				connection.query(
					'INSERT into piece set ? ', {
						content: content,
						user: user.id,
						addtime: moment().format("YYYY-MM-DD hh:mm:ss"),
						link: retweet && "http://weibo.com/" + retweet.user.id + "/" + retweet.id
					}, function() {
						connection.end();
						weibo.comment(mention.id, "已添加至收藏~");
					})
			}
		}

	}
}
Example #28
0
    dtor.distribute = function(callback) {
        var actor = activity.actor,
            recipients = activity.recipients(),
            toRecipients = function(cb) {
                Step(
                    function() {
                        var i, group = this.group();
                        for (i = 0; i < recipients.length; i++) {
                            toRecipient(recipients[i], group());
                        }
                    },
                    cb
                );
            },
            toDispatch = function(cb) {
                Step(
                    function() {
                        User.fromPerson(actor.id, this);
                    },
                    function(err, user) {
                        if (err) throw err;
                        if (user) {
                            // Send updates
                            outboxUpdates(user);
                            // Also inbox!
                            inboxUpdates(user);
                        }
                        this(null);
                    },
                    cb
                );
            },
            toEmail = function(cb) {
                var direct = directRecipients(activity),
                    people = _.where(direct, {objectType: ActivityObject.PERSON});

                Step(
                    function() {
                        var group = this.group();

                        _.each(people, function(person) {

                            var user,
                                callback = group();

                            Step(
                                function() {
                                    User.fromPerson(person.id, this);
                                },
                                function(err, results) {
                                    if (err) throw err;
                                    user = results;
                                    if (!user) {
                                        callback(null);
                                        return;
                                    }
                                    if (!user.email) {
                                        callback(null);
                                        return;
                                    }
                                    user.expand(this);
                                },
                                function(err) {
                                    notifyByEmail(user, activity, callback);
                                }
                            );
                        });
                    },
                    cb
                );
            },
            toFirehose = function(cb) {

                var recipients = activity.recipients(),
                    pub = _.where(recipients, {id: Collection.PUBLIC});

                // If it's not a public activity, skip

                if (!pub || pub.length === 0) {
                    cb(null);
                    return;
                }

                // If the actor is a local user, ping the firehose

                Step(
                    function() {
                        User.fromPerson(actor.id, this);
                    },
                    function(err, user) {
                        if (err) {
                            cb(err);
                        } else if (!user) {
                            cb(null);
                        } else {
                            Firehose.ping(activity, cb);
                        }
                    }
                );
            };

        Step(
            function() {
                if (!expanded) {
                    actor.expandFeeds(this);
                    expanded = true;
                } else {
                    this(null);
                }
            },
            function(err) {
                if (err) throw err;
                toRecipients(this.parallel());
                toDispatch(this.parallel());
                toFirehose(this.parallel());
                toEmail(this.parallel());
            },
            callback
        );
    };
Example #29
0
step(
  function createScraperForCategories(){
    var options = {
      url : 'http://www.mon-panier-de-campagne.fr/produits',
      engine : 'cheerio'
    };
    scrapinode.createScraper(options, this);
  },
  function getAllCategories(err, scraper){
    if(err) throw err;
    categories = scraper.get('categories');
    return categories;
  },
  function createScapersForSubCategories(err, categories){
    if(err) throw err;
    var group = this.group();
    categories.forEach(function(category){
      var options = {
        url : category.url,
        engine : 'cheerio'
      };
      scrapinode.createScraper(options, group());
    });
  },
  function getAllSubCategories(err, scrapers){
    if(err) throw err;
    scrapers.forEach(function(scraper){
      subCategories = subCategories.concat(scraper.get('categories'));
    });
    return subCategories;
  },
  function createScapersForProducts(err, subCategories){
    if(err) throw err;
    var group = this.group();
    subCategories.forEach(function(subCategory){
      var options = {
        url : subCategory.url,
        engine : 'cheerio'
      };
      scrapinode.createScraper(options, group());
    });
  },
  function getAllProducts(err, scrapers){
    if(err) throw err;
    scrapers.forEach(function(scraper){
      products = products.concat(scraper.get('products'));
    });
    return products;
  },
  function finish(err, products){
    if(err) return console.error(err, err.stack);
    console.log(JSON.stringify(products));
  }
);
NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
    if (!!this.err || this.mapConfig !== null) {
        return callback(this.err, this.mapConfig, this.rendererParams, this.context);
    }

    var self = this;

    var mapConfig = null;
    var rendererParams;
    var apiKey;

    var context = {};

    step(
        function getTemplate() {
            self.getTemplate(this);
        },
        function prepareDbParams(err, tpl) {
            assert.ifError(err);
            self.template = tpl;

            rendererParams = _.extend({}, self.params, {
                user: self.owner
            });
            self.setDBParams(self.owner, rendererParams, this);
        },
        function getUserApiKey(err) {
            assert.ifError(err);
            self.metadataBackend.getUserMapKey(self.owner, this);
        },
        function prepareParams(err, _apiKey) {
            assert.ifError(err);

            apiKey = _apiKey;

            var templateParams = {};
            if (self.config) {
                try {
                    templateParams = _.isString(self.config) ? JSON.parse(self.config) : self.config;
                } catch (e) {
                    throw new Error('malformed config parameter, should be a valid JSON');
                }
            }

            return templateParams;
        },
        function instantiateTemplate(err, templateParams) {
            assert.ifError(err);
            return self.templateMaps.instance(self.template, templateParams);
        },
        function prepareAdapterMapConfig(err, requestMapConfig) {
            assert.ifError(err);
            context.analysisConfiguration = {
                db: {
                    host: rendererParams.dbhost,
                    port: rendererParams.dbport,
                    dbname: rendererParams.dbname,
                    user: rendererParams.dbuser,
                    pass: rendererParams.dbpassword
                },
                batch: {
                    username: self.owner,
                    apiKey: apiKey
                }
            };
            self.mapConfigAdapter.getMapConfig(self.owner, requestMapConfig, rendererParams, context, this);
        },
        function prepareContextLimits(err, _mapConfig) {
            assert.ifError(err);
            mapConfig = _mapConfig;
            self.userLimitsApi.getRenderLimits(self.owner, this);
        },
        function cacheAndReturnMapConfig(err, renderLimits) {
            self.err = err;
            self.mapConfig = (mapConfig === null) ? null : new MapConfig(mapConfig, context.datasource);
            self.analysesResults = context.analysesResults || [];
            self.rendererParams = rendererParams;
            self.context.limits = renderLimits || {};
            return callback(self.err, self.mapConfig, self.rendererParams, self.context);
        }
    );
};