cloudStorageUtils.putBuffer(data, s3Path, headers, true, false, function(err, res) {
if ( err ) {
winston.handleError(err);
} else {
winston.doInfo('putBuffer successful');
}
});
utils.streamToBuffer( parsedMailAttachment.stream, false, function(err, attachmentBuffer ) {
if ( err ) {
isDone = true;
callback( err );
} else {
parsedMailAttachment.myId = crypto.randomBytes (32).toString ('hex');
if ( ( ! parsedMailAttachment.myId ) || ( attachmentBuffers[parsedMailAttachment.myId] ) ) {
isDone = true;
callback( winston.makeError('invalid or duplicate id for attachment', {randomBytes: parsedMailAttachment.myId}) );
} else {
attachmentBuffers[parsedMailAttachment.myId] = attachmentBuffer;
if ( parsedMailComplete && ( Object.keys(attachmentBuffers).length == parsedMailComplete.attachments.length ) ) {
winston.doInfo('streamed attachments finished after mailparser');
isDone = true;
if ( attachmentHandler.addAttachmentBuffersAndValidate( parsedMailComplete, attachmentBuffers ) ) {
mailReader.runMailProcessor( message, parsedMailComplete, mailId, userId, mailProcessorCallback );
} else {
callback( winston.makeError('invalid attachments', {mailId: mailId}) );
}
}
}
}
});
mailParser.on('end', function( parsedMail ) {
if ( isDone ) {
winston.doWarn('mailparser ended after already done');
} else {
parsedMailComplete = parsedMail;
if ( ! mailReaderConstants.STREAM_ATTACHMENTS ) {
isDone = true;
mailReader.runMailProcessor( message, parsedMailComplete, mailId, userId, mailProcessorCallback );
} else {
if ( ( ! parsedMailComplete.attachments )
|| ( ! ( parsedMailComplete.attachments.length > 0 ) )
|| ( Object.keys(attachmentBuffers).length === parsedMailComplete.attachments.length ) ) {
isDone = true;
if ( attachmentHandler.addAttachmentBuffersAndValidate( parsedMailComplete, attachmentBuffers ) ) {
mailReader.runMailProcessor( message, parsedMailComplete, mailId, userId, mailProcessorCallback );
} else {
callback( winston.makeError('invalid attachments', {mailId: mailId}) );
}
} else {
winston.doWarn('mailparser ended but attachments are still outstanding', {mailId: mailId, attachmentBuffersLength: attachmentBuffers.length, parsedMailCompleteAttachmentsLength: parsedMailComplete.attachments.length});
}
}
}
});
exports.setMailReaderState = function( mailId, state, userId, callback ) {
winston.doInfo('mailReader: setMailReaderState...', {mailId : mailId, state : state});
if ( ! mailId ) { callback( winston.makeMissingParamError('mailId') ); return; }
if ( ! state ) { callback( winston.makeMissingParamError('state') ); return; }
var filter = {
_id: mailId
};
var updateSet = {$set: {
mailReaderState: state
}, $inc : {
tries : 1
}};
MailModel.update( filter, updateSet, function (err, num) {
if ( err ) {
callback( winston.makeMongoError(err) );
} else if ( num === 0 ) {
callback( winston.makeError('no updated mail', {mailId: mailId, state: state, userId: userId}) );
} else {
callback();
}
});
}
attachmentInfo.save (function (err) {
count+=1;
winston.doInfo('saved attachmentInfo', {count: count});
if (err) {
winston.doMongoError (err);
}
});
UserModel.findOne({email:userEmail}, function(mongoErr, foundUser) {
if ( mongoErr ) {
callback( winston.makeMongoError(mongoErr) );
} else if ( ! foundUser ) {
callback( winston.makeMissingParamError('user') );
} else {
var userId = foundUser._id;
var extraDays = new ExtraDaysModel({
userId: userId
, numExtraDays: numExtraDays
, grantorLastName: grantorLastName
});
extraDays.save( function(mongoErr) {
if ( mongoErr ) {
callback( winston.makeMongoError(mongoErr) );
} else {
upgradeUtils.updateUserAccountStatusByUserId( userId, false, callback );
}
});
}
});
fs.readFile( emailFile, function(err, res) {
if ( err ) {
forEachSeriesCallback( winston.makeError('error reading file') );
} else if ( ! res ) {
forEachSeriesCallback( winston.makeMissingParamError('res') );
} else {
var mailParser = new MailParser();
mailParser.on('end', function( parsedMail ) {
threadLink.handleParsedMail( parsedMail, function(err) {
if ( err ) {
winston.handleError(err);
}
mailParserDoneCallback();
});
});
mailParser.write(res);
mailParser.end();
mailParserDoneCallback = forEachSeriesCallback;
}
});
exports.grantUserPromotionAction = function( userEmail, promotionAction, callback ) {
if ( ! userEmail ) { callback( winston.makeMissingParamError('userEmail') ); return; }
if ( ! promotionAction ) { callback( winston.makeMissingParamError('promotionAction') ); return; }
if ( ( promotionAction != serverCommonConstants.PROMOTION_TYPE_CHROMESTORE_REVIEW ) && ( promotionAction != serverCommonConstants.PROMOTION_TYPE_FACEBOOK_LIKE ) ) {
callback( winston.makeError('invalid promotionAction', {promotionAction: promotionAction}) );
return;
}
var updateData = {$set: {}};
if ( promotionAction == serverCommonConstants.PROMOTION_TYPE_CHROMESTORE_REVIEW ) {
updateData['$set']['clickedChromeStoreReview'] = true;
} else if ( promotionAction == serverCommonConstants.PROMOTION_TYPE_FACEBOOK_LIKE ) {
updateData['$set']['clickedFacebookLike'] = true;
}
UserModel.findOneAndUpdate({email: userEmail}, updateData, function(mongoErr, foundUser) {
if ( mongoErr ) {
callback( winston.makeMongoError( mongoErr ) );
} else {
var forceResumeDownloadJob = false;
upgradeUtils.updateUserAccountStatus( foundUser, forceResumeDownloadJob, callback );
}
});
}
function (err) {
if (err) {
winston.handleError (err);
} else {
winston.doInfo('all done for all users');
}
});
.exec (function (err, foundLinks) {
if (err) {
cb (winston.makeMongoError (err));
} else if (foundLinks && foundLinks.length) {
var len = foundLinks.length;
var totalCallbacks = 0;
winston.doInfo ('About to create link reindexing jobs for user ', {userId : userId, numLinks : len});
foundLinks.forEach (function (link) {
if (createReindexingJobs.jobAlreadyQueued (link)) {
totalCallbacks++;
}
else {
indexingHandler.createIndexingJobForDocument ( link, true, true, function (err) {
totalCallbacks++;
if (err) {
winston.doError ('could not push job to queue for link', {err :err, linkId : link._id});
} else if (len == totalCallbacks) {
cb ();
}
});
}
});
} else {
cb ();
}
});
.exec (function (err, foundAttachments) {
if (err) {
cb (winston.makeMongoError (err));
} else if (foundAttachments && foundAttachments.length) {
var len = foundAttachments.length;
var totalCallbacks = 0;
winston.doInfo ('About to create attachment reindexing jobs for user ', {userId : userId, numAttachments : len});
foundAttachments.forEach (function (attachment) {
if (createReindexingJobs.jobAlreadyQueued (attachment)) {
winston.doInfo('job already queued');
totalCallbacks++;
}
else {
indexingHandler.createIndexingJobForDocument ( attachment, false, true, function (err) {
totalCallbacks++;
if (err) {
winston.doError ('could not push job to queue for attachment', {err :err, attachmentId : attachment._id});
} else if (len == totalCallbacks) {
cb ();
}
});
}
});
} else {
cb ();
}
});
exports.buildAttachment = function( message, parsedMailAttachment, mail ) {
if ( ! parsedMailAttachment ) { winston.doMissingParamError('parsedMailAttachment'); return null; }
if ( ! mail ) { winston.doMissingParamError('mail'); return null; }
if ( ! mail.userId ) { winston.doMissingParamError('mail.userId'); return null; }
var attachment = new AttachmentModel({
userId: mail.userId
, mailId: mail._id
, filename: parsedMailAttachment.fileName
, contentType: parsedMailAttachment.contentType
, isImage: attachmentUtils.isAttachmentImage( parsedMailAttachment )
, docType : attachmentUtils.getDocType (parsedMailAttachment.contentType)
, fileSize: parsedMailAttachment.length
, sentDate: mail.sentDate
, sender: mailUtils.copySender( mail.sender )
, recipients: mail.recipients
, mailCleanSubject: mail.cleanSubject
, gmThreadId: mail.gmThreadId
, gmMsgId: mail.gmMsgId
, gmMsgHex : mailUtils.getHexValue (mail.gmMsgId)
, isPromoted: attachmentHandler.isPromotable(parsedMailAttachment )
, hash: attachmentHandler.getMailAttachmentHash(parsedMailAttachment )
//, image: image //If it's an image itself, we'll set a signedURL on this in the route.
});
return attachment;
}
exports.handleAttachment = function( message, parsedMailAttachment, parsedMail, mail, callback ) {
if ( ! parsedMailAttachment ) { callback( winston.makeMissingParamError('parsedMailAttachment') ); return; }
if ( ! parsedMail ) { callback( winston.makeMissingParamError('parsedMail') ); return; }
if ( ! mail ) { callback( winston.makeMissingParamError('mail') ); return; }
//winston.doInfo('attachmentHandler: handleAttachment...', {mailId: mail._id});
// set a default filename if it doesn't exist
if (! parsedMailAttachment.fileName ) {
parsedMailAttachment.fileName = "untitled";
}
//Filter invalid attachments...
if ( ! parsedMailAttachment.length || !parsedMailAttachment.contentType) {
var warnData = {mailId: mail._id, contentType: parsedMailAttachment.contentType};
winston.doWarn('attachmentHandler: handleAttachment: missing filename or contentType', warnData);
callback();
} else {
attachmentHandler.lookForDuplicateOnThread ( parsedMailAttachment, mail, function (err, duplicateAttachment) {
if (err) {
callback (err);
} else if (duplicateAttachment) {
attachmentHandler.handleDuplicateOnThread (message, parsedMailAttachment, mail, duplicateAttachment, callback);
} else {
attachmentHandler.buildCheckSaveUploadAttachment(message, parsedMailAttachment, parsedMail, mail, callback );
}
});
}
}
function (err) {
if (err) {
winston.handleError (err);
}
winston.doInfo('all done');
process.exit (1);
}]);
exports.lookForDuplicateOnThread = function (parsedMailAttachment, mail, callback) {
if ( ! parsedMailAttachment ) { callback( winston.makeMissingParamError('parsedMailAttachment') ); return; }
if ( ! mail ) { callback( winston.makeMissingParamError('mail') ); return; }
var hash = attachmentHandler.getMailAttachmentHash(parsedMailAttachment );
var fileSize = parsedMailAttachment.length;
var duplicateSearchCriteria = {
hash: hash
, fileSize: fileSize
, userId: mail.userId
, gmThreadId: mail.gmThreadId
}
AttachmentModel.findOne( duplicateSearchCriteria, function(err, foundAttachment) {
if ( err ) {
callback( winston.makeMongoError(err) );
} else if ( ! foundAttachment ) {
//No duplicate on this thread, move along...
callback();
} else {
callback (null, foundAttachment);
}
});
}
async.waterfall (operations, function (err) {
if (err && !err.warning) {
winston.doWarn ('Could not finish updating', err);
callback (err);
}
else {
winston.doInfo('closing mailbox ' + user.email);
// close the mailbox
imapConnect.closeMailbox (myConnection, function (err) {
if (err) {
winston.doError ('Could not close mailbox', {err : err});
}
else {
imapConnect.logout (myConnection, function (err) {
if (err) {
winston.doError ('Could not logout', {err : err});
}
});
}
});
callback ();
}
})
AttachmentModel.findOne ({hash : attachmentInfo.hash, fileSize : attachmentInfo.fileSize}, function (err, att) {
if (err) {
asyncCb (winston.makeMongoError (err));
} else if (!att) {
winston.doWarn ('AttachmentInfo without attachment', {hash : attachmentInfo.hash, fileSize : attachmentInfo.fileSize});
asyncCb ();
} else {
MailModel.findById (att.mailId, function (err, foundMail) {
if (err) {
asyncCb (winston.makeMongoError (err));
} else if (!foundMail) {
winston.doWarn ('Could not find mail with id', {_id : att.mailId})
asyncCb ();
} else {
foundMail.mailReaderState = 'softFail'
foundMail.save (function (err) {
if (err) {
asyncCb (winston.makeMongoError (err));
return;
} else {
winston.doInfo ('updated mail state to softFail', {mailId : att.mailId});
asyncCb ();
}
});
}
});
}
});
cloudStorageUtils.downloadAndSaveImage( url, true, function (err, path) {
if (err) {
winston.doError('test failed', {err: err});
return;
}
winston.doInfo('path', {path: path});
});
exports.updateMail = function( parsedMail, mail, callback ) {
//winston.doInfo('mailReader: updateMail...');
if ( ! parsedMail ) { callback( winston.makeMissingParamError('parsedMail') ); return; }
if ( ! mail ) { callback( winston.makeMissingParamError('mail') ); return; }
var sentDate = mailUtils.getSentDate( parsedMail );
if ( mail.gmDate ) {
sentDate = mail.gmDate;
}
var filter = {
_id: mail._id
};
var updateSet = { $set: {
messageId: mailUtils.getMessageId( parsedMail )
, subject: mailUtils.getDirtySubject( parsedMail.subject )
, cleanSubject: mailUtils.getCleanSubject( parsedMail.subject )
, numAttachments: mailUtils.getNumAttachments( parsedMail )
, sentDate: sentDate
, mailReaderState: 'started'
}};
MailModel.findOneAndUpdate( filter, updateSet, function(err, updatedMail) {
if ( err ) {
callback( winston.makeMongoError(err) );
} else {
updatedMail.bodyText = mailUtils.getBodyText( parsedMail );
updatedMail.bodyHTML = mailUtils.getBodyHTML( parsedMail );
callback(null, updatedMail);
}
});
}
exports.getLinks = function(req, res) {
if ( ( ! req ) || ( ! req.user ) || ( ! req.user._id ) ) {
winston.doWarn('routeLinks: getLinks: missing userId');
res.send(400, 'missing userId');
return;
}
var user = req.user;
var userId = user._id;
var before = req.query.before;
var after = req.query.after;
var limit = req.query.limit;
var isFavorite = req.query.isFavorite;
if (!isFavorite) {
isFavorite = false;
}
if (!limit) {
limit = 50
}
var query = LinkModel.find({userId:userId, 'isPromoted':true, 'isFollowed':true, 'isDeleted' : false, 'isFavorite' : isFavorite })
if ( before && ( before != Infinity ) && ( before != 'Infinity' ) ) {
query.where ('sentDate').lt(before);
}
if ( after && ( after != -Infinity ) && ( after != '-Infinity' ) ) {
query.where('sentDate').gt(after);
}
if ( ! user.isPremium ) {
var daysLimit = user.daysLimit;
if ( ! daysLimit ) {
winston.doError('user is not premium, but has no daysLimit', {userId: userId});
} else {
var currentTime = Date.now();
var cutoffDate = new Date(currentTime - daysLimit*constants.ONE_DAY_IN_MS);
if ( ( ! after ) || ( cutoffDate > new Date(after) ) ) {
query.where('sentDate').gt(cutoffDate);
}
}
}
query.sort ('-sentDate')
.limit (limit)
.select(constants.DEFAULT_FIELDS_LINK)
.exec(function(err, foundLinks) {
if ( err ) {
winston.doMongoError(err, null, res);
} else {
linkHelpers.addSignedURLs( foundLinks )
res.send( foundLinks );
}
}
);
}
.exec (function (err, linkInfos) {
if (err) {
winston.doMongoError (err);
}
else {
var numLinkInfos = linkInfos.length;
winston.doInfo ('requeing linkinfos ', {links : numLinkInfos});
async.forEach (linkInfos, function (linkInfo, asyncCb) {
// find random user that has the link
LinkModel.findOne ({comparableURLHash : linkInfo.comparableURLHash})
.select ('userId')
.exec (function (err, link) {
if (err) {
asyncCb (winston.makeMongoError (err));
} else if (!link) {
asyncCb ();
} else {
followLinkUtils.createFollowLinkJob (linkInfo, link.userId);
asyncCb();
}
});
}, function (err) {
if (err) {
callback (err);
} else {
if (numLinkInfos == limit) {
callback (null, numLinkInfos + skip);
} else {
callback ();
}
}
});
}
});
schemaModel.findById( modelId, function(err, foundModel) {
if ( err ) {
callback( winston.makeMongoError( err ) );
} else if ( ! foundModel ) {
callback( winston.makeError('no model with id', {modelId: modelId, isLink: isLink}) );
} else {
foundModel["index"] = undefined;
foundModel.save( function(mongoErr) {
if ( mongoErr ) {
callback( winston.makeMongoError( mongoErr ) );
} else {
schemaModel.findById( modelId, function(err, foundModel) {
if ( err ) {
callback( winston.makeMongoError( err ) );
} else if ( ! foundModel ) {
callback( winston.makeError('no model with id', {modelId: modelId, isLink: isLink}) );
} if ( foundModel.index && foundModel.index.length ) {
callback( winston.makeError('model still has index') );
} else {
var job = indexingHandler.getIndexingJobForDocument( foundModel, isLink );
job = JSON.parse(JSON.stringify(job));
if ( ! job ) {
callback( winston.makeError('no job') );
} else {
indexingHandler.doIndexingJob( job, function(err) {
if ( err ) {
callback( err );
} else {
schemaModel.findById( modelId, function(err, foundModel) {
if ( err ) {
callback( winston.makeMongoError( err ) );
} else if ( ! foundModel ) {
callback( winston.makeError('no model with id', {modelId: modelId, isLink: isLink}) );
} else if ( !foundModel.index ) {
callback( winston.makeError('model has NO index') );
} else {
callback();
}
});
}
});
}
}
});
}
});
}
});
exports.inheritAttachmentFromAttachmentInfo = function(attachment, attachmentInfo ) {
if ( ! attachment ) { winston.doMissingParamError('attachment'); return; }
if ( ! attachmentInfo ) { winston.doMissingParamError('attachmentInfo'); return; }
attachment.attachmentThumbExists = attachmentInfo.attachmentThumbExists;
}
}, function(err) {
if ( err ) {
winston.handleError(err);
} else {
mongoose.disconnect();
winston.doInfo('cleanup done!');
}
});
run( function(err) {
if ( err ) {
winston.handleError( err );
} else {
winston.doInfo('success');
}
mongoose.disconnect();
})
foundMail.save (function (err) {
if (err) {
asyncCb (winston.makeMongoError (err));
return;
} else {
winston.doInfo ('updated mail state to softFail', {mailId : att.mailId});
asyncCb ();
}
});
var processBatchCallback = function (err, newIndex) {
if (err) {
winston.doMongoError (err);
} else if (newIndex) {
processBatch (newIndex, processBatchCallback);
} else {
winston.doInfo('all done');
}
}
function doBatchCallback (err, skip) {
if (err) {
winston.doError('error', {err: err});
}
else if (skip) {
retryFollows.doBatch (skip, doBatchCallback);
} else {
winston.doInfo('all done');
}
}
exports.getAttachmentInfo = function(attachment, parsedMailAttachment, callback ) {
if ( ! attachment ) { callback( winston.makeMissingParamError('attachment') ); return; }
if ( ! parsedMailAttachment ) { callback( winston.makeMissingParamError('parsedMailAttachment') ); return; }
//winston.doInfo('attachmentHandler: getAttachmentInfo...', {attachmentHash: attachment.hash});
var filter = {
hash: attachment.hash
, fileSize: attachment.fileSize
}
var updateSet = {
$set: {
hash: attachment.hash
, fileSize: attachment.fileSize
, isImage: attachment.isImage
, docType: attachment.docType
}
};
if (attachment.contentType) {
updateSet['$set']['contentType'] = attachment.contentType.trim();
}
var options = {
upsert:true
, new: false
}
AttachmentInfoModel.findOneAndUpdate(filter, updateSet, options, function(err, previousAttachmentInfo) {
if ( err ) {
callback( winston.makeMongoError( err ) );
} else {
//Lookup the thing we just saved.
//This seems wasteful, but I'd really like to have the clean attachmentInfo.
AttachmentInfoModel.findOne(filter, function(err, attachmentInfo) {
if ( err ) {
callback(err);
} else if ( ! attachmentInfo ) {
callback( winston.makeError('failed to find attachmentInfo we just upserted', filter) );
} else if ( ( ! previousAttachmentInfo._id ) || ( ! attachmentInfo.isUploaded ) ) {
//we either just created this attachmentInfo, or we haven't uploaded it before
attachmentHandler.uploadAttachmentInfoToCloud(attachmentInfo, parsedMailAttachment, callback );
} else {
callback( null, attachmentInfo );
}
});
}
});
}
imapConnect.openMailbox (myConnection, user.email, function (err, mailbox) {
if (err) {
winston.addExtra( err, {userId : user._id, userEmail : user.email} );
winston.handleError(err);
return;
}
winston.doInfo('Connection opened for user: ' + user.email);
mailUpdateHelper.updateMailbox (user, myConnection, mailbox, 0, pollQueueCallback);
})