const download_file_s3 = function(bucket,key) {
var params = {
'Key' : key,
'Bucket' : bucket
};
return s3.getObject(params).promise();
};
return s3.listObjects(params).promise().then(function(result) {
var params = {Bucket: bucket_name, Delete: { Objects: [] }};
params.Delete.Objects = result.Contents.map(function(content) { return { Key: content.Key }; });
if (params.Delete.Objects.length < 1) {
return Promise.resolve({"data" : { "Deleted" : [] }});
}
return s3.deleteObjects(params).promise();
}).then(function(result) {
var refreshMetadata = function() {
var s3 = new AWS.S3();
var params = {
Bucket: bucket_name,
Prefix: "uploads/"
};
return s3.listObjectsV2(params).promise().then(function(result) {
let messages = result.Contents.map( (dataset) => { return { "path" : dataset.Key, "offset" : "dummy", "byte_offset" : dataset.Size > (1024*50) ? dataset.Size - (1024*50) : 0 }; } );
messages = messages.filter( (message) => message.path.indexOf('o_man') >= 0 );
console.log(messages);
return messages;
}).then((messages) => {
let queue = new Queue(split_queue);
Promise.all(messages.map( message => queue.sendMessage(message) ))
}).then( () => {
return require('lambda-helpers').lambda_promise(runSplitQueue)({'time' : 'triggered'});
});
};
const retrieve_file_s3 = function retrieve_file_s3(bucket,filekey) {
var params = {
'Key' : filekey,
'Bucket' : bucket
};
let stream = s3.getObject(params).createReadStream();
stream.source = params.Key;
return stream;
};
let written_promise = new Promise(function(resolve,reject) {
s3.upload(params, options,function(err,data) {
if (err) {
console.log("Writing error for ",taxid,params.Bucket,params.Key);
reject(err);
return;
}
resolve(data);
});
});
const get_interpro_set_keys = function(bucket,release) {
var params = {
Bucket: bucket,
Prefix: interpro_bucket_prefix+"/InterPro"+(release ? ("-"+release) : "")
};
return s3.listObjects(params).promise().then(function(result) {
let keys = result.Contents.map(function(content) { return content.Key; });
return keys;
});
};
var upload_data_record_s3 = function upload_data_record_s3(key,data) {
if ( ! key ) {
return Promise.resolve(true);
}
var params = {
'Bucket': bucket_name,
'Key': key,
'ContentType': 'application/json'
};
var datablock = JSON.stringify(data);
params.Body = datablock;
params.ContentMD5 = new Buffer(crypto.createHash('md5').update(datablock).digest('hex'),'hex').toString('base64');
var options = {partSize: 15 * 1024 * 1024, queueSize: 1};
return s3.upload(params, options).promise();
};
var retrieve_file_s3 = function retrieve_file_s3(filekey,md5_result,byte_offset) {
var params = {
'Key' : filekey,
'Bucket' : bucket_name,
'IfNoneMatch' : md5_result.old+"",
};
if (byte_offset) {
params.Range = 'bytes='+byte_offset+'-'
}
var request = s3.getObject(params);
var stream = request.createReadStream();
stream.on('finish',function() {
md5_result.md5 = request.response.data.ETag;
});
return stream;
};
var remove_folder_s3 = function remove_folder_s3(setkey) {
var params = {
Bucket: bucket_name,
Prefix: "data/latest/"+setkey+"/"
};
console.log(params);
return s3.listObjects(params).promise().then(function(result) {
var params = {Bucket: bucket_name, Delete: { Objects: [] }};
params.Delete.Objects = result.Contents.map(function(content) { return { Key: content.Key }; });
if (params.Delete.Objects.length < 1) {
return Promise.resolve({"data" : { "Deleted" : [] }});
}
return s3.deleteObjects(params).promise();
}).then(function(result) {
if (result.Deleted.length === 1000) {
return remove_folder(setkey);
}
return true;
});
};
const get_session = function(session_id) {
return s3.getObject({Bucket: bucket, Key: 'sessions/'+session_id }).promise()
.then( result => result.Body.toString() );
};
const upload_session = function(session_id,final_token) {
return s3.putObject({Bucket: bucket, Key: 'sessions/'+session_id, Body: final_token }).promise();
};