var azure = require('azure-storage');
var accessKey = 'GW3gpzW2TtcSqxS+EWsq4+CiwN7IRb5yTpEIVaLVVYxyMIjmnxqbbwnqzNKPekw5UohX71odmRP72n4TOxAiWA==';
var storageAccount = 'devox';
var containerName = 'images';
var blobService = azure.createBlobService(storageAccount, accessKey);
Array.prototype.select = function(expr){
var arr = this;
//do custom stuff
return arr.map(expr); //or $.map(expr);
};
function getFiles(response) {
console.log("Le gestionnaire 'getFiles' est appelé.");
var blobs = [];
function shuffleArray(array) {
for (var i = array.length - 1; i > 0; i--) {
var j = Math.floor(Math.random() * (i + 1));
var temp = array[i];
array[i] = array[j];
array[j] = temp;
}
return array;
}
function aggregateBlobs(err, result, cb) {
function Container(containerName, accountName, accountKey) {
this.name = containerName;
this.blobSvc = azure.createBlobService(accountName, accountKey);
}
(req, res, next) => {
const wtBody = (req.webtaskContext && req.webtaskContext.body) || req.body || {};
const wtHead = (req.webtaskContext && req.webtaskContext.headers) || {};
const isCron = (wtBody.schedule && wtBody.state === 'active') || (wtHead.referer === `${config('AUTH0_MANAGE_URL')}/` && wtHead['if-none-match']);
if (!isCron) {
return next();
}
const blobService = azure.createBlobService(config('STORAGE_ACCOUNT_NAME'), config('STORAGE_ACCOUNT_KEY'));
const remapLogs = (record) => {
record.type_code = record.type;
record.type = loggingTools.logTypes.get(record.type);
if (record.user_agent && record.user_agent.length) {
let agent = useragent.parse(record.user_agent);
record.os = agent.os.toString();
record.os_version = agent.os.toVersion();
record.device = agent.device.toString();
record.device_version = agent.device.toVersion();
}
return record;
};
const onLogsReceived = (logs, callback) => {
if (!logs || !logs.length) {
return callback();
}
logger.info(`Sending ${logs.length} logs to Azure Blob Storage.`);
async.eachLimit(logs.map(remapLogs), 5, (log, cb) => {
const date = moment(log.date);
const url = `${date.format('YYYY/MM/DD')}/${date.format('HH')}/${log._id}.json`;
blobService.createBlockBlobFromText(config('STORAGE_CONTAINER_NAME'), url, JSON.stringify(log), cb);
}, (err) => {
if (err) {
return callback(err);
}
logger.info('Upload complete.');
return callback();
});
};
const slack = new loggingTools.reporters.SlackReporter({
hook: config('SLACK_INCOMING_WEBHOOK_URL'),
username: '******',
title: 'Logs To Azure Blob Storage' });
const options = {
domain: config('AUTH0_DOMAIN'),
clientId: config('AUTH0_CLIENT_ID'),
clientSecret: config('AUTH0_CLIENT_SECRET'),
batchSize: parseInt(config('BATCH_SIZE'), 10),
startFrom: config('START_FROM'),
logTypes: config('LOG_TYPES'),
logLevel: config('LOG_LEVEL')
};
if (!options.batchSize || options.batchSize > 100) {
options.batchSize = 100;
}
if (options.logTypes && !Array.isArray(options.logTypes)) {
options.logTypes = options.logTypes.replace(/\s/g, '').split(',');
}
const auth0logger = new loggingTools.LogsProcessor(storage, options);
const sendDailyReport = (lastReportDate) => {
const current = new Date();
const end = current.getTime();
const start = end - 86400000;
auth0logger.getReport(start, end)
.then(report => slack.send(report, report.checkpoint))
.then(() => storage.read())
.then((data) => {
data.lastReportDate = lastReportDate;
return storage.write(data);
});
};
const checkReportTime = () => {
storage.read()
.then((data) => {
const now = moment().format('DD-MM-YYYY');
const reportTime = config('DAILY_REPORT_TIME') || 16;
if (data.lastReportDate !== now && new Date().getHours() >= reportTime) {
sendDailyReport(now);
}
})
};
return auth0logger
.run(onLogsReceived)
.then(result => {
if (result && result.status && result.status.error) {
slack.send(result.status, result.checkpoint);
} else if (config('SLACK_SEND_SUCCESS') === true || config('SLACK_SEND_SUCCESS') === 'true') {
slack.send(result.status, result.checkpoint);
}
checkReportTime();
res.json(result);
})
.catch(err => {
slack.send({ error: err, logsProcessed: 0 }, null);
checkReportTime();
next(err);
});
};
createBlobService: ()=>azure.createBlobService(account,key),
suite('stream', function() {
var azure = require('azure-storage');
var service = azure.createBlobService();
var https = require('https');
var fs = require('fs');
var uuid = require('uuid');
var BlockStream = require('./block_stream');
var Promise = require('promise');
if (process.env.DEBUG) {
service.logger = new azure.Logger(azure.Logger.LogLevels.DEBUG);
}
/**
Use the node http client to fetch the entire contents of the azure upload.
*/
function fetchContents() {
var promise = new Promise(function(accept, reject) {
var url = service.getUrl(container, path);
var buffer = new Buffer(0);
var req = https.get(url, function(res) {
res.on('data', function(incoming) {
buffer = Buffer.concat([buffer, incoming]);
});
res.on('end', function() {
accept({
content: buffer,
headers: res.headers
});
});
}).once('error', reject);
});
return promise;
}
var subject;
var path = 'mycommitfile.txt';
var container;
setup(function() {
container = uuid.v4();
subject = new BlockStream(service, container, path);
});
// create the container on azure
setup(function(done) {
service.createContainerIfNotExists(
container,
// allow any public GET operations
{ publicAccessLevel: 'container' },
done
);
});
// ensure we are always in a clean state
teardown(function(done) {
service.deleteContainer(container, done);
});
var fixture = __dirname + '/test/fixtures/travis_log.txt';
var subject;
setup(function() {
subject = new BlockStream(
service,
container,
path
);
});
suite('partial upload', function() {
var expectedContent = new Buffer('xxx');
setup(function(done) {
subject.write(expectedContent, null, done);
});
test('content should be written but not marked as finished', function() {
return fetchContents().then(
function(result) {
var headers = result.headers;
var content = result.content.toString();
// content is valid
assert.equal(expectedContent.toString(), content);
assert.ok(
!headers[BlockStream.COMPLETE_HEADER],
'is not marked as complete'
);
}
);
});
});
suite('upload an entire file', function() {
// setup the stream
setup(function(done) {
fs.createReadStream(fixture).pipe(subject);
assert.ok(!subject.closed, 'not closed');
subject.once('close', function() {
assert.ok(subject.closed, 'closed');
done();
});
subject.once('error', done);
});
test('read contents', function() {
var expected = fs.readFileSync(fixture);
return fetchContents().then(
function(result) {
var headers = result.headers;
var content = result.content.toString();
// content is valid
assert.equal(expected.toString(), content);
assert.ok(
headers[BlockStream.COMPLETE_HEADER],
'is marked as complete'
);
assert.equal(
headers['content-type'],
subject.contentType
);
assert.equal(
headers['content-encoding'],
subject.contentEncoding
);
}
);
});
});
});
it.skip('Imports then exports devices', function(done) {
this.timeout(120000);
var testDeviceCount = 10;
var registry = Registry.fromConnectionString(hubConnectionString);
var blobSvc = azureStorage.createBlobService(storageConnectionString);
var inputContainerName = 'nodee2e-import-' + uuid.v4();
var outputContainerName = 'nodee2e-export-' + uuid.v4();
var deviceFile = 'devices.txt';
var inputBlobSasUrl, outputBlobSasUrl;
var devicesToImport = [];
for (var i = 0; i < testDeviceCount; i++) {
var deviceId = 'nodee2e-' + uuid.v4();
var device = {
id: deviceId,
authentication: {
symmetricKey: {
primaryKey: new Buffer(uuid.v4()).toString('base64'),
secondaryKey: new Buffer(uuid.v4()).toString('base64')
}
},
status: 'enabled'
};
devicesToImport.push(device);
}
var devicesText = '';
devicesToImport.forEach(function (device) {
device.importMode = 'createOrUpdate';
devicesText += JSON.stringify(device) + '\r\n';
});
debug('Devices to import: ' + devicesText);
var createContainers = function() {
return new Promise(function(resolve, reject) {
debug('Create input container');
blobSvc.createContainerIfNotExists(inputContainerName, function (err) {
if(err) {
reject(new Error('Could not create input container: ' + err.message));
} else {
var startDate = new Date();
var expiryDate = new Date(startDate);
expiryDate.setMinutes(startDate.getMinutes() + 100);
startDate.setMinutes(startDate.getMinutes() - 100);
var inputSharedAccessPolicy = {
AccessPolicy: {
Permissions: 'rl',
Start: startDate,
Expiry: expiryDate
},
};
var outputSharedAccessPolicy = {
AccessPolicy: {
Permissions: 'rwd',
Start: startDate,
Expiry: expiryDate
},
};
var inputSasToken = blobSvc.generateSharedAccessSignature(inputContainerName, null, inputSharedAccessPolicy);
inputBlobSasUrl = blobSvc.getUrl(inputContainerName, null, inputSasToken);
debug('Create output container');
blobSvc.createContainerIfNotExists(outputContainerName, function (err) {
if (err) {
reject(new Error('Could not create output container: ' + err.message));
} else {
var outputSasToken = blobSvc.generateSharedAccessSignature(outputContainerName, null, outputSharedAccessPolicy);
outputBlobSasUrl = blobSvc.getUrl(outputContainerName, null, outputSasToken);
resolve();
}
});
}
});
});
};
var deleteContainers = function() {
return new Promise(function (resolve, reject) {
var blobSvc = azureStorage.createBlobService(storageConnectionString);
debug('Delete input container');
blobSvc.deleteContainer(inputContainerName, function (err) {
if(err) {
reject(new Error('Could not delete input container: ' + err.message));
} else {
debug('Delete output container');
blobSvc.deleteContainer(outputContainerName, function (err) {
if(err) {
reject(new Error('Could not delete output container: ' + err.message));
} else {
resolve();
}
});
}
});
});
};
var verifyDeviceProperties = function (importedDevice, exportedDevice) {
return importedDevice.id === exportedDevice.id &&
importedDevice.authentication.symmetricKey.primaryKey === exportedDevice.authentication.symmetricKey.primaryKey &&
importedDevice.authentication.symmetricKey.secondaryKey === exportedDevice.authentication.symmetricKey.secondaryKey &&
importedDevice.status === exportedDevice.status;
};
var verifyOutputBlob = function() {
return new Promise(function (resolve, reject){
debug('Verifying export blob');
blobSvc.getBlobToText(outputContainerName, deviceFile, function(err, result) {
if(err) {
reject(new Error('Could not get export blob: ' + err.message));
} else {
var devicesTextTable = result.split('\r\n');
var exportedDevices = [];
for (var i = 0; i < devicesTextTable.length; i++) {
debug('[' + i + '] ' + devicesTextTable[i]);
if (devicesTextTable[i].trim()) {
exportedDevices.push(JSON.parse(devicesTextTable[i]));
}
}
for (i = 0; i < devicesToImport.length; i++) {
var deviceFound = false;
for (var j = 0; j < exportedDevices.length; j++) {
if (verifyDeviceProperties(devicesToImport[i], exportedDevices[j])) {
deviceFound = true;
debug('Found device: ' + devicesToImport[i].id);
break;
}
}
if (!deviceFound) {
reject(new Error('Could not find ' + devicesToImport[i].id));
}
}
debug('Found all devices, cleaning device registry...');
resolve();
}
});
});
};
var runExportJob = function() {
return new Promise(function (resolve, reject){
debug('Starting export job');
registry.exportDevicesToBlob(outputBlobSasUrl, false, function (err, result) {
if(err) {
reject(new Error('Could not create export job: ' + err.message));
} else {
debug('Export job created');
var exportJobId = result.jobId;
var jobFinished = false;
var exportInterval = setInterval(function() {
registry.getJob(exportJobId, function (err, result) {
if (err) {
reject(new Error('Could not get export job status: ' + err.message));
} else {
var status = result.status;
if (status === "completed" && !jobFinished) {
jobFinished = true;
debug('Export job completed');
clearInterval(exportInterval);
resolve();
}
}
});
}, 1000);
}
});
});
};
var runImportJob = function(devicesText) {
return new Promise(function (resolve, reject){
blobSvc.createBlockBlobFromText(inputContainerName, deviceFile, devicesText, function (err) {
if(err) {
reject(new Error('Could not create blob for import job: ' + err.message));
} else {
debug('Starting import job');
registry.importDevicesFromBlob(inputBlobSasUrl, outputBlobSasUrl, function (err, result) {
if(err) {
reject(new Error('Could not create import job: ' + err.message));
} else {
var importJobId = result.jobId;
var jobFinished = false;
var importInterval = setInterval(function () {
registry.getJob(importJobId, function (err, result) {
if (err) {
reject(new Error('Could not get import job status: ' + err.message));
} else {
var status = result.status;
if (status === "completed" && !jobFinished) {
jobFinished = true;
debug('Import job completed');
clearInterval(importInterval);
resolve();
}
}
});
}, 1000);
}
});
}
});
});
};
var deleteTestDevices = function() {
return new Promise(function (resolve, reject){
debug('Cleaning devices');
var devicesText = '';
devicesToImport.forEach(function (device) {
device.importMode = 'delete';
devicesText += JSON.stringify(device) + '\r\n';
});
runImportJob(devicesText)
.then(resolve)
.catch(reject);
});
};
createContainers()
.then(function() { return runImportJob(devicesText); })
.then(runExportJob)
.then(verifyOutputBlob)
.then(deleteTestDevices)
.then(deleteContainers)
.then(function() { done(); })
.catch(function(err) { done(err); });
});
/**
* Append blob basics.
* @ignore
*
* @param {config} config The configuration which contains the connectionString.
* @param {errorOrResult} callback The callback function.
*/
function basicStorageAppendBlobOperations(config, callback) {
// Create a blob client for interacting with the blob service from connection string
// How to create a storage connection string - http://msdn.microsoft.com/en-us/library/azure/ee758697.aspx
var blobService = storage.createBlobService(config.connectionString);
var fileToUpload = "HelloAppend.dat";
writeRandomFile(fileToUpload, 1024);
var appendBlobContainerName = "demoappendblobcontainer-" + guid.v1();
var appendBlobName = "demoappendblob-" + fileToUpload;
console.log('Append Blob Sample');
// Create a container for organizing blobs within the storage account.
console.log('1. Creating Container');
blobService.createContainerIfNotExists(appendBlobContainerName, function (error) {
if (error) {
callback(error);
} else {
// To view the uploaded blob in a browser, you have two options. The first option is to use a Shared Access Signature (SAS) token to delegate
// access to the resource. See the documentation links at the top for more information on SAS. The second approach is to set permissions
// to allow public access to blobs in this container. Uncomment the line below to use this approach. Then you can view the file
// using: https://[InsertYourStorageAccountNameHere].blob.core.windows.net/demoappendblobcontainer-[guid]/demopageblob-HelloAppend.dat
// Upload a PageBlob to the newly created container
console.log('2. Uploading AppendBlob');
blobService.createAppendBlobFromLocalFile(appendBlobContainerName, appendBlobName, fileToUpload, function (error) {
if (error) {
callback(error);
} else {
// List all the blobs in the container
console.log('3. List Blobs in Container');
listBlobs(blobService, appendBlobContainerName, null, null, function (error, results) {
if (error) {
callback(error);
} else {
for (var i = 0; i < results.length; i++) {
console.log(util.format(' - %s (type: %s)'), results[i].name, results[i].blobType);
}
// Download a blob to your file system
console.log('4. Download Blob');
var downloadedFileName = util.format('CopyOf%s', fileToUpload);
blobService.getBlobToLocalFile(appendBlobContainerName, appendBlobName, downloadedFileName, function (error) {
if (error) {
callback(error);
} else {
fs.stat(downloadedFileName, function(error, stats) {
console.log('5. Append block to append blob');
blobService.appendBlockFromText(appendBlobContainerName, appendBlobName, 'text to be appended', { appendPosition: stats.size } /** Verify that the blob has NOT been written by another process */, function(error){
if (error) {
callback(error);
} else {
console.log(' Downloaded File Size: %s', stats.size);
try { fs.unlinkSync(downloadedFileName); } catch (e) { }
// Clean up after the demo
console.log('6. Delete Append Blob');
blobService.deleteBlob(appendBlobContainerName, appendBlobName, function (error) {
if (error) {
callback(error);
} else {
// Delete the container
console.log('7. Delete Container');
blobService.deleteContainerIfExists(appendBlobContainerName, function (error) {
try { fs.unlinkSync(fileToUpload); } catch (e) { }
callback(error);
});
}
});
}
});
});
}
});
}
});
}
});
}
});
}
var parseArgs = require('minimist');
var argv;
argv = parseArgs(process.argv.slice(2));
console.dir(argv);
var azure = require('azure-storage');
var storageaccount = "sitesscannerdev";
var storagekey = "WYLY1df7AVnv5Kh0ed6UXD+z7dQzHsMGm5BAgNs2b0iH6CCMV1QK+rmIMHALKnFgRuE5hdx+0L4AQXKLVhYXjw==";
if (process.env.Storage_AccountName)
storageaccount = process.env.Storage_AccountName;
if (process.env.Storage_AccessKey)
storagekey = process.env.Storage_AccessKey;
var blobSvc = azure.createBlobService(storageaccount, storagekey);
var filename = 'websites.csv';
if (argv.file)
filename = argv.file;
blobSvc.createBlockBlobFromLocalFile('dailyscan', filename, filename, function (error, result, response) {
if (error) {
console.log(error);
return;
}
});
var express = require('express');
var router = express.Router();
var azure = require('azure-storage');
var azure = require('azure-storage');
var nconf = require('nconf');
var multiparty = require('multiparty');
nconf.env()
.file({ file: 'config.json', search: true });
var accountName = nconf.get("STORAGE_NAME");
var accountKey = nconf.get("STORAGE_KEY");
var blobSvc = azure.createBlobService(accountName, accountKey);
router.get('/', function (req, res) {
res.render('index', { title: 'Express' });
});
router.post('/upload', function (req, res) {
var form = new multiparty.Form();
form.parse(req);
console.log(form)
form.on('part', function (part) {
if (part.filename) {
var filename = part.filename;
var size = part.byteCount;
var onError = function (error) {
if (error) {
add: function(container, name, overwrite) {
// connect to Azure Storage
var service = wasb.createBlobService(storageAccount, storageKey);
// use or create the container
var ensureContainer = new promise(function(resolve, reject) {
try {
service.createContainerIfNotExists(container, function(error, result, response) {
if (error) {
console.log("createContainerIfNotExists: " + error);
reject("container?");
} else {
resolve(result);
}
});
} catch (ex) {
console.log("createContainerIfNotExists: " + ex);
reject("container?");
}
});
// check to see if the blob already exists
var canWriteFile = new promise(function(resolve, reject) {
try {
service.doesBlobExist(container, name, function(error, result, response) {
if (error) {
console.log("doesBlobExist: " + error);
reject("exists?");
} else {
if (!result.exists || overwrite) {
resolve(result);
} else {
reject("exists");
}
}
});
} catch (ex) {
console.log("doesBlobExist: " + ex);
reject("exists?");
}
});
var begin = new promise(function(resolve, reject) {
try {
// create a writable block blob
var writer = service.createWriteStreamToBlockBlob(container, name, function(error, result, response) {
if (error) {
console.log("createWriteStreamToBlockBlob: " + error);
}
});
// create a reference for the file
var file = {
container: container,
name: name,
sequence: 0,
writer: writer
};
// add to pending list
pending.list.push(file);
resolve(file);
} catch (ex) {
console.log("createWriteStreamToBlockBlob: " + ex);
reject("write?");
}
});
// process those actions in serial
return promise.each([ensureContainer, canWriteFile, begin], function(result) { }).then(function(results) {
return results[2];
}, function(error) {
return promise.reject(error);
});
},
"use strict";
var moment = require('moment');
var fs = require('fs');
var azureStorage = require('azure-storage');
var blobService = azureStorage.createBlobService();
var AWS = require('aws-sdk');
var uuid = require('node-uuid');
//var pid = uuid.v4();
var os = require('os');
var pid = os.hostname() + '#' + process.argv[2];
var path = require('path');
AWS.config.loadFromPath(path.resolve(__dirname, 'config.json'));
var s3 = new AWS.S3();
var sourceBucketName = 's3stress-source';
var sourceKeyName = 'OneKBFile.txt';
var localFileName = 'downloaded' + '_' + pid +'.dat';
var dataProcessed = 0;
var transactionsProcessed = 0;
var errorEvents = 0;
var previousTime;
var operationLatency = 0;
var database;
var collection;
var DocumentClient = require('documentdb').DocumentClient;
var docDBClient = new DocumentClient('https://amitkulwasstress.documents.azure.com:443/', {masterKey:process.env.AZURE_DOCDB_AUTH_KEY});
setupDocumentDB('wasstress', 'wasstressresults', function(){
function Container(accountName, accountKey, containerName) {
this.name = containerName;
this.service = azure.createBlobService(accountName, accountKey);
}
const azure = require('azure-storage');
const blobService = azure.createBlobService(process.env.AZURE_STORAGE_ACCOUNT_NAME, process.env.AZURE_STORAGE_KEY);
const azureContainerName = 'groundout';
exports.createContainer = () => {
return new Promise((resolve, reject) => {
blobService.createContainerIfNotExists(azureContainerName, (error, result, response) => {
if (error) return reject(error);
return resolve(result);
});
});
}
exports.uploadLocalFile = (fileName, dir) => {
return new Promise((resolve, reject) => {
blobService.createBlockBlobFromLocalFile(azureContainerName, fileName, dir, (error, result) => {
if (error) return reject(error);
return resolve(result);
})
});
}
exports.deleteFile = (fileName) => {
return new Promise((resolve, reject) => {
blobService.deleteBlobIfExists(azureContainerName, fileName, (error) => {
if (error) return reject(error);
return resolve();
});
});
'use strict';
var router = require('express').Router(); // eslint-disable-line new-cap
module.exports = router;
var _ = require('lodash');
var chalk = require('chalk')
var db = require('../../../db');
var Classroom = db.model('classroom')
var User = db.model('user')
var Transaction = db.model('transaction')
var multer = require('multer')
var azure = require('azure-storage');
var blobSvc = azure.createBlobService('DefaultEndpointsProtocol=https;AccountName=edvances;AccountKey=E69FNxbG0QQF+rLoFRRYulGDKWOYMmfUn1WmNtf9uznDauN0yksEgFFZot+sYPcjEGoHSRl2ccPj8R8JAPaHYA==;EndpointSuffix=core.windows.net')
var storage = multer.memoryStorage();
var streamifier = require('streamifier');
var upload = multer({
storage: storage
})
var ensureAuthenticated = function(req, res, next) {
var err;
if (req.isAuthenticated()) {
next();
} else {
err = new Error('You must be logged in.');
err.status = 401;
next(err);
}
};
router.get('/', ensureAuthenticated, (req, res) => {
Classroom.findAll().then(classrooms => {
var azure = require('azure-storage');
var uuidV4 = require('uuid/v4');
var path = require('path');
var config = require('../config.json');
var blobService = azure.createBlobService(config.azure.accountName, config.azure.accountKey);
var uploadFile = function (localfile) {
return new Promise(function (resolve, reject) {
var remotename = uuidV4() + path.parse(localfile).ext;
blobService.createBlockBlobFromLocalFile(config.azure.containerName, remotename, localfile, function (err, result, response) {
if (!err) {
var url = `https://${config.azure.accountName}.blob.core.windows.net/${config.azure.containerName}/${remotename}`;
resolve({ remotename: remotename, url: url });
} else {
console.error(err);
reject();
}
});
});
};
module.exports = {
uploadFile: uploadFile
};
then(() => {
const blobService = azureStorage.createBlobService(options.account, options.key);
const listContainersSegmented = pify(blobService.listContainersSegmented.bind(blobService));
const listBlobsSegmented = pify(blobService.listBlobsSegmented.bind(blobService));
const getBlobToLocalFile = pify(blobService.getBlobToLocalFile.bind(blobService));
const listBlobsOptions = {};
if (options.snapshots) {
listBlobsOptions.include = "snapshots";
}
const everythingRegExp = new RegExp();
const containerRegExp = options.containerPattern
? new RegExp(options.containerPattern)
: everythingRegExp;
const blobRegExp = options.blobPattern
? new RegExp(options.blobPattern)
: everythingRegExp;
const startDate = options.startDate
? new Date(options.startDate)
: null;
const endDate = options.endDate
? new Date(options.endDate)
: null;
const startTime =
(startDate && !isNaN(startDate.valueOf()) && startDate.toUTCString()) ||
"[beginning of time]";
const endTime =
(endDate && !isNaN(endDate.valueOf()) && endDate.toUTCString()) ||
"[end of time]";
console.log(`Downloading blobs in ${options.account} from ${startTime} to ${endTime}.`);
const containersMatching = containerRegExp === everythingRegExp
? "[anything]"
: `/${containerRegExp.source}/`;
console.log(`Listing containers in ${options.account} matching ${containersMatching}...`);
const listNextContainers = function listNextContainers (continuationToken, containerNames) {
return listContainersSegmented(continuationToken || null).
then((listContainerResult) => {
const nextContainerNames = listContainerResult.entries.
map((containerResult) => {
return containerResult.name;
}).
filter((containerName) => {
return containerRegExp.test(containerName);
});
const combinedContainerNames = (containerNames || []).concat(nextContainerNames);
return listContainerResult.continuationToken
? listNextContainers(listContainerResult.continuationToken, combinedContainerNames)
: combinedContainerNames;
});
};
return listNextContainers().
then((containerNames) => {
const listNextBlobs = function listNextBlobs (containerName, continuationToken, blobInfos) {
return listBlobsSegmented(containerName, continuationToken || null, listBlobsOptions).
then((listBlobsResult) => {
const nextBlobInfos = listBlobsResult.entries.map((blobResult) => {
const blobName = blobResult.name;
const lastModified = new Date(blobResult.lastModified);
const snapshot = blobResult.snapshot || "";
return {
containerName,
blobName,
lastModified,
snapshot
};
});
const combinedBlobInfos = (blobInfos || []).concat(nextBlobInfos);
return listBlobsResult.continuationToken
? listNextBlobs(containerName, listBlobsResult.continuationToken, combinedBlobInfos)
: combinedBlobInfos;
});
};
return containerNames.reduce((containerPromise, containerName) => {
return containerPromise.then((cumulativeBlobInfos) => {
const blobsMatching = blobRegExp === everythingRegExp
? "[anything]"
: `/${blobRegExp.source}/`;
console.log(`Listing blobs in ${containerName} matching ${blobsMatching}...`);
return stat(containerName).
catch(() => {
return mkdir(containerName);
}).
then(() => {
return listNextBlobs(containerName);
}).
then((blobInfos) => {
return cumulativeBlobInfos.concat(blobInfos.
filter((blobInfo) => {
return blobRegExp.test(blobInfo.blobName) &&
(!startDate || (startDate <= blobInfo.lastModified)) &&
(!endDate || (blobInfo.lastModified <= endDate));
}));
});
});
}, Promise.resolve([]));
}).
then((blobInfos) => {
return blobInfos.reduce((blobPromise, blobInfo) => {
return blobPromise.then(() => {
const containerName = blobInfo.containerName;
const blobName = blobInfo.blobName;
const snapshot = blobInfo.snapshot;
const combinedName = blobName + (snapshot
? ` (${blobInfo.snapshot})`
: "");
console.log(`Downloading ${containerName} / ${combinedName}...`);
const fileName = path.join(sanitize(containerName), sanitize(combinedName));
const blobRequestOptions = {};
if (blobInfo.snapshot) {
blobRequestOptions.snapshotId = snapshot;
}
return getBlobToLocalFile(containerName, blobName, fileName, blobRequestOptions).
then(() => {
// Ensure write is complete before changing modified date
return stat(fileName);
}).
then(() => {
const lastModified = blobInfo.lastModified;
return utimes(fileName, lastModified, lastModified);
});
});
}, Promise.resolve());
});
}).
var imageUploadHandler = function (image_info,
user_id, username, resp_callback) {
var blobSvc = azure.createBlobService();
var generate_access_url = function (container, file_name) {
var start_date = new Date();
var expiry_date = new Date(start_date);
expiry_date.setMinutes(start_date.getMinutes() + 100);
start_date.setMinutes(start_date.getMinutes() - 100);
var shared_access_policy = {
AccessPolicy: {
Permissions: azure.BlobUtilities.SharedAccessPermissions.READ,
Start: start_date,
Expiry: expiry_date
},
};
var blob_sas = blobSvc.generateSharedAccessSignature(
container, file_name,shared_access_policy);
var url = container + '/' + file_name +'?' + blob_sas;
var host = blobSvc.host;
logger.infoMsg("IMAGE_DEBUG",util.inspect({host:host}));
image_info.url = host.primaryHost+url;
image_info.second_host = host.secondaryHost;
//resp.json(image_info);
resp_callback(null, image_info);
};
/* store checksum to image_info.md5 and call next function */
var file_checksum_cb = function (checksum) {
logger.infoMsg("IMAGE_DEBUG","file checksum: " + checksum);
//image_info.md5 = checksum;
image_info.type = "image";
image_info.suffix = image_info.name.split('.').pop().toLowerCase();
image_info.hash = utilities.hashMulStringsWithSHA256(
checksum, user_id.toString());
image_info.storage_name = image_info.hash + '.' + image_info.suffix;
image_info.container = "containername"+username.substring(0,50);
blobSvc.createContainerIfNotExists(image_info.container, container_create_cb);
};
/* if succeed, create blob*/
var container_create_cb = function(error, result, response){
if (error) {
logger.errorMsg("IMAGE","failed to create container: "+error);
//resp.json({ success: false, message: 'ER_FAIL_STORE_IMAGE'});
resp_callback(error, { success: false, message: 'ER_FAIL_STORE_IMAGE'});
return ;
}
logger.infoMsg("IMAGE","successfully create container");
blobSvc.createBlockBlobFromLocalFile(image_info.container,
image_info.storage_name,image_info.path,
file_blob_create_cb);
};
var file_blob_create_cb = function(error, result, response){
if (error) {
logger.errorMsg("IMAGE","failed to create blob: "+error);
//resp.json({ success: false, message: 'ER_FAIL_STORE_IMAGE'});
resp_callback(error,
{ success: false, message: 'ER_FAIL_STORE_IMAGE'});
return ;
}
if (!response.isSuccessful){
//resp.json({ success: false, message: 'ER_FAIL_STORE_IMAGE'});
resp_callback(new Error("ER_FAIL_STORE_IMAGE"),
{ success: false, message: 'ER_FAIL_STORE_IMAGE'});
return ;
}
logger.infoMsg("IMAGE", "successfully create file blob");
image_info.user_id = user_id;
mysqlHandler.addAsset(image_info, asset_callback, assettag_callback );
};
var asset_callback = function (error, asset_info, asset_id) {
if (error) {
logger.errorMsg("IMAGE","failed to insert asset item: "+error);
//resp.json({ success: false, message: 'ER_FAIL_UPDATE_ASSET_DB'});
resp_callback(error,
{ success: false, message: 'ER_FAIL_UPDATE_ASSET_DB'});
return ;
}
logger.infoMsg("IMAGE","successfully insert asset item with ID: "+asset_id);
var file_name = image_info.hash+'.'+image_info.suffix;
image_info.id = asset_id;
generate_access_url(image_info.container, file_name);
};
var remaining_tag_count = image_info.tags.length;
var assettag_callback = function (error, tag, assetid) {
if (error) {
logger.errorMsg("IMAGE","failed to insert assettag: "+error);
//resp.json({ success: false, message: 'ER_FAIL_UPDATE_ASSETTAG_DB'});
resp_callback(error,
{ success: false, message: 'ER_FAIL_UPDATE_ASSETTAG_DB'});
return ;
}
remaining_tag_count--;
logger.infoMsg("IMAGE","successfully insert assettag: " +
tag +" for asset: "+assetid);
if (remaining_tag_count === 0){
logger.infoMsg("IMAGE","successfully insert ALL assettags for asset: "+
assetid);
}
};
utilities.calcFileChecksum(image_info.path, file_checksum_cb);
};
var path = require('path');
var mime = require('mime');
var storage = require('azure-storage');
try {
var config = require('./config');
} catch(e) {
console.log('Please create and configure a config.js file using config.template.js');
process.exit(1);
}
var client = storage.createBlobService(config.storageName, config.storageKey);
var blobs = [];
function aggregateBlobs(err, result, cb) {
if (err) {
cb(er);
} else {
blobs = blobs.concat(result.entries);
if (result.continuationToken !== null) {
client.listBlobsSegmented(
config.containerName,
result.continuationToken,
aggregateBlobs);
} else {
cb(null, blobs);
}
}
}
client.listBlobsSegmented(config.containerName, null, function(err, result) {
function init (options) {
azureOptions = options;
blobService = azure.createBlobService(azureOptions.account, azureOptions.key);
}
before(function () {
blobService = azure.createBlobService(snowmaker.connectionString);
});
/**
* Page blob basics.
* @ignore
*
* @param {config} config The configuration which contains the connectionString.
* @param {errorOrResult} callback The callback function.
*/
function basicStorageBlockBlobOperations(config, callback) {
// Create a blob client for interacting with the blob service from connection string
// How to create a storage connection string - http://msdn.microsoft.com/en-us/library/azure/ee758697.aspx
var blobService = storage.createBlobService(config.connectionString);
var fileToUpload = "HelloWorld.dat";
writeRandomFile(fileToUpload, 1024);
var blockBlobContainerName = "demoblockblobcontainer-" + guid.v1();
var blockBlobName = "demoblockblob-" + fileToUpload;
console.log('Block Blob Sample');
// Create a container for organizing blobs within the storage account.
console.log('1. Creating Container');
blobService.createContainerIfNotExists(blockBlobContainerName, function (error) {
if (error) {
callback(error);
} else {
// To view the uploaded blob in a browser, you have two options. The first option is to use a Shared Access Signature (SAS) token to delegate
// access to the resource. See the documentation links at the top for more information on SAS. The second approach is to set permissions
// to allow public access to blobs in this container. Uncomment the line below to use this approach. Then you can view the image
// using: https://[InsertYourStorageAccountNameHere].blob.core.windows.net/demoblockblobcontainer-[guid]/demoblockblob-HelloWorld.png
// Upload a BlockBlob to the newly created container
console.log('2. Uploading BlockBlob');
blobService.createBlockBlobFromLocalFile(blockBlobContainerName, blockBlobName, fileToUpload, function (error) {
if (error) {
callback(error);
} else {
// List all the blobs in the container
console.log('3. List Blobs in Container');
listBlobs(blobService, blockBlobContainerName, null, null, function (error, results) {
if (error) {
callback(error);
} else {
for (var i = 0; i < results.length; i++) {
console.log(util.format(' - %s (type: %s)'), results[i].name, results[i].blobType);
}
// Download a blob to your file system
console.log('4. Download Blob');
var downloadedFileName = util.format('CopyOf%s', fileToUpload);
blobService.getBlobToLocalFile(blockBlobContainerName, blockBlobName, downloadedFileName, function (error) {
if (error) {
callback(error);
} else {
// Create a read-only snapshot of the blob
console.log('5. Create a read-only snapshot of the blob');
blobService.createBlobSnapshot(blockBlobContainerName, blockBlobName, function (error, snapshotId) {
if (error) {
callback(error);
} else {
// Create three new blocks and upload them to the existing blob
console.log('6. Create three new blocks and upload them to the existing blob');
var buffer = getRandomBuffer(1024);
var blockIds = [];
var blockCount = 0;
var blockId = getBlockId(blockCount);
var uploadBlockCallback = function (error) {
if (error) {
callback(error);
} else {
blockCount++;
if (blockCount <= 3) {
blockId = getBlockId(blockCount);
blockIds.push(blockId);
blobService.createBlockFromText(blockId, blockBlobContainerName, blockBlobName, buffer, uploadBlockCallback);
} else {
// Important: Please make sure that you call commitBlocks in order to commit the blocks to the blob
var blockList = { 'UncommittedBlocks': blockIds };
blobService.commitBlocks(blockBlobContainerName, blockBlobName, blockList, function (error) {
// Clean up after the demo
console.log('7. Delete block Blob and all of its snapshots');
var deleteOption = { deleteSnapshots: storage.BlobUtilities.SnapshotDeleteOptions.BLOB_AND_SNAPSHOTS };
blobService.deleteBlob(blockBlobContainerName, blockBlobName, deleteOption, function (error) {
try { fs.unlinkSync(downloadedFileName); } catch (e) { }
if (error) {
callback(error);
} else {
// Delete the container
console.log('8. Delete Container');
blobService.deleteContainerIfExists(blockBlobContainerName, function (error) {
try { fs.unlinkSync(fileToUpload); } catch (e) { }
callback(error);
});
}
});
});
}
}
};
blockIds.push(blockId);
blobService.createBlockFromText(blockId, blockBlobContainerName, blockBlobName, buffer, uploadBlockCallback);
}
});
}
});
}
});
}
});
}
});
}
function TableRequest(account, params) {
this.params = params;
this.blobSvc = azure.createBlobService(account.name, account.key);
}
var azure = require('azure-storage');
var Promise = require('promise');
var uuid = require('node-uuid');
var blobSvc = azure.createBlobService();
module.exports = {
upload: function(data) {
return new Promise(function(fulfil, reject) {
var filename = uuid.v4() + '.png';
var stream = blobSvc.createWriteStreamToBlockBlob('regardcharts', filename, {
contentType: 'image/png'
});
stream.on('error', reject);
stream.on('close', function() {
fulfil('https://regardproduction.blob.core.windows.net/regardcharts/' + filename);
});
stream.end(new Buffer(data));
});
}
}
req.webtaskContext.read('history', {}, function (err, data) {
let startCheckpointId = typeof data === 'undefined' ? null : data.checkpointId;
// Initialize both clients.
const auth0 = new Auth0({
domain: ctx.data.AUTH0_DOMAIN,
clientID: ctx.data.AUTH0_GLOBAL_CLIENT_ID,
clientSecret: ctx.data.AUTH0_GLOBAL_CLIENT_SECRET
});
const blobService = azure.createBlobService(ctx.data.STORAGE_ACCOUNT_NAME, ctx.data.STORAGE_ACCOUNT_KEY);
// Start the process.
async.waterfall([
(callback) => {
auth0.getAccessToken((err) => {
if (err) {
console.log('Error authenticating:', err);
}
return callback(err);
});
},
(callback) => {
try {
blobService.createContainerIfNotExists(ctx.data.STORAGE_CONTAINER_NAME, (err) => callback(err));
} catch (e) {
return callback(e);
}
},
(callback) => {
const getLogs = (context) => {
console.log(`Downloading logs from: ${context.checkpointId || 'Start'}.`);
context.logs = context.logs || [];
auth0.getLogs({ take: 200, from: context.checkpointId }, (err, logs) => {
if (err) {
return callback(err);
}
if (logs && logs.length) {
logs.forEach((l) => context.logs.push(l));
context.checkpointId = context.logs[context.logs.length - 1]._id;
return setImmediate(() => getLogs(context));
}
console.log(`Total logs: ${context.logs.length}.`);
return callback(null, context);
});
};
getLogs({ checkpointId: startCheckpointId });
},
(context, callback) => {
context.logs = context.logs.map((record) => {
let level = 0;
record.type_code = record.type;
if (logTypes[record.type]) {
level = logTypes[record.type].level;
record.type = logTypes[record.type].event;
}
let agent = useragent.parse(record.user_agent);
record.os = agent.os.toString();
record.os_version = agent.os.toVersion();
record.device = agent.device.toString();
record.device_version = agent.device.toVersion();
return record;
});
callback(null, context);
},
(context, callback) => {
console.log('Uploading blobs...');
async.eachLimit(context.logs, 5, (log, cb) => {
const date = moment(log.date);
const url = `${date.format('YYYY/MM/DD')}/${date.format('HH')}/${log._id}.json`;
console.log(`Uploading ${url}.`);
blobService.createBlockBlobFromText(ctx.data.STORAGE_CONTAINER_NAME, url, JSON.stringify(log), cb);
}, (err) => {
if (err) {
return callback(err);
}
console.log('Upload complete.');
return callback(null, context);
});
}
], function (err, context) {
if (err) {
console.log('Job failed.');
return req.webtaskContext.write('history', JSON.stringify({checkpointId: startCheckpointId}), {}, function (error) {
if (error) return res.status(500).send(error);
res.status(500).send({
error: err
});
});
}
console.log('Job complete.');
return req.webtaskContext.write('history', JSON.stringify({checkpointId: context.checkpointId, totalLogsProcessed: context.logs.length}), {}, function (error) {
if (error) return res.status(500).send(error);
res.sendStatus(200);
});
});
});