printToFile: function(config,report,filename,extension,createNew) {
var f, myfile,
keys = [], values = [];
for(var key in report)
{
if(report.hasOwnProperty(key))
{
keys.push(key);
values.push(report[key]);
}
}
if(phantom.args[3] && phantom.args[3] != 'wipe'){
myfile = 'reports/' + filename + '-' + phantom.args[3] + '.' + extension;
}else{
myfile = 'reports/' + filename + '.' + extension;
}
if(!createNew && fs.exists(myfile)){
//file exists so append line
try{
if(extension === 'json'){
var phantomLog = [];
var tempLine = JSON.parse(fs.read(myfile));
if(Object.prototype.toString.call( tempLine ) === '[object Array]'){
phantomLog = tempLine;
}
phantomLog.push(report);
fs.remove(myfile);
f = fs.open(myfile, "w");
f.writeLine(JSON.stringify(phantomLog));
f.close();
}else{
f = fs.open(myfile, "a");
f.writeLine(values);
f.close();
}
} catch (e) {
console.log("problem appending to file",e);
}
}else{
if(fs.exists(myfile)){
fs.remove(myfile);
}
//write the headers and first line
try {
f = fs.open(myfile, "w");
if(extension === 'json'){
f.writeLine(JSON.stringify(report));
}else{
f.writeLine(keys);
f.writeLine(values);
}
f.close();
} catch (e) {
console.log("problem writing to file",e);
}
}
}
nCasperSports.run(function() {
//this.echo("========================== PageInfo =============================\n");
//require("utils").dump(pageInfo);
//this.echo("========================== replyUrlList =============================\n");
//require("utils").dump(replyUrlList);
var i, il;
var fs = require('fs');
if(fs.exists("/projects/GiantEagle/webcrawler/dat/nate/eco/article_url.in")){
fs.remove("/projects/GiantEagle/webcrawler/dat/nate/eco/article_url.in");
}
if(fs.exists("/projects/GiantEagle/webcrawler/dat/nate/eco/reply_url.in")){
fs.remove("/projects/GiantEagle/webcrawler/dat/nate/eco/reply_url.in");
}
if(fs.exists("/projects/GiantEagle/webcrawler/dat/nate/eco/title.in")){
fs.remove("/projects/GiantEagle/webcrawler/dat/nate/eco/title.in");
}
var articleStream = fs.open("/projects/GiantEagle/webcrawler/dat/nate/eco/article_url.in", 'w');
var replyStream = fs.open("/projects/GiantEagle/webcrawler/dat/nate/eco/reply_url.in", 'w');
var titleStream = fs.open("/projects/GiantEagle/webcrawler/dat/nate/eco/title.in", 'w');
for(i = 0, il = replyUrlList.length; i < il; i++){
articleStream.write(replyUrlList[i]['article_url'] + ';' + i + '\n');
replyStream.write('\"' + replyUrlList[i]['reply_url'] + '\"' + ';' + i + '\n');
titleStream.write(replyUrlList[i]['title'] + ';' + i + '\n');
}
this.echo("File WRITE Finished\n");
articleStream.close(); replyStream.close(); titleStream.close();
this.exit(0);
});
exports.fetchFromGithub = function (url, name, version) {
var source = {
location: url,
name: name,
version: version
};
utils.processGithubRepository(source);
var realFile = source.filename;
var appPath = module.appPath();
if (appPath === undefined) {
fs.remove(realFile);
throw "javascript.app-path not set, rejecting app loading";
}
var path = fs.join(appPath, source.name + "-" + source.version);
if (fs.exists(path)) {
fs.remove(realFile);
var err = new ArangoError();
err.errorNum = arangodb.errors.ERROR_APP_ALREADY_EXISTS.code;
err.errorMessage = arangodb.errors.ERROR_APP_ALREADY_EXISTS.message;
throw err;
}
fs.makeDirectoryRecursive(path);
fs.unzipFile(realFile, path, false, true);
var gitFilename = "/gitinfo.json";
fs.write(path+gitFilename, JSON.stringify(url));
exports.scanAppDirectory();
return "app:" + source.name + ":" + source.version;
};
casper.then(function() {
fs.remove(fname+'.gfjm.1.png');
fs.remove(fname+'.gfjm.1.html');
fs.remove(fname+'.gfjm.2.png');
fs.remove(fname+'.gfjm.2.html');
casper.echo('OK');
// casper.echo('No Record 1');
});
setTimeout(function() {
if (fs.exists(html_file)) fs.remove(html_file);
if (fs.exists(img_file)) fs.remove(img_file);
fs.write(html_file, content, 'w');
if (scr) page.render(img_file, { format: "png" });
console.log(html_file);
phantom.exit(html_file);
}, 3000);
page.onConsoleMessage = function (msg) {
var time = timestamp();
console.log(time + msg);
var f = fs.open(wd + botName + "/log.txt", "a");
f.writeLine(time + msg);
f.close();
// This just might be the only way to access the internal DOM of the page
var transReq = msg.match("<req_trans>");
if (transReq)
{
// Remove previous translations, if any
if (fs.exists("/etc/pyphantomjs/translation/translation.txt"))
{
fs.remove("/etc/pyphantomjs/translation/translation.txt");
}
// Send out a translation request. The translation daemon should pick it up.
var trans = msg.replace("<req_trans>", "");
var f = fs.open("/etc/pyphantomjs/translation/translationRequest.txt", "w");
f.writeLine(trans);
f.close();
}
var deadVidCheckReq = msg.match("<req_deadvid_check>");
if (deadVidCheckReq)
{
// Remove previous playlist file
if (fs.exists("/etc/pyphantomjs/deadVideoCheck/playlist.txt"))
{
fs.remove("/etc/pyphantomjs/deadVideoCheck/playlist.txt");
}
// Output the innerHTML for the playlist with thumbnails. The video checker daemon will pick it up.
var ded = msg.replace("<req_deadvid_check>", "");
var f = fs.open("/etc/pyphantomjs/deadVideoCheck/playlist.txt", "w");
f.writeLine(ded);
f.close();
}
var transReqCheck = msg.match("<req_trans_check>");
if (transReqCheck)
{
checkTranslationResponse();
}
var deadVidReqCheck = msg.match("<req_deadvid_check_check>");
if (deadVidReqCheck)
{
checkDeadVideoResponse();
}
};
var copyImagesToReport = function copyImagesToReport(images) {
var baselineFilename = filename(images.baseline),
currentFilename = filename(images.current),
baselineReport = 'report/images/baseline/' + baselineFilename,
currentReport = 'report/images/current/' + currentFilename;
try { fs.remove(baselineReport); } catch(e) {}
try { fs.remove(currentReport); } catch(e) {}
fs.copy(images.baseline, baselineReport);
fs.copy(images.current, currentReport);
};
dCasperSports.run(function () {
//this.echo("========================== ReplyList =============================\n");
//this.echo("List Length = " + replyList.length);
//require("utils").dump(replyList);
//this.echo("========================== PageInfo =============================\n");
//this.echo("List Length = " + pageInfo.length);
//require("utils").dump(pageInfo);
var i, il;
(pageInfo.length < 30)?(il = pageInfo.length):(il = 30);
for(i = 0; i < il; i++){
var meta = {}
meta['article_url'] = pageInfo[i]['article_url'];
meta['title'] = "D " + pageInfo[i]['title'];
meta['reply_url'] = replyList[i]['reply_url'];
meta['best_replies'] = replyList[i]['best_replies'];
completeList.push(meta);
}
//this.echo("========================== CompleteList=============================\n");
//this.echo("List Length = " + completeList.length);
//require("utils").dump(completeList);
var fs = require('fs');
if(fs.exists("/projects/GiantEagle/webcrawler/daum/mobile_crwl/m_daum_spo_article_url.in")){
fs.remove("/projects/GiantEagle/webcrawler/daum/mobile_crwl/m_daum_spo_article_url.in");
}
var articleStream = fs.open("/projects/GiantEagle/webcrawler/daum/mobile_crwl/m_daum_spo_article_url.in", 'w');
for(i = 0; i < il; i++){
articleStream.write(pageInfo[i]['article_url'] + '\n');
}
articleStream.close();
if(fs.exists("/projects/GiantEagle/webcrawler/dat/crwl/daum_spo.crwl")){
fs.remove("/projects/GiantEagle/webcrawler/dat/crwl/daum_spo.crwl");
}
fs.write("/projects/GiantEagle/webcrawler/dat/crwl/daum_spo.crwl", JSON.stringify({
channel : 'daum_spo',
at : new Date(),
data : completeList
}), 'w');
this.echo("Done writing /projects/GiantEagle/webcrawler/dat/crwl/daum_spo.crwl");
this.exit(0);
});
fs.copy('./public/uploads/tmp/' + req.body.logo, './public/lib/images/logo1.png', function(err) {
if (err) {
console.error(err);
} else {
console.log("move file success!")
// remove tmp file
fs.remove('./public/uploads/tmp/' + req.body.logo,function(e){
if(e)
return console.error(e);
});
var easyimg = require('easyimage');
// crop file
// small size
easyimg.thumbnail(
{
src: './public/lib/images/logo1.png', dst: './public/lib/images/logo1.png',
width:168, height:32
},
function(err, image) {
if (err) console.error(err)
else
console.log('Resized : ' + image.width + ' x ' + image.height);
}
);
}
});
page.open(url, function (status) {
if (status !== 'success' || (statusCode != 200 && statusCode != null)) {
if (fs.exists(output)) {
fs.remove(output);
}
try {
fs.touch(output);
} catch (e) {
console.log(e);
}
error('Unable to load the URL: ' + url + ' (HTTP ' + statusCode + ')');
} else {
window.setTimeout(function () {
page.render(output + '_tmp.pdf');
if (fs.exists(output)) {
fs.remove(output);
}
try {
fs.move(output + '_tmp.pdf', output);
} catch (e) {
error(e);
}
console.log('Rendered to: ' + output, new Date().getTime());
phantom.exit(0);
}, render_time);
}
});
function rubyTests (options, ssl) {
var instanceInfo;
if (ssl) {
instanceInfo = startInstance("ssl",options);
}
else {
instanceInfo = startInstance("tcp",options);
}
var tmpname = fs.getTempFile()+".rb";
fs.write(tmpname,'RSpec.configure do |c|\n'+
' c.add_setting :ARANGO_SERVER\n'+
' c.ARANGO_SERVER = "' +
instanceInfo.endpoint.substr(6) + '"\n'+
' c.add_setting :ARANGO_SSL\n'+
' c.ARANGO_SSL = "' + (ssl ? '1' : '0') + '"\n'+
' c.add_setting :ARANGO_USER\n'+
' c.ARANGO_USER = "******"\n'+
' c.add_setting :ARANGO_PASSWORD\n'+
' c.ARANGO_PASSWORD = "******"\n'+
'end\n');
var logsdir = fs.join(findTopDir(),"logs");
try {
fs.makeDirectory(logsdir);
}
catch (err) {
}
var files = fs.list(fs.join("UnitTests","HttpInterface"));
var result = {};
var args;
var i;
for (i = 0; i < files.length; i++) {
var n = files[i];
if (n.substr(0,4) === "api-" && n.substr(-3) === ".rb") {
print("Considering",n,"...");
if ((n.indexOf("-cluster") === -1 || options.cluster) &&
(n.indexOf("-noncluster") === -1 || options.cluster === false) &&
n.indexOf("replication") === -1) {
args = ["--color", "-I", fs.join("UnitTests","HttpInterface"),
"--format", "d", "--require", tmpname,
fs.join("UnitTests","HttpInterface",n)];
var pid = executeExternal("rspec", args);
var r = statusExternal(pid, true);
result[n] = r.exit;
if (r.exit !== 0 && !options.force) {
break;
}
}
else {
print("Skipped because of cluster/non-cluster or replication.");
}
}
}
print("Shutting down...");
fs.remove(tmpname);
shutdownInstance(instanceInfo,options);
print("done.");
return result;
}
var update = function() {
var url = utils.buildGithubUrl(getFishbowlUrl());
var filename = fs.getTempFile("downloads", false);
var path = fs.getTempFile("zip", false);
try {
var result = download(url, "", {
method: "get",
followRedirects: true,
timeout: 30
}, filename);
if (result.code < 200 || result.code > 299) {
throwDownloadError("Github download from '" + url + "' failed with error code " + result.code);
}
updateFishbowlFromZip(filename);
filename = undefined;
}
catch (err) {
if (filename !== undefined && fs.exists(filename)) {
fs.remove(filename);
}
try {
fs.removeDirectoryRecursive(path);
}
catch (ignore) {
}
throw err;
}
};
printToFile: function(report, config) {
var keys = [], values = [];
for(var key in report) {
if(report.hasOwnProperty(key)) {
keys.push(key);
values.push(report[key]);
}
}
var myfile = './performance/' + encodeURIComponent(config.url) + '.' + config.format;
if(fs.exists(myfile)) {
fs.remove(myfile);
}
//write the headers and first line
var f = fs.open(myfile, "w");
switch (config.format) {
case 'json':
f.writeLine(JSON.stringify(report, undefined, 4));
break;
case 'xml':
f.writeLine(this.formatAsJUnit(keys, values));
break;
default:
f.writeLine(keys);
f.writeLine(values);
break;
}
f.close();
}
casper.waitUntilVisible('#recaptcha_challenge_image', function() {
var captcha_url = this.evaluate(function () {
return document.querySelector('#recaptcha_challenge_image').src;
});
if (fs.exists(captchaSavePath)) {
fs.remove(captchaSavePath);
}
this.download(captcha_url, captchaSavePath);
var code = null;
if (curFilePath.length > 1) {
curFilePath.pop();
fs.changeWorkingDirectory(curFilePath.join('/'));
}
cp.execFile('./antigate.sh', ['d70a01bb7ccbfa9869d3ebd71878fe88', captchaSavePath], {}, function(_, stdout, stderr) {
code = stdout.substring(0, stdout.indexOf('\n'));
});
this.waitFor(function check() {
return code !== null;
}, function () {
this.fill('form#recaptcha', {'recaptcha_response_field': code.trim()}, true);
}, null, 35000);
});
casper.then(function printExpiringSoon() {
var statfile = fs.open(config.statfile + '.tmp', 'w');
// utils.dump(teams);
// utils.dump(programs);
// utils.dump(profiles);
var team, program, cert, profile;
for (var i=0, teams_count=teams.length; i<teams_count; i++) {
team = teams[i];
program = programs[team.id];
if (program['expires_in'] <= config.deadline) {
statfile.writeLine(program['type'] + " for \"" + team.name + "\" team will expire in " + program['expires_in'] + " day(s) " + " (" + program['expires'] + ")");
hasExpirations = true;
}
for (var cert_name in certificates[team.id]) {
cert = certificates[team.id][cert_name];
if (cert['expires_in'] <= config.deadline) {
statfile.writeLine("Cert \"" + cert['type'] + ": " + cert_name + "\" for \"" + team.name + "\" team will expire in " + cert['expires_in'] + " day(s) " + " (" + cert['expires'] + ")");
hasExpirations = true;
}
}
for (var profile_name in profiles[team.id]) {
profile = profiles[team.id][profile_name];
if (profile['expires_in'] <= config.deadline) {
statfile.writeLine("Profile \"" + profile['type'] + ": " + profile_name + "\" for " + team.name + " team will expire in " + profile['expires_in'] + " day(s) " + " (" + profile['expires'] + ") Status: " + profile['status']);
hasExpirations = true;
}
}
}
if (! hasExpirations) { statfile.writeLine("No expirations in comming " + config.deadline + " day(s)"); }
statfile.close();
if (fs.exists(config.statfile)) {
fs.remove(config.statfile);
}
fs.move(config.statfile + '.tmp', config.statfile);
});
exports.tearDown = function() {
try {
fs.remove(testFile);
} catch (e) {
// do nothing
}
};
var intr = setInterval( function() {
if(! fs.isFile('captcha.png') )
{
web_browser.render('captcha.png')
console.log('warn: enter chars from captcha.png')
}
if( captcha = system.stdin.readLine().trim() )
{
fs.remove('captcha.png')
var error = web_browser.evaluateJavaScript(
"function() {\n\
var captcha = document.getElementById('captcha')\n\
if(! captcha)\n\
return 'captcha field not found'\n\
captcha.value = '" + captcha.replace(/[\r\n]/g, '') + "'\n\
node = captcha\n\
while( node = node.parentNode )\n\
if( node.tagName.toLowerCase() == 'form' || node.tagName.toLowerCase() == 'body' )\n\
break\n\
if( node.tagName.toLowerCase() != 'form')\n\
return 'captcha form not found'\n\
}"
)
web_browser.sendEvent('keypress', web_browser.event.key.Enter)
captcha = false
clearInterval(intr)
if(error)
console.log(error)
}
else
console.log('warn: enter chars from captcha.png')
}, 500 )
cleanupActions.startServers = function (dispatchers, cmd, isRelaunch) {
console.info("Cleaning up DBservers...");
var dataPath = fs.makeAbsolute(cmd.dataPath);
if (dataPath !== cmd.dataPath) { // path was relative
dataPath = fs.normalize(fs.join(ArangoServerState.dataPath(),cmd.dataPath));
}
var logPath = fs.makeAbsolute(cmd.logPath);
if (logPath !== cmd.logPath) { // path was relative
logPath = fs.normalize(fs.join(ArangoServerState.logPath(), cmd.logPath));
}
var servers = cmd.DBservers.concat(cmd.Coordinators);
var i;
var logfile, datadir;
for (i = 0; i < servers.length; i++) {
var id = servers[i];
logfile = fs.join(logPath,"log-"+cmd.agency.agencyPrefix+"-"+id);
if (fs.exists(logfile)) {
fs.remove(logfile);
}
datadir = fs.join(dataPath,"data-"+cmd.agency.agencyPrefix+"-"+id);
if (fs.exists(datadir)) {
fs.removeDirectoryRecursive(datadir,true);
}
}
return {"error": false, "isStartServers": true};
};
var moveAppToServer = function(serviceInfo) {
if (! fs.exists(serviceInfo)) {
throwFileNotFound("Cannot find file: " + serviceInfo + ".");
}
var filePath;
var shouldDelete = false;
if (fs.isDirectory(serviceInfo)) {
filePath = utils.zipDirectory(serviceInfo);
shouldDelete = true;
}
if (fs.isFile(serviceInfo)) {
filePath = serviceInfo;
}
if (!filePath) {
throwBadParameter("Invalid file: " + serviceInfo + ". Has to be a direcotry or zip archive");
}
var response = arango.SEND_FILE("/_api/upload", filePath);
if (shouldDelete) {
try {
fs.remove(filePath);
}
catch (err2) {
arangodb.printf("Cannot remove temporary file '%s'\n", filePath);
}
}
if (! response.filename) {
throw new ArangoError({
errorNum: errors.ERROR_SERVICE_UPLOAD_FAILED.code,
errorMessage: errors.ERROR_SERVICE_UPLOAD_FAILED.message
+ ": " + String(response.errorMessage)
});
}
return response.filename;
};
page.open(address, function (status) {
if (status !== 'success' || (statusCode != 200 && statusCode != null)) {
console.log(statusCode, 'Unable to load the address!');
if (fs.exists(output)) {
fs.remove(output);
}
try {
fs.touch(output);
}
catch (e) {
phantom.exit(1);
throw e
}
phantom.exit(1);
} else {
window.setTimeout(function () {
page.render(output + '_tmp.pdf');
if (fs.exists(output)) {
fs.remove(output);
}
try {
fs.move(output + '_tmp.pdf', output);
}
catch (e) {
console.log(e);
phantom.exit(1);
throw e
}
console.log('rendered to: ' + output, new Date().getTime());
phantom.exit();
}, render_time);
}
});
page.open(address, function (status) {
console.log('[SHRIMP DEBUG] Status: ' + status)
if (status !== 'success' || (statusCode != 200 && statusCode != null)) {
console.log(statusCode, 'Unable to load the address!');
if (fs.exists(output)) {
fs.remove(output);
}
try {
fs.touch(output);
}
catch (e) {
phantom.exit(1);
throw e;
}
phantom.exit(1);
} else {
/* Render header/footer */
if (page.evaluate(function(){return typeof PhantomJSPrinting == "object";})) {
paperSize = page.paperSize;
paperSize.header = {};
paperSize.footer = {};
paperSize.header.height = page.evaluate(function() {
return PhantomJSPrinting.header.height;
});
paperSize.header.contents = phantom.callback(function(pageNum, numPages) {
return page.evaluate(function(pageNum, numPages){return PhantomJSPrinting.header.contents(pageNum, numPages);}, pageNum, numPages);
});
paperSize.footer.height = page.evaluate(function() {
return PhantomJSPrinting.footer.height;
});
paperSize.footer.contents = phantom.callback(function(pageNum, numPages) {
return page.evaluate(function(pageNum, numPages){return PhantomJSPrinting.footer.contents(pageNum, numPages);}, pageNum, numPages);
});
page.paperSize = paperSize;
}
console.log('[SHRIMP DEBUG] Rendering Time: ' + render_time);
window.setTimeout(function () {
page.render(output + '_tmp.pdf');
if (fs.exists(output)) {
fs.remove(output);
}
try {
fs.move(output + '_tmp.pdf', output);
}
catch (e) {
console.log(e);
phantom.exit(1);
throw e
}
console.log('[SHRIMP DEBUG] rendered to: ' + output, new Date().getTime());
phantom.exit();
}, render_time);
}
});
function printToFile(data) {
var f
, g
, html
, myfile
, fileid
, myjson
, jspath
, keys = []
, values = []
, extension = 'html';
if(!phantom.args[1]){
fileid = phantom.args[0].replace('http://','').replace('https://','').replace(/\//g,'');
fileid = fileid.split('?')[0];
myjson = 'speedreports/' + fileid + '.js';
myfile = 'speedreports/' + fileid + '.' + extension;
}else{
fileid = phantom.args[1];
myjson = fileid;
myfile = null;
}
if(myfile!==null){
try {
data = "var reportdata = " + data + ";";
if(fs.exists(myfile)){
fs.remove(myfile);
}
if(!fs.exists('speedreport.html')){
html = fs.read('/reports/PhantomJS/loadreport/speedreport.html');
}else{
html = fs.read('speedreport.html');
}
html=html.replace('{{REPORT_DATA_URI}}', fileid + '.js');
if(phantom.args[1]){
html=html.replace('{{url}}', phantom.args[1]);
}else{
html=html.replace('{{url}}', phantom.args[0]);
}
f = fs.open(myfile, "w");
f.write(html);
f.flush();
f.close();
} catch (e) {
console.log("problem writing to file",e);
}
}
try {
g = fs.open(myjson, "w");
g.write(data);
g.flush();
g.close();
} catch (e) {
console.error("problem writing to file",e);
}
}
exports.tearDown = function() {
fs.remove(db);
if (sysProp) {
System.setProperty("GEOEXPLORER_DATA", sysProp);
} else {
System.clearProperty("GEOEXPLORER_DATA");
}
}
setUp: function(test) {
if (fs.exists(this.testFile) && fs.isFile(this.testFile)) {
try {
fs.remove(this.testFile);
} catch (e) {
}
}
},
fs.list(suite.diffDir).forEach(function (item) {
var file = path.join(suite.diffDir, item);
if (fs.exists(file)
&& item.slice(-4) == '.png'
) {
fs.remove(file);
}
});
page2.open(temporary_outfile, function() {
page2.render(outfile, {quality: '100'});
// delete the remnants
fs.remove(temporary_outfile);
phantom.exit();
});
fs.listTree(root).forEach(function (file) {
if (file.match(/\.json$/)) {
try {
fs.remove(fs.join(root, file));
}
catch (err3) {
}
}
});
fileDelete(path) {
if (this.context.isProduction) {
throw new Error('File cannot be deleted in production mode.');
}
path = this.validatePath(path);
if (!_fs.isFile(path)) {
throw new Error('File is not existed.')
}
return _fs.remove(path);
}
casper.start('file://' + phantom.casperPath + '/tests/site/index.html', function() {
var imageUrl = 'file://' + phantom.casperPath + '/tests/site/images/phantom.png',
image = this.base64encode(imageUrl);
test.assertEquals(image.length, 6160, 'Casper.base64encode() can retrieve base64 contents');
this.download(imageUrl, '__test_logo.png');
test.assert(fs.exists('__test_logo.png'), 'Casper.download() downloads a file');
if (fs.exists('__test_logo.png')) {
fs.remove('__test_logo.png');
}
}).run(function() {
this.inspect = function (path) {
var fullPath = fs.join(fs.getTempPath(), path);
try {
var result = this.inspectUploadedFile(fullPath);
fs.remove(fullPath);
return result;
} catch (e) {
throw new ArangoError();
}
};