Esempio n. 1
0
function crawlItBaby(options, cb) {
    'use strict';
    phantom.create(function(error, browser) {

        var done = Q.defer();
        done.promise.then(function() {
            //todo: use done() to handle reject
            browser.exit();
            //create report
            cb({
                used: state.usedSelectors,
                unused: state.unusedSelectors,
                errors: state.errorSelectors,
                visited: state.visited
            });
        });

        state.queue.push(options.url);
        dequeueAndCrawl(browser, options.url, done);
        // return browser.createPage(function(error, page) { createPage(error, page, browser, options.url, options.url, done); });
        
    }, {
        phantomPath: require('phantomjs').path,
        // parameters: {
        //     proxy: '127.0.0.1:8888'
        // }
    });
}
Esempio n. 2
0
File: index.js Progetto: nylen/navit
Navit.prototype.__initPhantom__ = function (callback) {
  var self = this;
  var phantomParameters = {};

  // Fill phantom options
  _.forEach({
    loadImages: 'load-images',
    ignoreSslErrors: 'ignore-ssl-errors',
    sslProtocol: 'ssl-protocol',
    webSecurity: 'web-security',
    proxy: 'proxy',
    proxyType: 'proxy-type',
    proxyAuth: 'proxy-auth',
    cookiesFile: 'cookies-file'
  }, function (phantomOptionName, navitOptionName) {
    if (self.__options__.hasOwnProperty(navitOptionName)) {
      phantomParameters[phantomOptionName] = self.__options__[navitOptionName];
    }
  });

  phantom.create(function (err, ph) {
    if (err) {
      callback(err);
      return;
    }

    self.__phantom__ = ph;
    callback();
  }, { parameters: phantomParameters, phantomPath: require('phantomjs').path });
};
PhantomInstance.prototype.createPhantomProcess = function (callback) {
    var self = this;

    var parameters = {
        "web-security": false,
        "ignore-ssl-errors": "yes"
    };

    logger.debug('create phantom process');
    phantom_simple.create({parameters: parameters}, function (error, browser) {

        if (error) {
            logger.error('error creating phantom process: ' + error);
            return callback(error);
        }

        self.browser = browser;
        if (self.options.proxy) {
            self.browser.setProxy(self.options.proxy.host, self.options.proxy.port, self.options.proxy.protocol, null, null, function () {
                callback();
            });
            return;
        }

        callback();
    });
};
Esempio n. 4
0
	return new HorsemanPromise(function(resolve, reject) {
		phantom.create(instantiationOptions, function(err, instance) {
			debug('phantom created');
			horseman.phantom = instance;
			if (err) {
				return reject(err);
			}
			if (debugv.enabled) {
				horseman.phantom.onError = function(msg, trace) {
					var str = msg;
					trace.forEach(function(t) {
						var fun = t.function || '<anonymous>';
						str += '\n\t' + t.file + ': ' + t.line + ' in ' + fun;
					});
					return debugv('onPhantomError', str, horseman.options.port);
				};
			}
			instance.get('version', function(err, ver) {
				if (err) { return reject(err); }
				var version = ver.major + '.' + ver.minor + '.' + ver.patch;
				debug('phantom version ' + version);
				return horseman.pageMaker()
					.then(resolve)
					.catch(reject);
			});
		});
	})
Esempio n. 5
0
  	return new Promise(resolve => {
      driver.create(this.driverOpts, (err, browser) => {
        this.browser = browser
        browser.createPage((err, page) => {
          this.page = page;

          page.onPageCreated = (page) => {
            var pageObj = {
              page: page,
              url: null
            }
            this.childPages.push(pageObj)
            page.onUrlChanged = (url) => {
              pageObj.url = url;
            };
          }

          page.onConsoleMessage = (msg) => {
            if (argv['verbose']) {
              console.log('[Console]', msg)
            }
          }

          page.open(url, (err, status) => {
            this.onOpen()
            resolve(status)
          })
        })
      })
    })
Esempio n. 6
0
 before(function(done) {
   // Create the headless webkit browser.
   phantom.create({
     path: phantomjs.path,
     parameters: {
       // Use the test server as a proxy server, so that all requests
       // go to this server (instead of trying real DNS lookups).
       proxy: '127.0.0.1:8193',
       // Set the disk storage to zero to avoid persisting localStorage
       // between test runs.
       'local-storage-quota': 0
     }
   }, function(error, ph) {
     assert.ifError(error);
     // Open a page for browsing.
     _ph = ph;
     _ph.createPage(function(err, page) {
       _page = page;
       page.onConsoleMessage = function(msg) {
         console.log(msg);
       }
       // Set the size to a modern laptop size.
       page.set('viewportSize', { width: 1200, height: 900 }, function(err) {
         assert.ifError(err);
         // Point it to a blank page to start
         page.open('about:blank', function(err, status){
           assert.ifError(err);
           assert.equal(status, 'success');
           done();
         });
       });
     });
   });
 });
Esempio n. 7
0
    return new Promise(resolve => {
      driver.create(this.driverOpts, (err, browser) => {
        this.browser = browser
        browser.createPage((err, page) => {
          this.page = page;

          options.settings = options.settings || {}
          for (var i in options.settings) {
            page.set('settings.' + i, options.settings[i])
          }

          if (options.headers) {
            page.set('customHeaders', options.headers)
          }

          if (options.viewportSize) {
            page.set('viewportSize', options.viewportSize)
          }

          /**
           * Allow content to pass a custom function into onResourceRequested.
           */
          if (options.onResourceRequested) {
            page.setFn('onResourceRequested', options.onResourceRequested)
          }

          page.onResourceTimeout = (url) => {
            console.log('page timeout when trying to load ', url)
          }

          page.onPageCreated = (page) => {
            var pageObj = {
              page: page,
              url: null
            }

            this.childPages.push(pageObj)

            page.onUrlChanged = (url) => {
              pageObj.url = url;
            }

            page.onClosing = (closingPage) => {
              this.childPages = this.childPages.filter(eachPage => eachPage === closingPage)
            }
          }

          page.onConsoleMessage = (msg) => {
            if (argv['verbose']) {
              console.log('[Console]', msg)
            }
          }

          page.open(url, (err, status) => {
            this.onOpen()
            resolve(status)
          })
        })
      })
    })
Esempio n. 8
0
describe('login page', function() {
	
	var page = null;
	var browser = new Browser();
	
	var url = 'http://www.phantomjs.org/';
	
	phantom.create(function(p){
		console.log(p)
		page = p;
	});
	

	
	
		
	it('should show login form', function(done){

	});
	
	it('should refuse empty submissions');
	it('should refuse partial submissions');
	it('should keep values on partial submissions');
	it('should refuse invalid username');
	it('should refuse invalid password');
	it('should accept complete submissions');
});
Esempio n. 9
0
Http.prototype.getPageContent2 = function(cb) {
    var self = this;

    driver.create({ path: require('phantomjs').path }, function(err, browser) {

        return browser.createPage(function(err, page) {

            page.set('settings.loadImages', 'false', function() {

                return page.open(self.link, function(err, status) {

                    if (status === 'fail') {
                        cb(status, null);
                        browser.exit();
                    } else {
                        page.includeJs("http://ajax.googleapis.com/ajax/libs/jquery/1.6.1/jquery.min.js", function(err) {
                            if (err) console.log(err);
                            page.evaluate(function(err) {
                                if (err) console.log(err);
                                $("ul.list-unstyled.phones-view span").click();
                            });

                            setTimeout(function() {

                                page.evaluate(function() {
                                    window.scrollTo(0, document.body.scrollHeight);
                                });

                                setTimeout(function() {
                                    /*page.evaluate(function() {
                                        window.scrollTo(0, document.body.scrollHeight);
                                    });*/

                                    page.set('viewportSize', { width: 1024, height: 768 });
                                    page.render('capture.png');

                                    page.get('content', function(err, html) {
                                        page.evaluate(function() {
                                            window.scrollTo(0, document.body.scrollHeight);
                                        });
                                        ///setTimeout(function() {

                                        cb(err, html);
                                        browser.exit();
                                        //}, 10000);
                                    });

                                }, 10000);
                            }, 10000);

                        });
                    }
                });
            });

        });
    });
};
Esempio n. 10
0
    var screenshot = function(opts, cb) {
      var viewport = opts.viewport;
      var type = opts.type;
      var path = opts.path;
      var src = opts.src;
      var dest = opts.dest;
      var delay = opts.delay;
	  
      phantom.create(function(err, ph) {
        if (err) {
          grunt.fail.warn(err.message);
          return;
        }
        return ph.createPage(function(err, page) {
          if (viewport) {
            var sets = viewport.match(/(\d+)x(\d+)/);
            if (sets[1] && sets[2]) {
              page.set('viewportSize', {
                width: sets[1],
                height: sets[2]
              });
            }
          }
          page.set('zoomFactor', options.zoomFactor);
          return page.open(src, function(err, status) {
            //var target = type + '-' + viewport + '-' + dest;
			var target = dest;
            // Background problem under self-host server
            page.evaluate(function() {
              var style = document.createElement('style');
              var text = document.createTextNode('body { background: #fff }');
              style.setAttribute('type', 'text/css');
              style.appendChild(text);
              document.head.insertBefore(style, document.head.firstChild);
            });
			
            if (delay) {
              setTimeout(function() {
                page.render(path + '/' + target, function() {
                  grunt.log.writeln('Delay ' + delay + ' to take a screenshot to ' + target);
                  ph.exit();
                  cb();
                });
              }, delay);
            } else {
              page.render(path + '/' + target, function() {
				i++;
				grunt.log.writeln("done "+i+'/'+obj.length + " to " + target);
                ph.exit();
                cb();
              });
            }
          });
        });
      }, {
        phantomPath: require('phantomjs').path
      });
    };
Esempio n. 11
0
		createBrowser: guardAgainstTimeout(function(next) {
			phantom.create(options.phantom, function(error, browser) {
				if (error) {
					return next(error);
				}
				options.log.info('PhantomJS browser created');
				state.browser = browser;
				next();
			});
		}),
Esempio n. 12
0
	return new HorsemanPromise(function(resolve, reject) {
		phantom.create(instantiationOptions, function(err, instance) {
			debug('phantom created.')
			horseman.phantom = instance;
			if (err) reject(err);
			else {
				return horseman.pageMaker()
					.then(resolve)
					.catch(reject);
			};
		});
	}).bind(horseman);
Esempio n. 13
0
        var screenshot = function(opts, cb) {
            var viewport = opts.viewport;
            var type     = opts.type;
            var path     = opts.path;
            var url      = opts.url;
            var file     = opts.file;

            phantom.create(function(err, ph) {

                ph.createPage(function(err, page) {

                    if (viewport) {
                        var sets = viewport.match(/(\d+)x(\d+)/);
                        if (sets[1] && sets[2]) {
                            page.set('viewportSize', {
                                width: sets[1],
                                height: sets[2]
                            });
                        }
                    }

                    page.set('zoomFactor', 1);

                    page.open(url, function(err, status) {
                        if(err) {
                            console.log(err, status);
                        }

                        var target = (file.replace(/\/|\?|=|&/g, '_')) + '_'+ viewport + '.png';

                        // Background problem under self-host server
                        page.evaluate(function() {
                            var style = document.createElement('style');
                            var text = document.createTextNode('body { background: #fff }');
                            style.setAttribute('type', 'text/css');
                            style.appendChild(text);
                            document.head.insertBefore(style, document.head.firstChild);
                        });

                        setTimeout(function() {
                            page.render(path + '/' + target, function() {
                                grunt.log.writeln('Taking a screenshot of: ' + target);
                                ph.exit();
                                cb();
                            });
                        }, options.timeout);
                    });
                });
            });
        };
Esempio n. 14
0
 before(function(done) {
   phantom.create(function(error,ph) {
     assert.ifError(error);
     _ph = ph;
     _ph.createPage(function(err,page) {
       _page = page;
       page.open('about:blank', function(err, status){
         assert.ifError(err);
         assert.equal(status,'success');
         done();
       });
     });
   }, { phantomPath: phantomjs.path });
 });
Esempio n. 15
0
function openPage(opts) {
  var phantomInstance;
  return phantom.create().then(function (ph) {
    phantomInstance = ph;
    return utils.promisify(ph.createPage)().then(function (page) {
      return createPage({
        options: opts,
        page: page,
        ph: ph
      });
    });
  }).catch(function () {
    phantomInstance.exit();  // Abort PhantomJS when an error occurs.
  });
}
Esempio n. 16
0
PhantomClient.prototype.init = function () {
	var thisClient = this;
	if (thisClient.phantom) {
		this.phantom.exit();
		this.phantom = null;
	}
	PhantomJS.create(function(err, phantom) {
		if (err || !phantom) {
			return thisClient.emit('error', 'phantom.create', err);
		}
		thisClient.phantom = phantom;
		thisClient.isReady = true;
		thisClient._get();
	}, this.phantomSettings);
	thisClient.isInitialized = true;
};
 function (done) {
     phantom.create(function (err, ph) {
         if (err) {
             return done(err);
         }
         instances.push({
             instance: ph,
             numPages: 0
         });
         --numThreads;
         return done(null);
     }, {
         phantomPath: phantomPath,
         parameters: {'ignore-ssl-errors': 'yes'}
     });
 },
Esempio n. 18
0
				/*Create phantom & page*/function (next) {
					return _phantomjs.create(phantomCfg.phantomOpt || phantomCfg, function(err, phantom) {

						if (err || !phantom) return next('phantomjs.create.err' + err);

						PhantomHelper.phantom = phantom;
						
						return phantom.createPage(function(err, newPage) {
							if (err || !newPage) return next('phantom.createPage.err' + err);

							PhantomHelper.page = page = newPage;

							return next();
						});
					});
				},
Esempio n. 19
0
function init( callback ) {

  phantom.create( createPhantomInstance,
    {
      phantomPath: phantomPath,
      parameters: { 'ignore-ssl-errors': 'yes' }
    } );

  function createPhantomInstance( err, phantomInstance ) {

    if ( err ) {
      return callback( err );
    }

    ph = phantomInstance;
    callback();
  }
}
Esempio n. 20
0
module.exports.createFixtureFromUrl = function(url, dest){
  phantom.create(function(err,ph) {
    return ph.createPage(function(err,page) {
      return page.open(url, function(err,status) {
        console.log(status);
        page.get('content', function (err,html) {
          // write the file to the specified location

          var fileName = __dirname + "/fixtures/" + dest + ".html";
          fs.writeFile(fileName, html, function (err) {
            if (err) throw err;

            console.log("file written");
          });
        });
      });
    });
  });
};
Esempio n. 21
0
function makeRequest(config, callback) {
  var phantom = require('node-phantom-simple');
  var params = {
    phantomPath: require('phantomjs').path
  };
  phantom.create(create, params);

  function create(err, ph) {
    if (err) {
      callback(err);
    } else {
      singleTest(ph, config, function(networkEvents) {
        ph.exit();
        var returnObject = returnObjectLib.getReturnObjectFromXhrNetworkEvents(config.url, networkEvents);
        callback(returnObject);
      });
    }
  }
}
Esempio n. 22
0
	it('Completed.', function(done) {
		this.timeout(10000);
		phantom.create(parameters, function(err, ph) {
			assert(typeof ph === 'object', 'Phantom object is present');
			var collection = new FlowCollection(ph, o);
			collection.run()
				.then(function() {
					console.log("Now we're completed with the whole thing");
					done();
				});

			// it('Waiting for test suite to complete', function(done) {
			// collection.completed(done);
			// });
			// done();
			// done();

		});

	});
Esempio n. 23
0
Wrathjs.prototype._initPhantom = function(parameters, phPath) {
    var ph = null;
    var opts = {
        parameters: parameters || {},
        path: phPath || 'phantomjs'
    };
    phantom.create(opts, function (err, phInCb) {
        if (err) {
            console.error(err);
            process.exit();
        } else {
            ph = phInCb;
        }
    });

    while (ph === null) {
        deasync.runLoopOnce();
    }

    return ph;
};
Esempio n. 24
0
 function request(create,exit) {
     if(activePhantoms.length >= phantomMax) {
         return;
     }
     phantom.create(function(err,ph){
         if(err) {
             eyeball.logger.error(err);
             return;
         }
         respond(ph,exit);
         create(ph);
     }, {
         phantomPath : phantomjs.path,
         parameters : {
             'proxy' : eyeball.HOSTNAME+":"+eyeball.PROXYPORT,
             'web-security' : false,
             'ignore-ssl-errors' : true,
             'disk-cache' : false
         }
     });
 }
Esempio n. 25
0
    var screenshot = function(opts, cb) {
      var viewport = opts.viewport;
      var url = opts.url;
      var filename = opts.filename;
      var type = opts.type;
      var path = opts.path;

      phantom.create(function(err, ph) {
        ph.createPage(function(err, page) {
          if (viewport) {
            var sets = viewport.match(/(\d+)x(\d+)/);
            if (sets[1] && sets[2]) {
              page.set('viewportSize', {
                width: sets[1],
                height: sets[2]
              });
            }
          }
          page.set('zoomFactor', 1);
          page.open(url, function(err, status) {
            var dest = filename + '.' + type;

            // Background problem under self-host server
            page.evaluate(function() {
              var style = document.createElement('style');
              var text = document.createTextNode('body { background: #fff }');
              style.setAttribute('type', 'text/css');
              style.appendChild(text);
              document.head.insertBefore(style, document.head.firstChild);
            });

            page.render(path + '/' + dest, function() {
              grunt.log.writeln('Take a screenshot to ' + dest);
              ph.exit();
              cb();
            });
          });
        });
      });
    };
Esempio n. 26
0
  return new generate(this, cb, function * (next) {
    let browser = yield phantom.create({ path }, next);
    let page = yield browser.createPage(next);

    for (let op of Object.keys(opts.page)) {
      yield page.set(op, opts.page[op], next);
    }

    yield page.open(url, next);
    yield setTimeout(next, opts.timeout);

    result.dimensions = yield page.evaluate(captureDimensions, next);
    result.output = yield page.evaluate(opts.capture, next);

    page.render(result.file, opts);

    yield setTimeout(next, 1000);

    browser.exit();

    return cb(null, result);
  }, cb);
Esempio n. 27
0
Navit.prototype.__initPhantom__ = function (callback) {
  var self = this;
  var driverOptions = {
    parameters: {},
    path: require(this.__options__.engine).path
  };

  // Fill engine parameters
  _.forEach(this.__engineOptions__, function (optionValue, optionName) {
    driverOptions.parameters[_.kebabCase(optionName)] = optionValue;
  });

  driver.create(driverOptions, function (err, instance) {
    if (err) {
      callback(err);
      return;
    }

    self.__engine__ = instance;
    callback();
  });
};
Esempio n. 28
0
  ScreenShooter.prototype.init = function init( options, callback, done ) {
    this.options = options || {};
    this.done = done;

    if( !this.options.viewport ) {
      this.options.viewport = '1024x768';
    }

    if( !this.options.delay ) {
      this.options.delay = 300;
    }

    if( !this.options.closeDelay ) {
      // // Workaround for https://github.com/ariya/phantomjs/issues/11084
      this.options.closeDelay = 1000;
    }

    var createOptions = {};
    if( this.options.parameters ) {
      createOptions.parameters = this.options.parameters;
    }

    return phantom.create( this.getCreateCallback( callback ), createOptions );
  };
Esempio n. 29
0
Http.prototype.getPageContent = function(cb) {
    var self = this;
    var resourceWait = 300,
        maxRenderWait = 5000,
        //url = 'https://twitter.com/nodejs',
        //url = 'https://auto.ria.com/search/?category_id=0&marka_id=0&model_id=0&state=0#state[0]=0&s_yers[0]=0&po_yers[0]=0&currency=1&marka_id[0]=0&model_id[0]=0&countpage=10',
        url = self.link,
        count = 0,
        forcedRenderTimeout,
        renderTimeout;

    //var page = require('node-phantom-simple');
    driver.create({ path: require('phantomjs').path }, function(err, phantom) {

        return phantom.createPage(function(err, page) {



            //page.viewportSize = { width: 1280, height: 1024 };
            //page.property('viewportSize', { width: 1024, height: 1024 });

            page.set('viewportSize', { width: 1024, height: 9000 });
            page.set('settings.loadImages', 'false');


            function doRender() {
                //page.set('viewportSize', { width: 1024, height: 768 });
                page.render('capture.png');
                //page.render('twitter.png');
                console.log("RENDERING PAGE");
                setTimeout(function() {

                    page.get('content', function(err, html) {
                        console.log('COUNT ' + count);
                        if (count === 0) {
                            clearTimeout(renderTimeout);
                            phantom.exit();
                            cb(null, html);                             
                        }

                    });
                }, 2000);
            }

            page.onResourceRequested = function(req) {
                count += 1;
                clearTimeout(renderTimeout);
            };

            page.onResourceReceived = function(res) {
                if (!res.stage || res.stage === 'end') {
                    count -= 1;
                    if (count === 0) {
                        renderTimeout = setTimeout(doRender, resourceWait);
                    }
                }
            };

            page.open(url, function(err, status) {

                if (status !== "success") {
                    console.log('Unable to load url');
                    phantom.exit();
                    cb('Http Fail', null);
                } else {
                    forcedRenderTimeout = setTimeout(function() {
                        doRender();
                    }, maxRenderWait);

                }

            });
        });
    });
}
Esempio n. 30
0
function renderInvoicePdf($template, $template_vars, $_invoice, $_company, $token, $req, $res) {
    var $compiled_template = $template($template_vars);

    var filename = $_invoice.filename; //'invoice_' + $_invoice.invoice_number + ".pdf";
    //compile the template
    var folder = settings.fs.invoice_store;

    LOG.d(TAG, 'Setting location to :' + folder + filename);

    phantom.create(function (error, ph) {
        ph.createPage(function (error, page) {
            page.settings = PAGESETTINGS.general;
            page.set('viewportSize', PAGESETTINGS.viewport);
            page.set('paperSize', PAGESETTINGS.paper);
            page.set('content', $compiled_template, function (error) {
                if (error) {
                    LOG.e(TAG, 'Error setting content: ' + error);
                }
            });

            page.onLoadFinished = function (status) {

                page.render(folder + filename, function (error) {
                    if (error) {
                        LOG.e(TAG, 'Error rendering PDF: ' + error);
                    }
                    else {
                        LOG.d(TAG, 'Reading contents of file: ' + folder + filename);

                        fs.readFile(folder + filename, "base64", function (a_error, data) {
                            if ($_invoice.towing_voucher_id != null) {
                                LOG.d(TAG, 'It seems that a towing voucher id was set');
                                //this is a document which needs to be added to a towing voucher
                                db.one(SQL_ADD_ATTACHMENT_TO_VOUCHER, [$_invoice.towing_voucher_id, filename, "application/pdf", data.length, data, $token], function ($error, $att, $fields) {
                                    db.one(SQL_INVOICE_ATT_LINK_WITH_DOCUMENT, [$_invoice.id, $att.document_id, $token], function ($error, $result, fields) {
                                        fetchTowingVoucherReportAndMail($_invoice, data, $_company, $token, $req, $res);
                                    });
                                });
                            } else {
                                LOG.d(TAG, 'This is a freeform invoice');
                                //this is a freeform invoice
                                db.one(SQL_INVOICE_ADD_DOCUMENT, [$_invoice.id, filename, "application/pdf", data.length, data, $token], function ($error, $result, fields) {
                                    //fetchTowingVoucherReportAndMail($_invoice, data, $_company, $token, $req, $res);
                                });
                            }

                            // delete the file
                            fs.unlink(folder + filename, function (err) {
                                if (err) {
                                    LOG.e(TAG, "Could not delete file: " + JSON.stringify(err));
                                } else {
                                    LOG.d(TAG, 'successfully deleted ' + folder + filename);
                                }
                            });

                            // ph.exit();
                        });
                    }
                });
                //ph.exit();
            }
        }); //end ph.create
    });
}