test({ viewport, browserTimezone }, logger) {
const userDataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'chromium-'));
const chromiumArgs = args({
userDataDir,
viewport,
verboseLogging: true,
disableSandbox: this.browserConfig.disableSandbox,
proxyConfig: this.browserConfig.proxy,
});
return puppeteer
.launch({
userDataDir,
executablePath: this.binaryPath,
ignoreHTTPSErrors: true,
args: chromiumArgs,
env: {
TZ: browserTimezone,
},
})
.catch(error => {
logger.warning(
`The Reporting plugin encountered issues launching Chromium in a self-test. You may have trouble generating reports: [${error}]`
);
logger.warning(`See Chromium's log output at "${getChromeLogLocation(this.binaryPath)}"`);
return null;
});
}
(async () => {
const browser = await puppeteer.launch({
executablePath: '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'
});
const page = await browser.newPage();
await page.goto('https://so.com');
await page.screenshot({path: 'example.png'});
await browser.close();
})();
module.exports = async function() {
const args = [
'--no-sandbox',
'--ignore-certificate-errors'
];
const options = {
executablePath: '/usr/bin/google-chrome',
args,
ignoreHTTPSErrors: true
};
const browser = await puppeteer.launch(options);
// store the browser instance so we can teardown it later
// this global is only available in the teardown but not in TestEnvironments
global.__BROWSER_GLOBAL__ = browser;
// use the file system to expose the wsEndpoint for TestEnvironments
mkdirp.sync(DIR);
fs.writeFileSync(path.join(DIR, 'wsEndpoint'), browser.wsEndpoint());
};
(async () => {
async function onConsole(msg) {
let args = msg.args();
let value = await args[0].jsonValue();
let passed = value.passedExpectations || [];
let failed = value.failedExpectations || [];
for (let i=0;i<passed.length;i++) { process.stdout.write("."); }
failed.length && console.log(failed);
let status = value["overallStatus"];
if (!status) { return; }
console.log(status);
process.exitCode = (status == "passed" ? 0 : 1);
browser.close();
}
const browser = await puppeteer.launch({executablePath: "/usr/bin/google-chrome"});
const page = await browser.newPage();
let url = `file://${path.dirname(module.filename)}/index.html`;
page.on("console", onConsole);
page.goto(url);
})();
return Rx.Observable.create(async observer => {
const userDataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'chromium-'));
const chromiumArgs = args({
userDataDir,
viewport,
verboseLogging: this.logger.isVerbose,
disableSandbox: this.browserConfig.disableSandbox,
proxyConfig: this.browserConfig.proxy,
});
let browser;
let page;
try {
browser = await puppeteer.launch({
userDataDir,
executablePath: this.binaryPath,
ignoreHTTPSErrors: true,
args: chromiumArgs,
env: {
TZ: browserTimezone,
},
});
page = await browser.newPage();
// All navigation/waitFor methods default to 30 seconds,
// which can cause the job to fail even if we bump timeouts in
// the config. Help alleviate errors like
// "TimeoutError: waiting for selector ".application" failed: timeout 30000ms exceeded"
page.setDefaultTimeout(this.queueTimeout);
} catch (err) {
observer.error(new Error(`Error spawning Chromium browser: [${err}]`));
throw err;
}
safeChildProcess(
{
async kill() {
await browser.close();
},
},
observer
);
// Register with a few useful puppeteer event handlers:
// https://pptr.dev/#?product=Puppeteer&version=v1.10.0&show=api-event-error
// https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page
const stderr$ = Rx.fromEvent(page, 'console').pipe(
filter(line => line._type === 'error'),
map(line => line._text),
share()
);
const [consoleMessage$, message$] = stderr$.pipe(
partition(msg => msg.match(/\[\d+\/\d+.\d+:\w+:CONSOLE\(\d+\)\]/))
);
const driver$ = Rx.of(
new HeadlessChromiumDriver(page, {
maxScreenshotDimension: this.browserConfig.maxScreenshotDimension,
logger: this.logger,
inspect: this.browserConfig.inspect,
})
);
const processError$ = Rx.fromEvent(page, 'error').pipe(
mergeMap(err => Rx.throwError(new Error(`Unable to spawn Chromium: [${err}]`)))
);
const processPageError$ = Rx.fromEvent(page, 'pageerror').pipe(
mergeMap(err => Rx.throwError(new Error(`Uncaught exception within the page: [${err}]`)))
);
const processRequestFailed$ = Rx.fromEvent(page, 'requestfailed').pipe(
mergeMap(req => {
const failure = req.failure && req.failure();
if (failure) {
return Rx.throwError(
new Error(`Request to [${req.url()}] failed! [${failure.errorText}]`)
);
}
return Rx.throwError(new Error(`Unknown failure! [${JSON.stringify(req)}]`));
})
);
const processExit$ = Rx.fromEvent(browser, 'disconnected').pipe(
mergeMap(code =>
Rx.throwError(new Error(`Chromium exited with: [${JSON.stringify({ code })}]`))
)
);
const nssError$ = message$.pipe(
filter(line => line.includes('error while loading shared libraries: libnss3.so')),
mergeMap(() => Rx.throwError(new Error(`You must install nss for Reporting to work`)))
);
const fontError$ = message$.pipe(
filter(line =>
line.includes('Check failed: InitDefaultFont(). Could not find the default font')
),
mergeMap(() =>
Rx.throwError(new Error('You must install freetype and ttf-font for Reporting to work'))
)
);
const noUsableSandbox$ = message$.pipe(
filter(line => line.includes('No usable sandbox! Update your kernel')),
mergeMap(() =>
Rx.throwError(
new Error(
compactWhitespace(`
Unable to use Chromium sandbox. This can be disabled at your own risk with
'xpack.reporting.capture.browser.chromium.disableSandbox'
`)
)
)
)
);
const exit$ = Rx.merge(
processError$,
processPageError$,
processRequestFailed$,
processExit$,
nssError$,
fontError$,
noUsableSandbox$
);
observer.next({
driver$,
consoleMessage$,
message$,
exit$,
});
// unsubscribe logic makes a best-effort attempt to delete the user data directory used by chromium
return () => {
this.logger.debug(`deleting chromium user data directory at [${userDataDir}]`);
// the unsubscribe function isn't `async` so we're going to make our best effort at
// deleting the userDataDir and if it fails log an error.
rimraf(userDataDir, err => {
if (err) {
return this.logger.error(
`error deleting user data directory at [${userDataDir}]: [${err}]`
);
}
});
};
});
return Rx.Observable.create(async observer => {
const userDataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'chromium-'));
const chromiumArgs = args({
userDataDir,
viewport,
verboseLogging: this.logger.isVerbose,
disableSandbox: this.browserConfig.disableSandbox,
proxyConfig: this.browserConfig.proxy,
});
let chromium;
let page;
try {
chromium = await puppeteer.launch({
userDataDir,
executablePath: this.binaryPath,
ignoreHTTPSErrors: true,
args: chromiumArgs,
});
page = await chromium.newPage();
} catch (err) {
observer.error(new Error(`Caught error spawning Chromium`));
return;
}
safeChildProcess({
async kill() {
await chromium.close();
}
}, observer);
const stderr$ = Rx.fromEvent(page, 'console').pipe(
filter(line => line._type === 'error'),
map(line => line._text),
share()
);
const [consoleMessage$, message$] = stderr$.pipe(
partition(msg => msg.match(/\[\d+\/\d+.\d+:\w+:CONSOLE\(\d+\)\]/))
);
const driver$ = Rx.of(new HeadlessChromiumDriver(page, {
maxScreenshotDimension: this.browserConfig.maxScreenshotDimension,
logger: this.logger
}));
const processError$ = Rx.fromEvent(page, 'error').pipe(
map((err) => this.logger.error(err)),
mergeMap(() => Rx.throwError(new Error(`Unable to spawn Chromium`))),
);
const processExit$ = Rx.fromEvent(chromium, 'disconnected').pipe(
mergeMap((err) => Rx.throwError(new Error(`Chromium exited with code: ${err}. ${JSON.stringify(err)}`)))
);
const nssError$ = message$.pipe(
filter(line => line.includes('error while loading shared libraries: libnss3.so')),
mergeMap(() => Rx.throwError(new Error(`You must install nss for Reporting to work`)))
);
const fontError$ = message$.pipe(
filter(line => line.includes('Check failed: InitDefaultFont(). Could not find the default font')),
mergeMap(() => Rx.throwError(new Error('You must install freetype and ttf-font for Reporting to work')))
);
const noUsableSandbox$ = message$.pipe(
filter(line => line.includes('No usable sandbox! Update your kernel')),
mergeMap(() => Rx.throwError(new Error(compactWhitespace(`
Unable to use Chromium sandbox. This can be disabled at your own risk with
'xpack.reporting.capture.browser.chromium.disableSandbox'
`))))
);
const exit$ = Rx.merge(processError$, processExit$, nssError$, fontError$, noUsableSandbox$);
observer.next({
driver$,
consoleMessage$,
message$,
exit$
});
// unsubscribe logic makes a best-effort attempt to delete the user data directory used by chromium
return () => {
this.logger.debug(`deleting chromium user data directory at ${userDataDir}`);
// the unsubscribe function isn't `async` so we're going to make our best effort at
// deleting the userDataDir and if it fails log an error.
rimraf(userDataDir, (err) => {
if (err) {
return this.logger.error(`error deleting user data directory at ${userDataDir}: ${err}`);
}
});
};
});
exports.getPageData = async options => {
const browserOptions = {};
if (options.browserHeadless !== undefined) {
browserOptions.headless = options.browserHeadless && !options.browserDebug;
}
browserOptions.args = [];
if (options.browserDisableWebSecurity === undefined || options.browserDisableWebSecurity) {
browserOptions.args.push("--disable-web-security");
}
browserOptions.args.push("--no-pings");
if (!options.browserHeadless && options.browserDebug) {
browserOptions.args.push("--auto-open-devtools-for-tabs");
}
if (options.browserWidth && options.browserHeight) {
browserOptions.args.push("--window-size=" + options.browserWidth + "," + options.browserHeight);
}
if (options.browserExecutablePath) {
browserOptions.executablePath = options.browserExecutablePath || "chrome";
}
let browser;
try {
browser = await puppeteer.launch(browserOptions);
const page = await browser.newPage();
if (options.userAgent) {
await page.setUserAgent(options.userAgent);
}
if (options.browserWidth && options.browserHeight) {
await page.setViewport({
width: options.browserWidth,
height: options.browserHeight
});
}
if (options.browserBypassCSP === undefined || options.browserBypassCSP) {
await page.setBypassCSP(true);
}
let scripts = SCRIPTS.concat(options.browserScripts).map(scriptPath => fs.readFileSync(require.resolve(scriptPath)).toString()).join("\n");
const fileContents = {
"/lib/hooks/content/content-hooks-web.js": fs.readFileSync(require.resolve("../../lib/hooks/content/content-hooks-web.js")).toString(),
"/lib/hooks/content/content-hooks-frames-web.js": fs.readFileSync(require.resolve("../../lib/hooks/content/content-hooks-frames-web.js")).toString(),
};
scripts = scripts + ";this.singlefile.lib.getFileContent = filename => (" + JSON.stringify(fileContents) + ")[filename];";
await page.evaluateOnNewDocument(scripts);
if (options.browserDebug) {
await page.waitFor(3000);
}
await page.goto(options.url, {
timeout: 0,
waitUntil: options.browserWaitUntil || "networkidle0"
});
return await page.evaluate(async options => {
singlefile.lib.helper.initDoc(document);
options.insertSingleFileComment = true;
options.insertFaviconLink = true;
const preInitializationPromises = [];
if (!options.saveRawPage) {
if (!options.removeFrames) {
preInitializationPromises.push(singlefile.lib.frameTree.content.frames.getAsync(options));
}
if (options.loadDeferredImages) {
preInitializationPromises.push(singlefile.lib.lazy.content.loader.process(options));
}
}
[options.frames] = await Promise.all(preInitializationPromises);
options.doc = document;
options.win = window;
const SingleFile = singlefile.lib.SingleFile.getClass();
const singleFile = new SingleFile(options);
await singleFile.run();
return await singleFile.getPageData();
}, options);
} finally {
if (browser && !options.browserDebug) {
await browser.close();
}
}
};
async function render(_opts = {}) {
const opts = _.merge({
cookies: [],
scrollPage: false,
emulateScreenMedia: true,
ignoreHttpsErrors: false,
html: null,
viewport: {
width: 1600,
height: 1200,
},
goto: {
waitUntil: 'networkidle2',
timeout: 5000,
},
pdf: {
format: 'A4',
printBackground: true,
},
}, _opts);
if (_.get(_opts, 'pdf.width') && _.get(_opts, 'pdf.height')) {
// pdf.format always overrides width and height, so we must delete it
// when user explicitly wants to set width and height
opts.pdf.format = undefined;
}
logOpts(opts);
const puppeterOptions = {
headless: !config.DEBUG_MODE,
ignoreHTTPSErrors: opts.ignoreHttpsErrors,
args: ['--no-sandbox', '--headless', '--disable-gpu', '--disable-software-rasterizer', '--disable-dev-shm-usage'],
sloMo: config.DEBUG_MODE ? 250 : undefined,
};
if (typeof process.env.CHROME_PATH !== 'undefined' && process.env.CHROME_PATH) {
puppeterOptions.executablePath = process.env.CHROME_PATH;
}
const browser = await puppeteer.launch(puppeterOptions);
const page = await browser.newPage();
page.on('console', (...args) => logger.info('PAGE LOG:', ...args));
page.on('error', (err) => {
logger.error(`Error event emitted: ${err}`);
logger.error(err.stack);
browser.close();
});
var request_id = uuid();
let data;
try {
logger.info('Set browser viewport..');
await page.setViewport(opts.viewport);
if (opts.emulateScreenMedia) {
logger.info('Emulate @media screen..');
await page.emulateMedia('screen');
}
logger.info('Setting cookies..');
opts.cookies.map(async (cookie) => {
await page.setCookie(cookie);
});
if (opts.html) {
logger.info('Set HTML ..');
// https://github.com/GoogleChrome/puppeteer/issues/728
fs.appendFileSync(config.TMP_PATH + `/${request_id}.html`, opts.html)
await page.goto(`http://127.0.0.1:${config.PORT}/tmp/${request_id}.html`, opts.goto);
} else {
logger.info(`Goto url ${opts.url} ..`);
await page.goto(opts.url, opts.goto);
}
if (_.isNumber(opts.waitFor) || _.isString(opts.waitFor)) {
logger.info(`Wait for ${opts.waitFor} ..`);
await page.waitFor(opts.waitFor);
}
if (opts.scrollPage) {
logger.info('Scroll page ..');
await scrollPage(page);
}
logger.info('Render PDF ..');
if (config.DEBUG_MODE) {
const msg = `\n\n---------------------------------\n
Chrome does not support PDF rendering in "headed" mode.
See this issue: https://github.com/GoogleChrome/puppeteer/issues/576
\n---------------------------------\n\n
`;
throw new Error(msg);
}
data = await page.pdf(opts.pdf);
} catch (err) {
logger.error(`Error when rendering page: ${err}`);
logger.error(err.stack);
throw err;
} finally {
logger.info('Closing browser..');
if (!config.DEBUG_MODE) {
await browser.close();
await fs.unlink(config.TMP_PATH + `/${request_id}.html`, (error) => { });
}
}
return data;
}