Client.prototype.finishRun_ = function(job, isRunFinished) {
'use strict';
logger.alert('Finished run %s/%s (isRunFinished=%s) of job %s',
job.runNumber, job.runs, isRunFinished, job.id);
// Expected finish of the current job
if (this.currentJob_ === job) {
global.clearTimeout(this.timeoutTimer_);
this.timeoutTimer_ = undefined;
this.currentJob_ = undefined;
this.submitResult_(job, isRunFinished, function(e) {
this.handlingUncaughtException_ = undefined;
if (e) {
logger.error('Unable to submit result: %s', e.stack);
}
// Run until we finish the last iteration.
// Do not increment job.runNumber past job.runs.
if (e || (isRunFinished && job.runNumber === job.runs)) {
this.emit('done', job);
} else {
// Continue running
if (isRunFinished) {
job.runNumber += 1;
if (job.runNumber > job.runs) { // Sanity check.
throw new Error('Internal error: job.runNumber > job.runs');
}
}
this.startNextRun_(job);
}
}.bind(this));
} else { // Belated finish of an old already timed-out job
logger.error('Timed-out job finished, but too late: %s', job.id);
this.handlingUncaughtException_ = undefined;
}
};
Client.prototype.onSignal_ = function(signal_name) {
'use strict';
// Set our signal to the max(new_signal, old_signal)
var old_signal = this.handlingSignal_;
var new_signal = SIGNAL_NAMES[Math.max(
SIGNAL_NAMES.indexOf(signal_name), SIGNAL_NAMES.indexOf(old_signal))];
if (this.noJobTimer_) {
// Exit now. We check the noJobTimer_ instead of !currentJob_ because
// (a) noJobTimer_ implies !currentJob_ and, more importantly,
// (b) we don't want to exit in the middle of requesting a new job.
logger.alert('Received %s, exiting.', signal_name);
exports.process.exit();
} else if (SIGPIPE === new_signal) {
logger.alert('Received %s, ignoring because we have a job.', signal_name);
} else {
// Exit later, when we're 'done' or get a 'nojob' event.
this.handlingSignal_ = new_signal;
if (!old_signal) {
this.removeAllListeners();
['done', 'nojob'].forEach(function(event_name) {
this.on(event_name, function() {
logger.alert('Exiting due to %s.', this.handlingSignal_);
exports.process.exit();
}.bind(this));
}.bind(this));
}
logger.alert('Received %s, will exit after the current %s.', signal_name,
(SIGQUIT === new_signal ? 'job finishes' :
SIGABRT === new_signal ? 'run finishes' :
SIGTERM === new_signal ? 'run aborts' : 'run is killed'));
var job = this.currentJob_;
if (job &&
(SIGTERM === new_signal || SIGINT === new_signal) &&
(SIGTERM !== old_signal && SIGINT !== old_signal)) {
job.agentError = this.handlingSignal_;
this.abortJob_(job);
}
}
};
}.bind(this)).addBoth(function(errOrBool) {
var wasOffline = (errOrBool !== false);
if (wasOffline) {
job.agentError = job.agentError || 'Agent was offline';
}
var isOffline = (errOrBool instanceof Error);
if (isOffline) {
logger.error('Agent is offline: ' + errOrBool.message);
job.agentError = job.agentError || errOrBool.message;
}
var isAbort = (
(SIGTERM === this.handlingSignal_ ||
SIGINT === this.handlingSignal_) || // Abort run
(SIGABRT === this.handlingSignal_ && isRunFinished)); // Abort job
if (isAbort) {
job.agentError = this.handlingSignal_;
}
// Retry on agentError, at most once per run, but only if we're online.
//
// There are many other definitions that we could use instead, e.g.
// retry on any job.testError, retry up to N times per job, etc.
var shouldRetry = (
!!job.agentError && !isOffline && !isAbort && !job.retryError);
var isJobFinished = (!shouldRetry && (
isAbort || wasOffline || isOffline ||
(job.runNumber === job.runs && isRunFinished) ||
// Failed WPR record-run terminates the whole job.
(job.runNumber === 0 && job.testError)));
logger.alert('%s run %d%s%s/%d of %sjob %s%s%s%s',
((job.testError || job.agentError) ? 'Failed' : 'Finished'),
job.runNumber,
(job.isFirstViewOnly ? '' : (job.isCacheWarm ? 'b' : 'a')),
(job.retryError ? '\'' : ''),
job.runs, (isJobFinished ? 'finished ' : ''),
job.id, (job.testError || job.agentError ? ': ' : ''),
(job.testError || ''), (job.testError && job.agentError ? ' ' : ''),
(job.agentError ? '(' + job.agentError + ')' : ''));
if (shouldRetry) {
job.retryError = job.agentError || 'Unknown';
job.isCacheWarm = false;
this.startNextRun_(job);
return;
}
this.submitResult_(job, isJobFinished,
this.endOfRun_.bind(this, job, isRunFinished, isJobFinished));
}.bind(this));
Client.prototype.finishRun_ = function(job, isRunFinished) {
'use strict';
logger.alert('Finished run %s/%s (isRunFinished=%s) of job %s',
job.runNumber, job.runs, isRunFinished, job.id);
// Expected finish of the current job
if (this.currentJob_ === job) {
global.clearTimeout(this.timeoutTimer_);
this.timeoutTimer_ = undefined;
this.currentJob_ = undefined;
if (0 === job.runNumber) { // Do not submit a WebPageReplay recording run.
this.endOfRun_(job, isRunFinished, /*e=*/undefined);
} else {
this.submitResult_(job, isRunFinished,
this.endOfRun_.bind(this, job, isRunFinished));
}
} else { // Belated finish of an old already timed-out job
logger.error('Timed-out job finished, but too late: %s', job.id);
this.handlingUncaughtException_ = undefined;
}
};
Client.prototype.finishRun_ = function(job, isRunFinished) {
'use strict';
if (job !== this.currentJob_) {
// Unexpected job finish: not the current job
logger.error('Ignoring old job %s != current job %s', job.id,
(this.currentJob_ ? this.currentJob_.id : 'None'));
this.handlingUncaughtException_ = undefined;
return;
}
global.clearTimeout(this.timeoutTimer_);
this.timeoutTimer_ = undefined;
this.currentJob_ = undefined;
if (this.onAlive)
this.onAlive();
var isAbort = (
(SIGTERM === this.handlingSignal_ ||
SIGINT === this.handlingSignal_) || // Abort run
(SIGABRT === this.handlingSignal_ && isRunFinished)); // Abort job
if (isAbort) {
job.agentError = this.handlingSignal_;
}
var isJobFinished = (job.runNumber === job.runs && isRunFinished) ||
// Failed WPR record-run terminates the whole job.
(job.runNumber === 0 && job.testError);
logger.alert('%s run %d%s%s/%d of %sjob %s%s%s%s',
((job.testError || job.agentError) ? 'Failed' : 'Finished'),
job.runNumber,
(job.isFirstViewOnly ? '' : (job.isCacheWarm ? 'b' : 'a')),
(job.retryError ? '\'' : ''),
job.runs, (isJobFinished ? 'finished ' : ''),
job.id, (job.testError || job.agentError ? ': ' : ''),
(job.testError || ''), (job.testError && job.agentError ? ' ' : ''),
(job.agentError ? '(' + job.agentError + ')' : ''));
this.submitResult_(job, isJobFinished,
this.endOfRun_.bind(this, job, isRunFinished, isJobFinished));
};
this.on(event_name, function() {
logger.alert('Exiting due to %s.', this.handlingSignal_);
exports.process.exit();
}.bind(this));