componentDidMount() {
const dataCounts = this.props.job.data_counts;
if (dataCounts.processed_record_count > 0) {
// Get the list of all the forecasts with results at or later than the specified 'from' time.
mlForecastService.getForecastsSummary(
this.props.job,
null,
dataCounts.earliest_record_timestamp,
MAX_FORECASTS)
.then((resp) => {
this.setState({
isLoading: false,
forecasts: resp.forecasts
});
})
.catch((resp) => {
console.log('Error loading list of forecasts for jobs list:', resp);
this.setState({
isLoading: false,
errorMessage: this.props.intl.formatMessage({
id: 'xpack.ml.jobsList.jobDetails.forecastsTable.loadingErrorMessage',
defaultMessage: 'Error loading the list of forecasts run on this job'
}),
forecasts: []
});
});
}
}
openModal = () => {
const job = this.props.job;
if (typeof job === 'object') {
// Get the list of all the finished forecasts for this job with results at or later than the dashboard 'from' time.
const bounds = this.props.timefilter.getActiveBounds();
const statusFinishedQuery = {
term: {
forecast_status: FORECAST_REQUEST_STATE.FINISHED
}
};
mlForecastService.getForecastsSummary(
job,
statusFinishedQuery,
bounds.min.valueOf(),
FORECASTS_VIEW_MAX)
.then((resp) => {
this.setState({
previousForecasts: resp.forecasts
});
})
.catch((resp) => {
console.log('Time series forecast modal - error obtaining forecasts summary:', resp);
this.addMessage('Error obtaining list of previous forecasts', MESSAGE_LEVEL.ERROR);
});
// Display a warning about running a forecast if there is high number
// of partitioning fields.
const entityFieldNames = this.props.entities.map(entity => entity.fieldName);
if (entityFieldNames.length > 0) {
ml.getCardinalityOfFields({
index: job.datafeed_config.indices,
types: job.datafeed_config.types,
fieldNames: entityFieldNames,
query: job.datafeed_config.query,
timeFieldName: job.data_description.time_field,
earliestMs: job.data_counts.earliest_record_timestamp,
latestMs: job.data_counts.latest_record_timestamp
})
.then((results) => {
let numPartitions = 1;
Object.values(results).forEach((cardinality) => {
numPartitions = numPartitions * cardinality;
});
if (numPartitions > WARN_NUM_PARTITIONS) {
this.addMessage(
`Note that this data contains more than ${WARN_NUM_PARTITIONS} ` +
`partitions so running a forecast may take a long time and consume a high amount of resource`,
MESSAGE_LEVEL.WARNING);
}
})
.catch((resp) => {
console.log('Time series forecast modal - error obtaining cardinality of fields:', resp);
});
}
this.setState({ isModalVisible: true });
}
};
$scope.loadForForecastId = function (forecastId) {
mlForecastService.getForecastDateRange(
$scope.selectedJob,
forecastId
).then((resp) => {
const bounds = timefilter.getActiveBounds();
const earliest = moment(resp.earliest || timefilter.getTime().from);
const latest = moment(resp.latest || timefilter.getTime().to);
// Store forecast ID in the appState.
$scope.appState.mlTimeSeriesExplorer.forecastId = forecastId;
// Set the zoom to centre on the start of the forecast range, depending
// on the time range of the forecast and data.
const earliestDataDate = _.first($scope.contextChartData).date;
const zoomLatestMs = Math.min(earliest + ($scope.autoZoomDuration / 2), latest.valueOf());
const zoomEarliestMs = Math.max(zoomLatestMs - $scope.autoZoomDuration, earliestDataDate.getTime());
const zoomState = {
from: moment(zoomEarliestMs).toISOString(),
to: moment(zoomLatestMs).toISOString()
};
$scope.appState.mlTimeSeriesExplorer.zoom = zoomState;
$scope.appState.save();
// Ensure the forecast data will be shown if hidden previously.
$scope.showForecast = true;
if (earliest.isBefore(bounds.min) || latest.isAfter(bounds.max)) {
const earliestMs = Math.min(earliest.valueOf(), bounds.min.valueOf());
const latestMs = Math.max(latest.valueOf(), bounds.max.valueOf());
timefilter.setTime({
from: moment(earliestMs).toISOString(),
to: moment(latestMs).toISOString()
});
} else {
// Refresh to show the requested forecast data.
$scope.refresh();
}
}).catch((resp) => {
console.log('Time series explorer - error loading time range of forecast from elasticsearch:', resp);
});
};
runForecast = (closeJobAfterRunning) => {
this.setState({
forecastProgress: 0
});
// Always supply the duration to the endpoint in seconds as some of the moment duration
// formats accepted by Kibana (w, M, y) are not valid formats in Elasticsearch.
const durationInSeconds = parseInterval(this.state.newForecastDuration).asSeconds();
mlForecastService.runForecast(this.props.job.job_id, `${durationInSeconds}s`)
.then((resp) => {
// Endpoint will return { acknowledged:true, id: <now timestamp> } before forecast is complete.
// So wait for results and then refresh the dashboard to the end of the forecast.
if (resp.forecast_id !== undefined) {
this.waitForForecastResults(resp.forecast_id, closeJobAfterRunning);
} else {
this.runForecastErrorHandler(resp);
}
})
.catch(this.runForecastErrorHandler);
};
$scope.refreshFocusData = function (fromDate, toDate) {
// Counter to keep track of the queries to populate the chart.
let awaitingCount = 3;
// This object is used to store the results of individual remote requests
// before we transform it into the final data and apply it to $scope. Otherwise
// we might trigger multiple $digest cycles and depending on how deep $watches
// listen for changes we could miss updates.
const refreshFocusData = {};
// finish() function, called after each data set has been loaded and processed.
// The last one to call it will trigger the page render.
function finish() {
awaitingCount--;
if (awaitingCount === 0) {
// Tell the results container directives to render the focus chart.
refreshFocusData.focusChartData = processDataForFocusAnomalies(
refreshFocusData.focusChartData,
refreshFocusData.anomalyRecords,
$scope.timeFieldName);
refreshFocusData.focusChartData = processScheduledEventsForChart(
refreshFocusData.focusChartData,
refreshFocusData.scheduledEvents);
// All the data is ready now for a scope update.
// Use $evalAsync to ensure the update happens after the child scope is updated with the new data.
$scope.$evalAsync(() => {
$scope = Object.assign($scope, refreshFocusData);
console.log('Time series explorer focus chart data set:', $scope.focusChartData);
$scope.loading = false;
});
}
}
const detectorIndex = +$scope.detectorId;
const nonBlankEntities = _.filter($scope.entities, entity => entity.fieldValue.length > 0);
// Calculate the aggregation interval for the focus chart.
const bounds = { min: moment(fromDate), max: moment(toDate) };
$scope.focusAggregationInterval = calculateAggregationInterval(bounds, CHARTS_POINT_TARGET, CHARTS_POINT_TARGET);
// Ensure the search bounds align to the bucketing interval so that the first and last buckets are complete.
// For sum or count detectors, short buckets would hold smaller values, and model bounds would also be affected
// to some extent with all detector functions if not searching complete buckets.
const searchBounds = getBoundsRoundedToInterval(bounds, $scope.focusAggregationInterval, false);
// Query 1 - load metric data across selected time range.
mlTimeSeriesSearchService.getMetricData(
$scope.selectedJob,
detectorIndex,
nonBlankEntities,
searchBounds.min.valueOf(),
searchBounds.max.valueOf(),
$scope.focusAggregationInterval.expression
).then((resp) => {
refreshFocusData.focusChartData = processMetricPlotResults(resp.results, $scope.modelPlotEnabled);
$scope.showModelBoundsCheckbox = ($scope.modelPlotEnabled === true) &&
(refreshFocusData.focusChartData.length > 0);
finish();
}).catch((resp) => {
console.log('Time series explorer - error getting metric data from elasticsearch:', resp);
});
// Query 2 - load all the records across selected time range for the chart anomaly markers.
mlResultsService.getRecordsForCriteria(
[$scope.selectedJob.job_id],
$scope.criteriaFields,
0,
searchBounds.min.valueOf(),
searchBounds.max.valueOf(),
ANOMALIES_MAX_RESULTS
).then((resp) => {
// Sort in descending time order before storing in scope.
refreshFocusData.anomalyRecords = _.chain(resp.records)
.sortBy(record => record[$scope.timeFieldName])
.reverse()
.value();
console.log('Time series explorer anomalies:', refreshFocusData.anomalyRecords);
finish();
});
// Query 3 - load any scheduled events for the selected job.
mlResultsService.getScheduledEventsByBucket(
[$scope.selectedJob.job_id],
searchBounds.min.valueOf(),
searchBounds.max.valueOf(),
$scope.focusAggregationInterval.expression,
1,
MAX_SCHEDULED_EVENTS
).then((resp) => {
refreshFocusData.scheduledEvents = resp.events[$scope.selectedJob.job_id];
finish();
}).catch((resp) => {
console.log('Time series explorer - error getting scheduled events from elasticsearch:', resp);
});
// Plus query for forecast data if there is a forecastId stored in the appState.
const forecastId = _.get($scope, 'appState.mlTimeSeriesExplorer.forecastId');
if (forecastId !== undefined) {
awaitingCount++;
let aggType = undefined;
const detector = $scope.selectedJob.analysis_config.detectors[detectorIndex];
const esAgg = mlFunctionToESAggregation(detector.function);
if ($scope.modelPlotEnabled === false && (esAgg === 'sum' || esAgg === 'count')) {
aggType = { avg: 'sum', max: 'sum', min: 'sum' };
}
mlForecastService.getForecastData(
$scope.selectedJob,
detectorIndex,
forecastId,
nonBlankEntities,
searchBounds.min.valueOf(),
searchBounds.max.valueOf(),
$scope.focusAggregationInterval.expression,
aggType)
.then((resp) => {
refreshFocusData.focusForecastData = processForecastResults(resp.results);
refreshFocusData.showForecastCheckbox = (refreshFocusData.focusForecastData.length > 0);
finish();
}).catch((resp) => {
console.log(`Time series explorer - error loading data for forecast ID ${forecastId}`, resp);
});
}
// Load the data for the anomalies table.
loadAnomaliesTableData(searchBounds.min.valueOf(), searchBounds.max.valueOf());
};
$scope.refresh = function () {
if ($scope.selectedJob === undefined) {
return;
}
$scope.loading = true;
$scope.hasResults = false;
delete $scope.chartDetails;
delete $scope.contextChartData;
delete $scope.focusChartData;
delete $scope.contextForecastData;
delete $scope.focusForecastData;
// Counter to keep track of what data sets have been loaded.
$scope.loadCounter++;
let awaitingCount = 3;
// finish() function, called after each data set has been loaded and processed.
// The last one to call it will trigger the page render.
function finish(counterVar) {
awaitingCount--;
if (awaitingCount === 0 && (counterVar === $scope.loadCounter)) {
if (($scope.contextChartData && $scope.contextChartData.length) ||
($scope.contextForecastData && $scope.contextForecastData.length)) {
$scope.hasResults = true;
} else {
$scope.hasResults = false;
}
$scope.loading = false;
// Set zoomFrom/zoomTo attributes in scope which will result in the metric chart automatically
// selecting the specified range in the context chart, and so loading that date range in the focus chart.
if ($scope.contextChartData.length) {
const focusRange = calculateInitialFocusRange();
$scope.zoomFrom = focusRange[0];
$scope.zoomTo = focusRange[1];
}
// Tell the results container directives to render.
// Need to use $timeout to ensure the broadcast happens after the child scope is updated with the new data.
if (($scope.contextChartData && $scope.contextChartData.length) ||
($scope.contextForecastData && $scope.contextForecastData.length)) {
$timeout(() => {
$scope.$broadcast('render');
}, 0);
}
}
}
const bounds = timefilter.getActiveBounds();
const detectorIndex = +$scope.detectorId;
$scope.modelPlotEnabled = isModelPlotEnabled($scope.selectedJob, detectorIndex, $scope.entities);
// Only filter on the entity if the field has a value.
const nonBlankEntities = _.filter($scope.entities, (entity) => { return entity.fieldValue.length > 0; });
$scope.criteriaFields = [{
'fieldName': 'detector_index',
'fieldValue': detectorIndex }
].concat(nonBlankEntities);
// Calculate the aggregation interval for the context chart.
// Context chart swimlane will display bucket anomaly score at the same interval.
$scope.contextAggregationInterval = calculateAggregationInterval(bounds, CHARTS_POINT_TARGET, CHARTS_POINT_TARGET);
console.log('aggregationInterval for context data (s):', $scope.contextAggregationInterval.asSeconds());
// Ensure the search bounds align to the bucketing interval so that the first and last buckets are complete.
// For sum or count detectors, short buckets would hold smaller values, and model bounds would also be affected
// to some extent with all detector functions if not searching complete buckets.
const searchBounds = getBoundsRoundedToInterval(bounds, $scope.contextAggregationInterval, false);
// Query 1 - load metric data at low granularity across full time range.
// Pass a counter flag into the finish() function to make sure we only process the results
// for the most recent call to the load the data in cases where the job selection and time filter
// have been altered in quick succession (such as from the job picker with 'Apply time range').
const counter = $scope.loadCounter;
mlTimeSeriesSearchService.getMetricData(
$scope.selectedJob,
detectorIndex,
nonBlankEntities,
searchBounds.min.valueOf(),
searchBounds.max.valueOf(),
$scope.contextAggregationInterval.expression
).then((resp) => {
const fullRangeChartData = processMetricPlotResults(resp.results, $scope.modelPlotEnabled);
$scope.contextChartData = fullRangeChartData;
console.log('Time series explorer context chart data set:', $scope.contextChartData);
finish(counter);
}).catch((resp) => {
console.log('Time series explorer - error getting metric data from elasticsearch:', resp);
});
// Query 2 - load max record score at same granularity as context chart
// across full time range for use in the swimlane.
mlResultsService.getRecordMaxScoreByTime(
$scope.selectedJob.job_id,
$scope.criteriaFields,
searchBounds.min.valueOf(),
searchBounds.max.valueOf(),
$scope.contextAggregationInterval.expression
).then((resp) => {
const fullRangeRecordScoreData = processRecordScoreResults(resp.results);
$scope.swimlaneData = fullRangeRecordScoreData;
console.log('Time series explorer swimlane anomalies data set:', $scope.swimlaneData);
finish(counter);
}).catch((resp) => {
console.log('Time series explorer - error getting bucket anomaly scores from elasticsearch:', resp);
});
// Query 3 - load details on the chart used in the chart title (charting function and entity(s)).
mlTimeSeriesSearchService.getChartDetails(
$scope.selectedJob,
detectorIndex,
$scope.entities,
searchBounds.min.valueOf(),
searchBounds.max.valueOf()
).then((resp) => {
$scope.chartDetails = resp.results;
finish(counter);
}).catch((resp) => {
console.log('Time series explorer - error getting entity counts from elasticsearch:', resp);
});
// Plus query for forecast data if there is a forecastId stored in the appState.
const forecastId = _.get($scope, 'appState.mlTimeSeriesExplorer.forecastId');
if (forecastId !== undefined) {
awaitingCount++;
let aggType = undefined;
const detector = $scope.selectedJob.analysis_config.detectors[detectorIndex];
const esAgg = mlFunctionToESAggregation(detector.function);
if ($scope.modelPlotEnabled === false && (esAgg === 'sum' || esAgg === 'count')) {
aggType = { avg: 'sum', max: 'sum', min: 'sum' };
}
mlForecastService.getForecastData(
$scope.selectedJob,
detectorIndex,
forecastId,
nonBlankEntities,
searchBounds.min.valueOf(),
searchBounds.max.valueOf(),
$scope.contextAggregationInterval.expression,
aggType)
.then((resp) => {
$scope.contextForecastData = processForecastResults(resp.results);
finish(counter);
}).catch((resp) => {
console.log(`Time series explorer - error loading data for forecast ID ${forecastId}`, resp);
});
}
loadEntityValues();
};
this.forecastChecker = setInterval(() => {
mlForecastService.getForecastRequestStats(this.props.job, forecastId)
.then((resp) => {
// Get the progress (stats value is between 0 and 1).
const progress = _.get(resp, ['stats', 'forecast_progress'], previousProgress);
const status = _.get(resp, ['stats', 'forecast_status']);
this.setState({ forecastProgress: Math.round(100 * progress) });
// Display any messages returned in the request stats.
let messages = _.get(resp, ['stats', 'forecast_messages'], []);
messages = messages.map((message) => ({ message, status: MESSAGE_LEVEL.WARNING }));
this.setState({ messages });
if (status === FORECAST_REQUEST_STATE.FINISHED) {
clearInterval(this.forecastChecker);
if (closeJobAfterRunning === true) {
this.setState({ jobClosingState: PROGRESS_STATES.WAITING });
mlJobService.closeJob(this.props.job.job_id)
.then(() => {
this.setState({
jobClosingState: PROGRESS_STATES.DONE
});
this.props.loadForForecastId(forecastId);
this.closeAfterRunningForecast();
})
.catch((response) => {
// Load the forecast data in the main page,
// but leave this dialog open so the error can be viewed.
console.log('Time series forecast modal - could not close job:', response);
this.addMessage(
intl.formatMessage({
id: 'xpack.ml.timeSeriesExplorer.forecastingModal.errorWithClosingJobAfterRunningForecastErrorMessage',
defaultMessage: 'Error closing job after running forecast',
}),
MESSAGE_LEVEL.ERROR
);
this.setState({
jobClosingState: PROGRESS_STATES.ERROR
});
this.props.loadForForecastId(forecastId);
});
} else {
this.props.loadForForecastId(forecastId);
this.closeAfterRunningForecast();
}
} else {
// Display a warning and abort check if the forecast hasn't
// progressed for WARN_NO_PROGRESS_MS.
if (progress === previousProgress) {
noProgressMs += FORECAST_STATS_POLL_FREQUENCY;
if (noProgressMs > WARN_NO_PROGRESS_MS) {
console.log(`Forecast request has not progressed for ${WARN_NO_PROGRESS_MS}ms. Cancelling check.`);
this.addMessage(
intl.formatMessage({
id: 'xpack.ml.timeSeriesExplorer.forecastingModal.noProgressReportedForNewForecastErrorMessage',
defaultMessage: 'No progress reported for the new forecast for {WarnNoProgressMs}ms.' +
'An error may have occurred whilst running the forecast.'
}, { WarnNoProgressMs: WARN_NO_PROGRESS_MS }),
MESSAGE_LEVEL.ERROR
);
// Try and load any results which may have been created.
this.props.loadForForecastId(forecastId);
this.setState({ forecastProgress: PROGRESS_STATES.ERROR });
clearInterval(this.forecastChecker);
}
} else {
previousProgress = progress;
}
}
}).catch((resp) => {
console.log('Time series forecast modal - error loading stats of forecast from elasticsearch:', resp);
this.addMessage(
intl.formatMessage({
id: 'xpack.ml.timeSeriesExplorer.forecastingModal.errorWithLoadingStatsOfRunningForecastErrorMessage',
defaultMessage: 'Error loading stats of running forecast.',
}),
MESSAGE_LEVEL.ERROR
);
this.setState({
forecastProgress: PROGRESS_STATES.ERROR
});
clearInterval(this.forecastChecker);
});
}, FORECAST_STATS_POLL_FREQUENCY);