componentDidMount() {
// make the interval 2 times the bucket span
if (this.state.bucketSpan) {
const intervalObject = parseInterval(this.state.bucketSpan);
let bs = intervalObject.asMinutes() * 2;
if (bs < 1) {
bs = 1;
}
const interval = `${bs}m`;
this.setState({ interval }, () => {
this.config.interval = interval;
});
}
// load elasticsearch settings to see if email has been configured
ml.getNotificationSettings().then((resp) => {
if (has(resp, 'defaults.xpack.notification.email')) {
this.setState({ emailEnabled: true });
}
});
mlCreateWatchService.loadWatch(this.state.jobId)
.then(() => {
this.setState({ watchAlreadyExists: true });
})
.catch(() => {
this.setState({ watchAlreadyExists: false });
});
}
.then(data => {
// for count, scale the model upper and lower by the
// ratio of chart interval to bucketspan.
// this will force the model bounds to be drawn in the correct location
let scale = 1;
if (formConfig &&
(formConfig.agg.type.mlName === 'count' ||
formConfig.agg.type.mlName === 'high_count' ||
formConfig.agg.type.mlName === 'low_count' ||
formConfig.agg.type.mlName === 'distinct_count')) {
const chartIntervalSeconds = formConfig.chartInterval.getInterval().asSeconds();
const bucketSpan = parseInterval(formConfig.bucketSpan);
if (bucketSpan !== null) {
scale = chartIntervalSeconds / bucketSpan.asSeconds();
}
}
this.chartData.model = this.chartData.model.concat(processLineChartResults(data.results, scale));
const lastBucket = this.chartData.model[this.chartData.model.length - 1];
const time = (lastBucket !== undefined) ? lastBucket.time : formConfig.start;
const pcnt = ((time - formConfig.start + formConfig.resultsIntervalSeconds) / (formConfig.end - formConfig.start) * 100);
this.chartData.percentComplete = Math.round(pcnt);
resolve(this.chartData);
})
return singleTimeSeriesJobs.map(job => {
const bucketSpan = parseInterval(job.analysis_config.bucket_span);
return {
id: job.job_id,
selected: false,
bucketSpanSeconds: bucketSpan.asSeconds()
};
});
cloneJob(job) {
// create a deep copy of a job object
// also remove items from the job which are set by the server and not needed
// in the future this formatting could be optional
const tempJob = angular.copy(job);
// remove all of the items which should not be copied
// such as counts, state and times
delete tempJob.state;
delete tempJob.job_version;
delete tempJob.data_counts;
delete tempJob.create_time;
delete tempJob.finished_time;
delete tempJob.last_data_time;
delete tempJob.model_size_stats;
delete tempJob.node;
delete tempJob.average_bucket_processing_time_ms;
delete tempJob.model_snapshot_id;
delete tempJob.open_time;
delete tempJob.established_model_memory;
delete tempJob.calendars;
delete tempJob.analysis_config.use_per_partition_normalization;
_.each(tempJob.analysis_config.detectors, (d) => {
delete d.detector_index;
});
// remove parts of the datafeed config which should not be copied
if (tempJob.datafeed_config) {
delete tempJob.datafeed_config.datafeed_id;
delete tempJob.datafeed_config.job_id;
delete tempJob.datafeed_config.state;
delete tempJob.datafeed_config.node;
// remove query_delay if it's between 60s and 120s
// the back-end produces a random value between 60 and 120 and so
// by deleting it, the back-end will produce a new random value
if (tempJob.datafeed_config.query_delay) {
const interval = parseInterval(tempJob.datafeed_config.query_delay);
if (interval !== null) {
const queryDelay = interval.asSeconds();
if (queryDelay > 60 && queryDelay < 120) {
delete tempJob.datafeed_config.query_delay;
}
}
}
}
// when jumping from a wizard to the advanced job creation,
// the wizard's created_by information should be stripped.
if (tempJob.custom_settings && tempJob.custom_settings.created_by) {
delete tempJob.custom_settings.created_by;
}
return tempJob;
}
function render() {
// Clear any existing elements from the visualization,
// then build the svg elements for the bar chart.
const chartElement = d3.select(element.get(0)).select('.content-wrapper');
chartElement.selectAll('*').remove();
if (scope.chartData === undefined) {
return;
}
const svg = chartElement.append('svg')
.attr('width', svgWidth)
.attr('height', svgHeight);
// Set the size of the left margin according to the width
// of the largest y axis tick label.
const maxYVal = d3.max(scope.chartData, (d) => d.value);
yScale = yScale.domain([0, maxYVal]);
const yAxis = d3.svg.axis().scale(yScale).orient('left').outerTickSize(0);
// barChartGroup translate doesn't seem to be relative
// to parent svg, so have to add an extra 5px on.
const maxYAxisLabelWidth = calculateTextWidth(maxYVal, true, svg);
margin.left = Math.max(maxYAxisLabelWidth + yAxis.tickPadding() + 5, 25);
chartWidth = Math.max(svgWidth - margin.left - margin.right, 0);
const bounds = timefilter.getActiveBounds();
xScale = d3.time.scale()
.domain([new Date(bounds.min.valueOf()), new Date(bounds.max.valueOf())])
.range([0, chartWidth]);
if (scope.chartData.length > 0) {
// x axis tick format and bar width determined by data aggregation interval.
const buckets = new MlTimeBuckets();
const aggInterval = _.get(scope, ['card', 'stats', 'documentCounts', 'interval']);
buckets.setInterval(aggInterval);
buckets.setBounds(bounds);
xAxisTickFormat = buckets.getScaledDateFormat();
const intervalMs = parseInterval(aggInterval).asMilliseconds();
barWidth = xScale(scope.chartData[0].time + intervalMs) - xScale(scope.chartData[0].time);
}
const xAxis = d3.svg.axis().scale(xScale).orient('bottom')
.outerTickSize(0).ticks(numTicksForDateFormat(chartWidth, xAxisTickFormat))
.tickFormat((d) => {
return moment(d).format(xAxisTickFormat);
});
barChartGroup = svg.append('g')
.attr('class', 'bar-chart')
.attr('transform', `translate(${margin.left}, ${margin.top})`);
drawBarChartAxes(xAxis, yAxis);
drawBarChartPaths();
}
export function validateInterval(bounds, panel, maxBuckets) {
const { interval } = panel;
const { min, max } = bounds;
// No need to check auto it will return around 100
if (!interval) return;
if (interval === 'auto') return;
const greaterThanMatch = interval.match(GTE_INTERVAL_RE);
if (greaterThanMatch) return;
const duration = parseInterval(interval);
if (duration) {
const span = max.valueOf() - min.valueOf();
const buckets = Math.floor(span / duration.asMilliseconds());
if (buckets > maxBuckets) {
throw new Error(`Max buckets exceeded: ${buckets} is greater than ${maxBuckets}, try a larger time interval in the panel options.`);
}
}
}
_.each(jobsListCopy, (jobObj) => {
const analysisConfig = jobObj.analysis_config;
const bucketSpan = parseInterval(analysisConfig.bucket_span);
const job = {
id: jobObj.job_id,
bucketSpanSeconds: bucketSpan.asSeconds()
};
if (_.has(jobObj, 'description') && /^\s*$/.test(jobObj.description) === false) {
job.description = jobObj.description;
} else {
// Just use the id as the description.
job.description = jobObj.job_id;
}
job.detectors = _.get(analysisConfig, 'detectors', []);
detectorsByJob[job.id] = job.detectors;
if (_.has(jobObj, 'custom_settings.custom_urls')) {
job.customUrls = [];
_.each(jobObj.custom_settings.custom_urls, (url) => {
if (_.has(url, 'url_name') && _.has(url, 'url_value') && isWebUrl(url.url_value)) {
// Only make web URLs (i.e. http or https) available in dashboard drilldowns.
job.customUrls.push(url);
}
});
// Only add an entry for a job if customUrls have been defined.
if (job.customUrls.length > 0) {
customUrlsByJob[job.id] = job.customUrls;
}
}
localJobService.jobDescriptions[job.id] = job.description;
localJobService.basicJobs[job.id] = job;
processedJobsList.push(job);
});
link: function ($scope) {
$scope.config = mlCreateWatchService.config;
$scope.status = mlCreateWatchService.status;
$scope.STATUS = mlCreateWatchService.STATUS;
$scope.ui = {
emailEnabled: false,
embedded: $scope.embedded,
watchAlreadyExists: false
};
// make the interval 2 times the bucket span
if ($scope.bucketSpan) {
const interval = parseInterval($scope.bucketSpan);
let bs = interval.asMinutes() * 2;
if (bs < 1) {
bs = 1;
}
$scope.config.interval = `${bs}m`;
}
// load elasticsearch settings to see if email has been configured
ml.getNotificationSettings().then((resp) => {
if (_.has(resp, 'defaults.xpack.notification.email')) {
$scope.ui.emailEnabled = true;
}
});
// check to see whether a watch for this job has already been created.
// display a warning if it has.
mlCreateWatchService.loadWatch($scope.jobId)
.then(() => {
$scope.ui.watchAlreadyExists = true;
})
.catch(() => {
$scope.ui.watchAlreadyExists = false;
});
}
_.each(mlJobService.jobs, (job) => {
if (isTimeSeriesViewJob(job) === true) {
const bucketSpan = parseInterval(job.analysis_config.bucket_span);
newJobs.push({ id: job.job_id, selected: false, bucketSpanSeconds: bucketSpan.asSeconds() });
}
});
.then((timeRange) => {
const bucketSpan = parseInterval(job.analysis_config.bucket_span);
const earliestMs = timeRange.start.epoch;
const latestMs = +timeRange.start.epoch + (10 * bucketSpan.asMilliseconds());
const body = {
query: {
bool: {
must: [
{
range: {
[job.data_description.time_field]: {
gte: earliestMs,
lt: latestMs,
format: 'epoch_millis'
}
}
},
query
]
}
}
};
// if aggs or aggregations is set, add it to the search
const aggregations = job.datafeed_config.aggs || job.datafeed_config.aggregations;
if (aggregations && Object.keys(aggregations).length) {
body.size = 0;
body.aggregations = aggregations;
// add script_fields if present
const scriptFields = job.datafeed_config.script_fields;
if (scriptFields && Object.keys(scriptFields).length) {
body.script_fields = scriptFields;
}
} else {
// if aggregations is not set and retrieveWholeSource is not set, add all of the fields from the job
body.size = ML_DATA_PREVIEW_COUNT;
// add script_fields if present
const scriptFields = job.datafeed_config.script_fields;
if (scriptFields && Object.keys(scriptFields).length) {
body.script_fields = scriptFields;
}
const fields = {};
// get fields from detectors
if (job.analysis_config.detectors) {
_.each(job.analysis_config.detectors, (dtr) => {
if (dtr.by_field_name) {
fields[dtr.by_field_name] = {};
}
if (dtr.field_name) {
fields[dtr.field_name] = {};
}
if (dtr.over_field_name) {
fields[dtr.over_field_name] = {};
}
if (dtr.partition_field_name) {
fields[dtr.partition_field_name] = {};
}
});
}
// get fields from influencers
if (job.analysis_config.influencers) {
_.each(job.analysis_config.influencers, (inf) => {
fields[inf] = {};
});
}
// get fields from categorizationFieldName
if (job.analysis_config.categorization_field_name) {
fields[job.analysis_config.categorization_field_name] = {};
}
// get fields from summary_count_field_name
if (job.analysis_config.summary_count_field_name) {
fields[job.analysis_config.summary_count_field_name] = {};
}
// get fields from time_field
if (job.data_description.time_field) {
fields[job.data_description.time_field] = {};
}
// console.log('fields: ', fields);
const fieldsList = Object.keys(fields);
if (fieldsList.length) {
body._source = fieldsList;
}
}
const data = {
index: job.datafeed_config.indices,
body
};
ml.esSearch(data)
.then((resp) => {
resolve(resp);
})
.catch((resp) => {
reject(resp);
});
})
getJobFromConfig(formConfig) {
const job = mlJobService.getBlankJob();
job.data_description.time_field = formConfig.timeField;
let func = formConfig.agg.type.mlName;
if (formConfig.isSparseData) {
if (formConfig.agg.type.dslName === 'count') {
func = func.replace(/count/, 'non_zero_count');
} else if(formConfig.agg.type.dslName === 'sum') {
func = func.replace(/sum/, 'non_null_sum');
}
}
const dtr = {
function: func
};
const query = formConfig.combinedQuery;
if (formConfig.field && formConfig.field.id) {
dtr.field_name = formConfig.field.id;
}
job.analysis_config.detectors.push(dtr);
job.analysis_config.bucket_span = formConfig.bucketSpan;
job.analysis_limits = {
model_memory_limit: formConfig.modelMemoryLimit
};
delete job.data_description.field_delimiter;
delete job.data_description.quote_character;
delete job.data_description.time_format;
delete job.data_description.format;
const bucketSpanSeconds = parseInterval(formConfig.bucketSpan).asSeconds();
const indices = formConfig.indexPattern.title.split(',').map(i => i.trim());
job.datafeed_config = {
query,
indices,
};
job.job_id = formConfig.jobId;
job.description = formConfig.description;
job.groups = formConfig.jobGroups;
job.model_plot_config = {
enabled: true
};
if (formConfig.useDedicatedIndex) {
job.results_index_name = job.job_id;
}
if (formConfig.usesSavedSearch === false) {
// Jobs created from saved searches cannot be cloned in the wizard as the
// ML job config holds no reference to the saved search ID.
job.custom_settings = {
created_by: WIZARD_TYPE.SINGLE_METRIC
};
}
// Use the original es agg type rather than the ML version
// e.g. count rather than high_count
const aggType = formConfig.agg.type.dslName;
const interval = bucketSpanSeconds * 1000;
switch (aggType) {
case 'count':
job.analysis_config.summary_count_field_name = 'doc_count';
job.datafeed_config.aggregations = {
buckets: {
date_histogram: {
field: formConfig.timeField,
fixed_interval: `${interval}ms`
},
aggregations: {
[formConfig.timeField]: {
max: {
field: formConfig.timeField
}
}
}
}
};
break;
case 'avg':
case 'median':
case 'sum':
case 'min':
case 'max':
job.analysis_config.summary_count_field_name = 'doc_count';
job.datafeed_config.aggregations = {
buckets: {
date_histogram: {
field: formConfig.timeField,
fixed_interval: `${((interval / 100) * 10)}ms` // use 10% of bucketSpan to allow for better sampling
},
aggregations: {
[dtr.field_name]: {
[aggType]: {
field: formConfig.field.name
}
},
[formConfig.timeField]: {
max: {
field: formConfig.timeField
}
}
}
}
};
break;
case 'cardinality':
job.analysis_config.summary_count_field_name = 'dc_' + dtr.field_name;
job.datafeed_config.aggregations = {
buckets: {
date_histogram: {
field: formConfig.timeField,
fixed_interval: `${interval}ms`
},
aggregations: {
[formConfig.timeField]: {
max: {
field: formConfig.timeField
}
},
[job.analysis_config.summary_count_field_name]: {
[aggType]: {
field: formConfig.field.name
}
}
}
}
};
// finally, modify the detector before saving
dtr.function = 'non_zero_count';
// add a description using the original function name rather 'non_zero_count'
// as the user may not be aware it's been changed
dtr.detector_description = `${func} (${dtr.field_name})`;
delete dtr.field_name;
break;
default:
break;
}
return job;
}
// Builds the full URL for testing out a custom URL configuration, which
// may contain dollar delimited partition / influencer entity tokens and
// drilldown time range settings.
function getTestUrl(job, urlConfig) {
const urlValue = urlConfig.url_value;
const bucketSpanSecs = parseInterval(job.analysis_config.bucket_span).asSeconds();
// By default, return configured url_value. Look to substitute any dollar-delimited
// tokens with values from the highest scoring anomaly, or if no anomalies, with
// values from a document returned by the search in the job datafeed.
let testUrl = urlConfig.url_value;
// Query to look for the highest scoring anomaly.
const body = {
query: {
bool: {
must: [
{ term: { job_id: job.job_id } },
{ term: { result_type: 'record' } }
]
}
},
size: 1,
_source: {
excludes: []
},
sort: [
{ record_score: { order: 'desc' } }
]
};
return new Promise((resolve, reject) => {
ml.esSearch({
index: ML_RESULTS_INDEX_PATTERN,
body
})
.then((resp) => {
if (resp.hits.total > 0) {
const record = resp.hits.hits[0]._source;
testUrl = replaceTokensInUrlValue(urlConfig, bucketSpanSecs, record, 'timestamp');
resolve(testUrl);
} else {
// No anomalies yet for this job, so do a preview of the search
// configured in the job datafeed to obtain sample docs.
mlJobService.searchPreview(job)
.then((response) => {
let testDoc;
const docTimeFieldName = job.data_description.time_field;
// Handle datafeeds which use aggregations or documents.
if (response.aggregations) {
// Create a dummy object which contains the fields necessary to build the URL.
const firstBucket = response.aggregations.buckets.buckets[0];
testDoc = {
[docTimeFieldName]: firstBucket.key
};
// Look for bucket aggregations which match the tokens in the URL.
urlValue.replace((/\$([^?&$\'"]{1,40})\$/g), (match, name) => {
if (name !== 'earliest' && name !== 'latest' && firstBucket[name] !== undefined) {
const tokenBuckets = firstBucket[name];
if (tokenBuckets.buckets) {
testDoc[name] = tokenBuckets.buckets[0].key;
}
}
});
} else {
if (response.hits.total > 0) {
testDoc = response.hits.hits[0]._source;
}
}
if (testDoc !== undefined) {
testUrl = replaceTokensInUrlValue(urlConfig, bucketSpanSecs, testDoc, docTimeFieldName);
}
resolve(testUrl);
});
}
})
.catch((resp) => {
reject(resp);
});
});
}
// work out the default frequency based on the bucket_span
function calculateDatafeedFrequencyDefaultSeconds() {
const bucketSpan = parseInterval($scope.job.analysis_config.bucket_span);
if (bucketSpan !== null) {
$scope.ui.datafeed.frequencyDefault = juCalculateDatafeedFrequencyDefaultSeconds(bucketSpan.asSeconds()) + 's';
}
}
function check(value) {
ngModelCntrl.$setValidity('dateInterval', parseInterval(value) != null);
return value;
}