From f1ba6743865800d99569e993bc88eda29964b189 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Fri, 13 Dec 2019 16:52:41 +0000 Subject: [PATCH 01/29] [ML] Categorization wizard --- .../plugins/ml/common/constants/new_job.ts | 2 + .../job_creator/categorization_job_creator.ts | 70 ++++++ .../jobs/new_job/common/job_creator/index.ts | 2 + .../common/job_creator/job_creator_factory.ts | 4 + .../new_job/common/job_creator/type_guards.ts | 10 +- .../charts/anomaly_chart/anomaly_chart.tsx | 2 +- .../{anomaly_chart => common}/anomalies.tsx | 0 .../components/charts/common/settings.ts | 1 + .../event_rate_chart/event_rate_chart.tsx | 13 +- .../components/advanced_view/settings.tsx | 16 +- .../categorization_field.tsx | 6 +- .../categorization_view.tsx | 46 ++++ .../categorization_view/field_examples.tsx | 71 ++++++ .../components/categorization_view/index.ts | 7 + .../categorization_view/metric_selection.tsx | 193 +++++++++++++++ .../metric_selection_summary.tsx | 78 +++++++ .../categorization_view/settings.tsx | 30 +++ .../categorization_view/top_categories.tsx | 89 +++++++ .../components/multi_metric_view/settings.tsx | 16 +- .../components/population_view/settings.tsx | 16 +- .../single_metric_view/settings.tsx | 15 +- .../pick_fields_step/pick_fields.tsx | 4 + .../detector_chart/detector_chart.tsx | 2 + .../preconfigured_job_redirect.ts | 25 +- .../jobs/new_job/pages/job_type/page.tsx | 16 ++ .../jobs/new_job/pages/new_job/page.tsx | 17 +- .../routing/routes/new_job/wizard.tsx | 18 ++ .../services/ml_api_service/index.d.ts | 11 +- .../services/ml_api_service/jobs.js | 33 +++ .../ml/server/client/elasticsearch_ml.js | 26 +++ .../ml/server/models/job_service/index.js | 3 +- .../job_service/new_job/categorization.ts | 219 ++++++++++++++++++ .../models/job_service/new_job/index.ts | 1 + .../ml/server/routes/anomaly_detectors.js | 19 ++ .../plugins/ml/server/routes/job_service.js | 41 ++++ 35 files changed, 1047 insertions(+), 75 deletions(-) create mode 100644 x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts rename x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/{anomaly_chart => common}/anomalies.tsx (100%) create mode 100644 x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/categorization_view.tsx create mode 100644 x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx create mode 100644 x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/index.ts create mode 100644 x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx create mode 100644 x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection_summary.tsx create mode 100644 x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/settings.tsx create mode 100644 x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx create mode 100644 x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts diff --git a/x-pack/legacy/plugins/ml/common/constants/new_job.ts b/x-pack/legacy/plugins/ml/common/constants/new_job.ts index 004824bef1c9d..d62000fb7f19f 100644 --- a/x-pack/legacy/plugins/ml/common/constants/new_job.ts +++ b/x-pack/legacy/plugins/ml/common/constants/new_job.ts @@ -9,12 +9,14 @@ export enum JOB_TYPE { MULTI_METRIC = 'multi_metric', POPULATION = 'population', ADVANCED = 'advanced', + CATEGORIZATION = 'categorization', } export enum CREATED_BY_LABEL { SINGLE_METRIC = 'single-metric-wizard', MULTI_METRIC = 'multi-metric-wizard', POPULATION = 'population-wizard', + CATEGORIZATION = 'categorization-wizard', } export const DEFAULT_MODEL_MEMORY_LIMIT = '10MB'; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts new file mode 100644 index 0000000000000..aa367f0e0879b --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IndexPattern } from 'ui/index_patterns'; +import { SavedSearchSavedObject } from '../../../../../../common/types/kibana'; +import { JobCreator } from './job_creator'; +import { Field, Aggregation, mlCategory } from '../../../../../../common/types/fields'; +import { Job, Datafeed, Detector } from './configs'; +import { createBasicDetector } from './util/default_configs'; +import { JOB_TYPE, CREATED_BY_LABEL } from '../../../../../../common/constants/new_job'; +import { getRichDetectors } from './util/general'; + +export class CategorizationJobCreator extends JobCreator { + protected _type: JOB_TYPE = JOB_TYPE.CATEGORIZATION; + private _createDetector: () => void = () => {}; + + constructor( + indexPattern: IndexPattern, + savedSearch: SavedSearchSavedObject | null, + query: object + ) { + super(indexPattern, savedSearch, query); + this.createdBy = CREATED_BY_LABEL.CATEGORIZATION; + } + + public setDefaultDetectorProperties(count: Aggregation | null, eventRate: Field | null) { + if (count === null || eventRate === null) { + return; + } + const dtr: Detector = createBasicDetector(count, eventRate); + dtr.by_field_name = mlCategory.id; + this._createDetector = () => { + this._addDetector(dtr, count, mlCategory); + }; + } + + public set categorizationFieldName(fieldName: string | null) { + this.removeAllDetectors(); + this.removeAllInfluencers(); + + if (fieldName !== null) { + this._job_config.analysis_config.categorization_field_name = fieldName; + this._createDetector(); + this.addInfluencer(mlCategory.id); + } else { + delete this._job_config.analysis_config.categorization_field_name; + } + } + + public get categorizationFieldName(): string | null { + return this._job_config.analysis_config.categorization_field_name || null; + } + + public cloneFromExistingJob(job: Job, datafeed: Datafeed) { + this._overrideConfigs(job, datafeed); + this.createdBy = CREATED_BY_LABEL.CATEGORIZATION; + const detectors = getRichDetectors(job, datafeed, this.scriptFields, false); + + this.removeAllDetectors(); + + const dtr = detectors[0]; + if (detectors.length && dtr.agg !== null && dtr.field !== null) { + this.setDefaultDetectorProperties(dtr.agg, dtr.field); + this._createDetector(); + } + } +} diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/index.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/index.ts index 8422223ad91fb..88bacdf49c38a 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/index.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/index.ts @@ -9,11 +9,13 @@ export { SingleMetricJobCreator } from './single_metric_job_creator'; export { MultiMetricJobCreator } from './multi_metric_job_creator'; export { PopulationJobCreator } from './population_job_creator'; export { AdvancedJobCreator } from './advanced_job_creator'; +export { CategorizationJobCreator } from './categorization_job_creator'; export { JobCreatorType, isSingleMetricJobCreator, isMultiMetricJobCreator, isPopulationJobCreator, isAdvancedJobCreator, + isCategorizationJobCreator, } from './type_guards'; export { jobCreatorFactory } from './job_creator_factory'; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator_factory.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator_factory.ts index 868346d0188ea..8655b83a244ad 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator_factory.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator_factory.ts @@ -10,6 +10,7 @@ import { MultiMetricJobCreator } from './multi_metric_job_creator'; import { PopulationJobCreator } from './population_job_creator'; import { AdvancedJobCreator } from './advanced_job_creator'; import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public'; +import { CategorizationJobCreator } from './categorization_job_creator'; import { JOB_TYPE } from '../../../../../../common/constants/new_job'; @@ -32,6 +33,9 @@ export const jobCreatorFactory = (jobType: JOB_TYPE) => ( case JOB_TYPE.ADVANCED: jc = AdvancedJobCreator; break; + case JOB_TYPE.CATEGORIZATION: + jc = CategorizationJobCreator; + break; default: jc = SingleMetricJobCreator; break; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/type_guards.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/type_guards.ts index 9feb0416dd267..25ea80e18eeb3 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/type_guards.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/type_guards.ts @@ -8,13 +8,15 @@ import { SingleMetricJobCreator } from './single_metric_job_creator'; import { MultiMetricJobCreator } from './multi_metric_job_creator'; import { PopulationJobCreator } from './population_job_creator'; import { AdvancedJobCreator } from './advanced_job_creator'; +import { CategorizationJobCreator } from './categorization_job_creator'; import { JOB_TYPE } from '../../../../../../common/constants/new_job'; export type JobCreatorType = | SingleMetricJobCreator | MultiMetricJobCreator | PopulationJobCreator - | AdvancedJobCreator; + | AdvancedJobCreator + | CategorizationJobCreator; export function isSingleMetricJobCreator( jobCreator: JobCreatorType @@ -37,3 +39,9 @@ export function isPopulationJobCreator( export function isAdvancedJobCreator(jobCreator: JobCreatorType): jobCreator is AdvancedJobCreator { return jobCreator.type === JOB_TYPE.ADVANCED; } + +export function isCategorizationJobCreator( + jobCreator: JobCreatorType +): jobCreator is CategorizationJobCreator { + return jobCreator.type === JOB_TYPE.CATEGORIZATION; +} diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/anomaly_chart/anomaly_chart.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/anomaly_chart/anomaly_chart.tsx index 728229fc3091d..e519b86278ed8 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/anomaly_chart/anomaly_chart.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/anomaly_chart/anomaly_chart.tsx @@ -7,7 +7,7 @@ import React, { FC } from 'react'; import { Chart, Settings, TooltipType } from '@elastic/charts'; import { ModelItem, Anomaly } from '../../../../common/results_loader'; -import { Anomalies } from './anomalies'; +import { Anomalies } from '../common/anomalies'; import { ModelBounds } from './model_bounds'; import { Line } from './line'; import { Scatter } from './scatter'; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/anomaly_chart/anomalies.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/common/anomalies.tsx similarity index 100% rename from x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/anomaly_chart/anomalies.tsx rename to x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/common/anomalies.tsx diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/common/settings.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/common/settings.ts index b1852cbb259c1..7dec882429dce 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/common/settings.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/common/settings.ts @@ -20,6 +20,7 @@ const themeName = IS_DARK_THEME ? darkTheme : lightTheme; export const LINE_COLOR = themeName.euiColorPrimary; export const MODEL_COLOR = themeName.euiColorPrimary; export const EVENT_RATE_COLOR = themeName.euiColorPrimary; +export const EVENT_RATE_COLOR_WITH_ANOMALIES = themeName.euiColorLightShade; export interface ChartSettings { width: string; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/event_rate_chart/event_rate_chart.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/event_rate_chart/event_rate_chart.tsx index 5423e80a82f15..ddbeb3f0f5b04 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/event_rate_chart/event_rate_chart.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/event_rate_chart/event_rate_chart.tsx @@ -8,24 +8,32 @@ import React, { FC } from 'react'; import { BarSeries, Chart, ScaleType, Settings, TooltipType } from '@elastic/charts'; import { Axes } from '../common/axes'; import { LineChartPoint } from '../../../../common/chart_loader'; -import { EVENT_RATE_COLOR } from '../common/settings'; +import { Anomaly } from '../../../../common/results_loader'; +import { EVENT_RATE_COLOR, EVENT_RATE_COLOR_WITH_ANOMALIES } from '../common/settings'; import { LoadingWrapper } from '../loading_wrapper'; +import { Anomalies } from '../common/anomalies'; interface Props { eventRateChartData: LineChartPoint[]; + anomalyData?: Anomaly[]; height: string; width: string; showAxis?: boolean; loading?: boolean; + fadeChart?: boolean; } export const EventRateChart: FC = ({ eventRateChartData, + anomalyData, height, width, showAxis, loading = false, + fadeChart, }) => { + const barColor = fadeChart ? EVENT_RATE_COLOR_WITH_ANOMALIES : EVENT_RATE_COLOR; + return (
= ({ {showAxis === true && } + = ({ xAccessor={'time'} yAccessors={['value']} data={eventRateChartData} - customSeriesColors={[EVENT_RATE_COLOR]} + customSeriesColors={[barColor]} /> diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_view/settings.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_view/settings.tsx index bae9767a39077..e5aee97256798 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_view/settings.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_view/settings.tsx @@ -4,10 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { Fragment, FC, useContext, useEffect, useState } from 'react'; +import React, { Fragment, FC } from 'react'; import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; -import { JobCreatorContext } from '../../../job_creator_context'; import { BucketSpan } from '../bucket_span'; import { Influencers } from '../influencers'; import { ModelMemoryLimitInput } from '../../../common/model_memory_limit'; @@ -17,19 +16,6 @@ interface Props { } export const AdvancedSettings: FC = ({ setIsValid }) => { - const { jobCreator, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext); - const [bucketSpan, setBucketSpan] = useState(jobCreator.bucketSpan); - - useEffect(() => { - jobCreator.bucketSpan = bucketSpan; - jobCreatorUpdate(); - setIsValid(bucketSpan !== ''); - }, [bucketSpan]); - - useEffect(() => { - setBucketSpan(jobCreator.bucketSpan); - }, [jobCreatorUpdated]); - return ( diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_field/categorization_field.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_field/categorization_field.tsx index f9edf79364c97..015300debb156 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_field/categorization_field.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_field/categorization_field.tsx @@ -25,8 +25,10 @@ export const CategorizationField: FC = () => { ); useEffect(() => { - jobCreator.categorizationFieldName = categorizationFieldName; - jobCreatorUpdate(); + if (jobCreator.categorizationFieldName !== categorizationFieldName) { + jobCreator.categorizationFieldName = categorizationFieldName; + jobCreatorUpdate(); + } }, [categorizationFieldName]); useEffect(() => { diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/categorization_view.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/categorization_view.tsx new file mode 100644 index 0000000000000..9b394dac948aa --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/categorization_view.tsx @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { Fragment, FC, useEffect, useState } from 'react'; +import { EuiHorizontalRule } from '@elastic/eui'; + +import { CategorizationDetectors } from './metric_selection'; +import { CategorizationDetectorsSummary } from './metric_selection_summary'; +import { CategorizationSettings } from './settings'; + +interface Props { + isActive: boolean; + setCanProceed?: (proceed: boolean) => void; +} + +export const CategorizationView: FC = ({ isActive, setCanProceed }) => { + const [categoryFieldValid, setCategoryFieldValid] = useState(false); + const [settingsValid, setSettingsValid] = useState(false); + + useEffect(() => { + if (typeof setCanProceed === 'function') { + setCanProceed(categoryFieldValid && settingsValid); + } + }, [categoryFieldValid, settingsValid]); + + return ( + + {isActive === false ? ( + + ) : ( + + + {categoryFieldValid && ( + + + + + )} + + )} + + ); +}; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx new file mode 100644 index 0000000000000..b2a47830fc54f --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC } from 'react'; +import { EuiBasicTable, EuiText } from '@elastic/eui'; + +interface Props { + fieldExamples: Cat[]; +} + +interface Token { + token: string; + start_offset: number; + end_offset: number; + type: string; + position: number; +} + +interface Cat { + text: string; + tokens: Token[]; +} + +const TOKEN_HIGHLIGHT_COLOR = '#e9f2ff'; + +export const FieldExamples: FC = ({ fieldExamples }) => { + if (fieldExamples.length === 0) { + return null; + } + + const columns = [ + { + field: 'example', + name: 'Examples', + render: (example: any) => ( + + {example} + + ), + }, + ]; + const items = fieldExamples.map((example, e) => { + const txt = []; + let tokenCounter = 0; + let buffer = ''; + let i = 0; + while (i < example.text.length) { + const token = example.tokens[tokenCounter]; + if (token && i === token.start_offset) { + txt.push(buffer); + buffer = ''; + txt.push({token.token}); + i += token.end_offset - token.start_offset; + tokenCounter++; + } else { + buffer += example.text[i]; + i++; + } + } + txt.push(buffer); + return { example: txt }; + }); + return ; +}; + +const Token: FC = ({ children }) => ( + {children} +); diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/index.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/index.ts new file mode 100644 index 0000000000000..f61dfd88a37bb --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { CategorizationView } from './categorization_view'; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx new file mode 100644 index 0000000000000..1d22f697aaa69 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx @@ -0,0 +1,193 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { Fragment, FC, useContext, useEffect, useState } from 'react'; + +import { EuiCode } from '@elastic/eui'; +import { JobCreatorContext } from '../../../job_creator_context'; +import { CategorizationJobCreator } from '../../../../../common/job_creator'; +import { ml } from '../../../../../../../services/ml_api_service'; +// import { LineChartData } from '../../../../../common/chart_loader'; +// import { DropDownProps, createLabel } from '../agg_select'; +import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service'; +import { mlCategory } from '../../../../../../../../../common/types/fields'; +import { CategorizationField } from '../categorization_field'; +import { useKibanaContext } from '../../../../../../../contexts/kibana'; +import { FieldExamples } from './field_examples'; + +interface Props { + setIsValid: (na: boolean) => void; +} + +interface Token { + token: string; + start_offset: number; + end_offset: number; + type: string; + position: number; +} + +interface Cat { + text: string; + tokens: Token[]; +} + +const NUM_OF_EXAMPLES = 5; + +// const DTR_IDX = 0; + +export const CategorizationDetectors: FC = ({ setIsValid }) => { + const kibanaContext = useKibanaContext(); + const { + jobCreator: jc, + jobCreatorUpdate, + jobCreatorUpdated, + // chartLoader, + // chartInterval, + } = useContext(JobCreatorContext); + const jobCreator = jc as CategorizationJobCreator; + + // const { fields } = newJobCapsService; + // const [selectedOptions, setSelectedOptions] = useState( + // jobCreator.aggFieldPair !== null ? [{ label: createLabel(jobCreator.aggFieldPair) }] : [] + // ); + // const [aggFieldPair, setAggFieldPair] = useState(jobCreator.aggFieldPair); + // const [lineChartsData, setLineChartData] = useState({}); + const [loadingData, setLoadingData] = useState(false); + const [start, setStart] = useState(jobCreator.start); + const [end, setEnd] = useState(jobCreator.end); + const [fieldExamples, setFieldExamples] = useState([]); + const [tokens, setTokens] = useState([]); + // const [bucketSpanMs, setBucketSpanMs] = useState(jobCreator.bucketSpanMs); + + const [categorizationFieldName, setCategorizationFieldName] = useState( + jobCreator.categorizationFieldName + ); + + // useEffect(() => { + // if (jobCreator.detectors.length !== 1) { + // const agg = newJobCapsService.getAggById('count'); + // if (agg !== null) { + // jobCreator.setDetector(agg, mlCategory); + // jobCreatorUpdate(); + // } + // } + // }, []); + + useEffect(() => { + if (jobCreator.categorizationFieldName !== categorizationFieldName) { + jobCreator.categorizationFieldName = categorizationFieldName; + jobCreatorUpdate(); + } + loadFieldExamples(); + }, [categorizationFieldName]); + + // function detectorChangeHandler(selectedOptionsIn: DropDownLabel[]) { + // setSelectedOptions(selectedOptionsIn); + // if (selectedOptionsIn.length) { + // const option = selectedOptionsIn[0]; + // if (typeof option !== 'undefined') { + // setAggFieldPair({ agg: option.agg, field: option.field }); + // } else { + // setAggFieldPair(null); + // } + // } + // } + + // useEffect(() => { + // if (aggFieldPair !== null) { + // jobCreator.setDetector(aggFieldPair.agg, aggFieldPair.field); + // jobCreatorUpdate(); + // loadChart(); + // setIsValid(aggFieldPair !== null); + // } + // }, [aggFieldPair]); + + useEffect(() => { + if (jobCreator.start !== start || jobCreator.end !== end) { + setStart(jobCreator.start); + setEnd(jobCreator.end); + loadFieldExamples(); + } + if (jobCreator.categorizationFieldName !== categorizationFieldName) { + setCategorizationFieldName(jobCreator.categorizationFieldName); + } + }, [jobCreatorUpdated]); + + async function loadFieldExamples() { + if (categorizationFieldName !== null) { + setLoadingData(true); + const resp = await ml.jobs.categorizationFieldExamples( + kibanaContext.currentIndexPattern.title, + kibanaContext.combinedQuery, + NUM_OF_EXAMPLES, + categorizationFieldName, + 0, + 0 + ); + // const examples = resp.map(r => r.text); + // const tkns = resp.map(r => r.tokens); + // console.log(resp); + setFieldExamples(resp); + // const resp = await ml.esSearch({ + // index: kibanaContext.currentIndexPattern.title, + // size: 10, + // body: { + // _source: categorizationFieldName, + // query: kibanaContext.combinedQuery, + // }, + // }); + // try { + // const cs = getChartSettings(jobCreator, chartInterval); + // const resp: LineChartData = await chartLoader.loadLineCharts( + // jobCreator.start, + // jobCreator.end, + // [aggFieldPair], + // null, + // null, + // cs.intervalMs + // ); + // if (resp[DTR_IDX] !== undefined) { + // setLineChartData(resp); + // } + // } catch (error) { + // mlMessageBarService.notify.error(error); + // setLineChartData({}); + // } + setLoadingData(false); + } else { + setFieldExamples([]); + } + setIsValid(categorizationFieldName !== null); + } + + return ( + + + + + {/* */} + {/* {(lineChartsData[DTR_IDX] !== undefined || loadingData === true) && ( + + + + )} */} + + ); +}; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection_summary.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection_summary.tsx new file mode 100644 index 0000000000000..2c91e91ec51d2 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection_summary.tsx @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { Fragment, FC, useContext, useEffect, useState } from 'react'; +import { JobCreatorContext } from '../../../job_creator_context'; +import { CategorizationJobCreator } from '../../../../../common/job_creator'; +import { Results, Anomaly } from '../../../../../common/results_loader'; +import { LineChartPoint } from '../../../../../common/chart_loader'; +import { EventRateChart } from '../../../charts/event_rate_chart'; +import { TopCategories } from './top_categories'; + +const DTR_IDX = 0; + +export const CategorizationDetectorsSummary: FC = () => { + const { jobCreator: jc, chartLoader, resultsLoader, chartInterval } = useContext( + JobCreatorContext + ); + const jobCreator = jc as CategorizationJobCreator; + + const [loadingData, setLoadingData] = useState(false); + const [anomalyData, setAnomalyData] = useState([]); + const [eventRateChartData, setEventRateChartData] = useState([]); + const [jobIsRunning, setJobIsRunning] = useState(false); + + function setResultsWrapper(results: Results) { + const anomalies = results.anomalies[DTR_IDX]; + if (anomalies !== undefined) { + setAnomalyData(anomalies); + } + } + + function watchProgress(progress: number) { + setJobIsRunning(progress > 0); + } + + useEffect(() => { + // subscribe to progress and results + const resultsSubscription = resultsLoader.subscribeToResults(setResultsWrapper); + jobCreator.subscribeToProgress(watchProgress); + loadChart(); + return () => { + resultsSubscription.unsubscribe(); + }; + }, []); + + async function loadChart() { + setLoadingData(true); + try { + const resp = await chartLoader.loadEventRateChart( + jobCreator.start, + jobCreator.end, + chartInterval.getInterval().asMilliseconds() + ); + setEventRateChartData(resp); + } catch (error) { + setEventRateChartData([]); + } + setLoadingData(false); + } + + return ( + + + + + ); +}; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/settings.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/settings.tsx new file mode 100644 index 0000000000000..0044a99b72887 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/settings.tsx @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { Fragment, FC } from 'react'; +import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; + +import { BucketSpan } from '../bucket_span'; +import { SparseDataSwitch } from '../sparse_data'; + +interface Props { + setIsValid: (proceed: boolean) => void; +} + +export const CategorizationSettings: FC = ({ setIsValid }) => { + return ( + + + + + + + + + + + ); +}; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx new file mode 100644 index 0000000000000..d82c0c6d36454 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, useContext, useEffect, useState } from 'react'; +import { EuiBasicTable, EuiText } from '@elastic/eui'; +import { JobCreatorContext } from '../../../job_creator_context'; +import { CategorizationJobCreator } from '../../../../../common/job_creator'; +import { Results } from '../../../../../common/results_loader'; +import { ml } from '../../../../../../../services/ml_api_service'; + +const DTR_IDX = 0; + +interface Category { + // MOVE!!!!!!! + job_id: string; + category_id: number; + terms: string; + regex: string; + max_matching_length: number; + examples: string[]; + grok_pattern: string; +} + +export const TopCategories: FC = () => { + const { jobCreator: jc, chartLoader, resultsLoader, chartInterval } = useContext( + JobCreatorContext + ); + const jobCreator = jc as CategorizationJobCreator; + + const [categories, setCategories] = useState([]); + const [tableRow, setTableRow] = useState>([]); + const [totalCategories, setTotalCategories] = useState(0); + + function setResultsWrapper(results: Results) { + loadTopCats(); + } + + async function loadTopCats() { + const results = await ml.jobs.topCategories(jobCreator.jobId, 5); + setCategories(results.categories); + setTableRow(results.categories.map(c => ({ count: c.count, regex: c.category.regex }))); + setTotalCategories(results.total); + // console.log(topCats.total); + } + + useEffect(() => { + // subscribe to result updates + const resultsSubscription = resultsLoader.subscribeToResults(setResultsWrapper); + return () => { + resultsSubscription.unsubscribe(); + }; + }, []); + + const columns = [ + { + field: 'count', + name: 'count', + width: '100px', + render: (regex: any) => ( + + {regex} + + ), + }, + { + field: 'regex', + name: 'Regex', + render: (regex: any) => ( + + {regex} + + ), + }, + ]; + + return ( + <> + {totalCategories > 0 && ( + <> +
Total categories: {totalCategories}
+ + + )} + + ); +}; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/multi_metric_view/settings.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/multi_metric_view/settings.tsx index 5e800de755f26..b28a9d3da81dc 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/multi_metric_view/settings.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/multi_metric_view/settings.tsx @@ -4,10 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { Fragment, FC, useContext, useEffect, useState } from 'react'; +import React, { Fragment, FC } from 'react'; import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; -import { JobCreatorContext } from '../../../job_creator_context'; import { BucketSpan } from '../bucket_span'; import { SplitFieldSelector } from '../split_field'; import { Influencers } from '../influencers'; @@ -18,19 +17,6 @@ interface Props { } export const MultiMetricSettings: FC = ({ setIsValid }) => { - const { jobCreator, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext); - const [bucketSpan, setBucketSpan] = useState(jobCreator.bucketSpan); - - useEffect(() => { - jobCreator.bucketSpan = bucketSpan; - jobCreatorUpdate(); - setIsValid(bucketSpan !== ''); - }, [bucketSpan]); - - useEffect(() => { - setBucketSpan(jobCreator.bucketSpan); - }, [jobCreatorUpdated]); - return ( diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/population_view/settings.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/population_view/settings.tsx index 46cdf44ce0f7d..b9de755e6c946 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/population_view/settings.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/population_view/settings.tsx @@ -4,10 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { Fragment, FC, useContext, useEffect, useState } from 'react'; +import React, { Fragment, FC } from 'react'; import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; -import { JobCreatorContext } from '../../../job_creator_context'; import { BucketSpan } from '../bucket_span'; import { Influencers } from '../influencers'; @@ -16,19 +15,6 @@ interface Props { } export const PopulationSettings: FC = ({ setIsValid }) => { - const { jobCreator, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext); - const [bucketSpan, setBucketSpan] = useState(jobCreator.bucketSpan); - - useEffect(() => { - jobCreator.bucketSpan = bucketSpan; - jobCreatorUpdate(); - setIsValid(bucketSpan !== ''); - }, [bucketSpan]); - - useEffect(() => { - setBucketSpan(jobCreator.bucketSpan); - }, [jobCreatorUpdated]); - return ( diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/single_metric_view/settings.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/single_metric_view/settings.tsx index c750235051a6f..f8e7275cf15bb 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/single_metric_view/settings.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/single_metric_view/settings.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { Fragment, FC, useContext, useEffect, useState } from 'react'; +import React, { Fragment, FC, useContext } from 'react'; import { FormattedMessage } from '@kbn/i18n/react'; import { EuiFlexGroup, EuiFlexItem, EuiButtonEmpty } from '@elastic/eui'; @@ -19,18 +19,7 @@ interface Props { } export const SingleMetricSettings: FC = ({ setIsValid }) => { - const { jobCreator, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext); - const [bucketSpan, setBucketSpan] = useState(jobCreator.bucketSpan); - - useEffect(() => { - jobCreator.bucketSpan = bucketSpan; - jobCreatorUpdate(); - setIsValid(bucketSpan !== ''); - }, [bucketSpan]); - - useEffect(() => { - setBucketSpan(jobCreator.bucketSpan); - }, [jobCreatorUpdated]); + const { jobCreator } = useContext(JobCreatorContext); const convertToMultiMetric = () => { convertToMultiMetricJob(jobCreator); diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/pick_fields.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/pick_fields.tsx index 795dfc30f954a..c5830ccac82f6 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/pick_fields.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/pick_fields.tsx @@ -15,6 +15,7 @@ import { SingleMetricView } from './components/single_metric_view'; import { MultiMetricView } from './components/multi_metric_view'; import { PopulationView } from './components/population_view'; import { AdvancedView } from './components/advanced_view'; +import { CategorizationView } from './components/categorization_view'; import { JsonEditorFlyout, EDITOR_MODE } from '../common/json_editor_flyout'; import { DatafeedPreviewFlyout } from '../common/datafeed_preview_flyout'; @@ -50,6 +51,9 @@ export const PickFieldsStep: FC = ({ setCurrentStep, isCurrentStep }) {jobType === JOB_TYPE.ADVANCED && ( )} + {jobType === JOB_TYPE.CATEGORIZATION && ( + + )} setCurrentStep( diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/summary_step/components/detector_chart/detector_chart.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/summary_step/components/detector_chart/detector_chart.tsx index f72ff6cf985e5..564228604d71e 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/summary_step/components/detector_chart/detector_chart.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/summary_step/components/detector_chart/detector_chart.tsx @@ -11,6 +11,7 @@ import { SingleMetricView } from '../../../pick_fields_step/components/single_me import { MultiMetricView } from '../../../pick_fields_step/components/multi_metric_view'; import { PopulationView } from '../../../pick_fields_step/components/population_view'; import { AdvancedView } from '../../../pick_fields_step/components/advanced_view'; +import { CategorizationView } from '../../../pick_fields_step/components/categorization_view'; export const DetectorChart: FC = () => { const { jobCreator } = useContext(JobCreatorContext); @@ -21,6 +22,7 @@ export const DetectorChart: FC = () => { {jobCreator.type === JOB_TYPE.MULTI_METRIC && } {jobCreator.type === JOB_TYPE.POPULATION && } {jobCreator.type === JOB_TYPE.ADVANCED && } + {jobCreator.type === JOB_TYPE.CATEGORIZATION && } ); }; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/index_or_search/preconfigured_job_redirect.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/index_or_search/preconfigured_job_redirect.ts index 8500279e742b7..51fc226751ae2 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/index_or_search/preconfigured_job_redirect.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/index_or_search/preconfigured_job_redirect.ts @@ -32,15 +32,24 @@ function getWizardUrlFromCloningJob(job: CombinedJob) { const created = job?.custom_settings?.created_by; let page = ''; - if (created === CREATED_BY_LABEL.SINGLE_METRIC) { - page = JOB_TYPE.SINGLE_METRIC; - } else if (created === CREATED_BY_LABEL.MULTI_METRIC) { - page = JOB_TYPE.MULTI_METRIC; - } else if (created === CREATED_BY_LABEL.POPULATION) { - page = JOB_TYPE.POPULATION; - } else { - page = JOB_TYPE.ADVANCED; + switch (created) { + case CREATED_BY_LABEL.SINGLE_METRIC: + page = JOB_TYPE.SINGLE_METRIC; + break; + case CREATED_BY_LABEL.MULTI_METRIC: + page = JOB_TYPE.MULTI_METRIC; + break; + case CREATED_BY_LABEL.POPULATION: + page = JOB_TYPE.POPULATION; + break; + case CREATED_BY_LABEL.CATEGORIZATION: + page = JOB_TYPE.CATEGORIZATION; + break; + default: + page = JOB_TYPE.ADVANCED; + break; } + const indexPatternId = getIndexPatternIdFromName(job.datafeed_config.indices[0]); return `jobs/new_job/${page}?index=${indexPatternId}&_g=()`; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/job_type/page.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/job_type/page.tsx index dbae1948cbe0f..26136bc9ecfda 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/job_type/page.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/job_type/page.tsx @@ -151,6 +151,22 @@ export const Page: FC = () => { }), id: 'mlJobTypeLinkAdvancedJob', }, + { + href: getUrl('#jobs/new_job/categorization'), + icon: { + type: 'createAdvancedJob', + ariaLabel: i18n.translate('xpack.ml.newJob.wizard.jobType.categorizationAriaLabel', { + defaultMessage: 'Categorization job', + }), + }, + title: i18n.translate('xpack.ml.newJob.wizard.jobType.categorizationTitle', { + defaultMessage: 'Categorization', + }), + description: i18n.translate('xpack.ml.newJob.wizard.jobType.categorizationDescription', { + defaultMessage: 'All your categories are belong to us', + }), + id: 'mlJobTypeLinkCategorizationJob', + }, ]; return ( diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx index 2e025c4931818..43f6ebd07fe8e 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx @@ -19,8 +19,12 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; import { Wizard } from './wizard'; import { WIZARD_STEPS } from '../components/step_types'; -import { jobCreatorFactory, isAdvancedJobCreator } from '../../common/job_creator'; import { getJobCreatorTitle } from '../../common/job_creator/util/general'; +import { + jobCreatorFactory, + isAdvancedJobCreator, + isCategorizationJobCreator, +} from '../../common/job_creator'; import { JOB_TYPE, DEFAULT_MODEL_MEMORY_LIMIT, @@ -34,6 +38,8 @@ import { getTimeFilterRange } from '../../../../components/full_time_range_selec import { TimeBuckets } from '../../../../util/time_buckets'; import { ExistingJobsAndGroups, mlJobService } from '../../../../services/job_service'; import { expandCombinedJobConfig } from '../../common/job_creator/configs'; +import { newJobCapsService } from '../../../../services/new_job_capabilities_service'; +import { EVENT_RATE_FIELD_ID } from '../../../../../../common/types/fields'; const PAGE_WIDTH = 1200; // document.querySelector('.single-metric-job-container').width(); const BAR_TARGET = PAGE_WIDTH > 2000 ? 1000 : PAGE_WIDTH / 2; @@ -118,6 +124,15 @@ export const Page: FC = ({ existingJobsAndGroups, jobType }) => { jobCreator.createdBy = null; } + if (isCategorizationJobCreator(jobCreator)) { + // categorization job will always use a count agg, so give it + // to the job creator now + const count = newJobCapsService.getAggById('count'); + const eventRate = newJobCapsService.getFieldById(EVENT_RATE_FIELD_ID); + jobCreator.setDefaultDetectorProperties(count, eventRate); + jobCreator.modelPlot = true; + } + // auto set the time range if creating a new advanced job autoSetTimeRange = isAdvancedJobCreator(jobCreator); } diff --git a/x-pack/legacy/plugins/ml/public/application/routing/routes/new_job/wizard.tsx b/x-pack/legacy/plugins/ml/public/application/routing/routes/new_job/wizard.tsx index ea1baefdce0d1..99c0511cd09ce 100644 --- a/x-pack/legacy/plugins/ml/public/application/routing/routes/new_job/wizard.tsx +++ b/x-pack/legacy/plugins/ml/public/application/routing/routes/new_job/wizard.tsx @@ -72,6 +72,16 @@ const advancedBreadcrumbs = [ }, ]; +const categorizationBreadcrumbs = [ + ...baseBreadcrumbs, + { + text: i18n.translate('xpack.ml.jobsBreadcrumbs.categorizationLabel', { + defaultMessage: 'Categorization', + }), + href: '', + }, +]; + export const singleMetricRoute: MlRoute = { path: '/jobs/new_job/single_metric', render: (props, config, deps) => ( @@ -104,6 +114,14 @@ export const advancedRoute: MlRoute = { breadcrumbs: advancedBreadcrumbs, }; +export const categorizationRoute: MlRoute = { + path: '/jobs/new_job/categorization', + render: (props, config, deps) => ( + + ), + breadcrumbs: categorizationBreadcrumbs, +}; + const PageWrapper: FC = ({ location, config, jobType, deps }) => { const { index, savedSearchId } = queryString.parse(location.search); const { context, results } = useResolver(index, savedSearchId, config, { diff --git a/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/index.d.ts b/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/index.d.ts index 58692754d95d7..c9e1d61b0ac19 100644 --- a/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/index.d.ts +++ b/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/index.d.ts @@ -107,7 +107,7 @@ declare interface Ml { checkManageMLPrivileges(): Promise; getJobStats(obj: object): Promise; getDatafeedStats(obj: object): Promise; - esSearch(obj: object): any; + esSearch(obj: object): Promise; esSearch$(obj: object): Observable; getIndices(): Promise; dataRecognizerModuleJobsExist(obj: { moduleId: string }): Promise; @@ -171,6 +171,15 @@ declare interface Ml { start: number, end: number ): Promise<{ progress: number; isRunning: boolean; isJobClosed: boolean }>; + categorizationFieldExamples( + indexPatternTitle: string, + query: object, + size: number, + field: string, + start: number, + end: number + ): Promise; + topCategories(jobId: string, count: number): Promise<{ total: number; categories: any[] }>; }; estimateBucketSpan(data: BucketSpanEstimatorData): Promise; diff --git a/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/jobs.js b/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/jobs.js index 4bec070b2cfdf..7e0058469a196 100644 --- a/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/jobs.js +++ b/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/jobs.js @@ -206,4 +206,37 @@ export const jobs = { }, }); }, + + categorizationFieldExamples( + indexPatternTitle, + query, + size, + field, + start, + end, + ) { + return http({ + url: `${basePath}/jobs/categorization_field_examples`, + method: 'POST', + data: { + indexPatternTitle, + query, + size, + field, + start, + end, + } + }); + }, + + topCategories(jobId, count) { + return http({ + url: `${basePath}/jobs/top_categories`, + method: 'POST', + data: { + jobId, + count, + } + }); + }, }; diff --git a/x-pack/legacy/plugins/ml/server/client/elasticsearch_ml.js b/x-pack/legacy/plugins/ml/server/client/elasticsearch_ml.js index 2ca21efb0bd6a..7b713b269c31a 100644 --- a/x-pack/legacy/plugins/ml/server/client/elasticsearch_ml.js +++ b/x-pack/legacy/plugins/ml/server/client/elasticsearch_ml.js @@ -753,4 +753,30 @@ export const elasticsearchJsPlugin = (Client, config, components) => { ], method: 'GET', }); + + ml.categories = ca({ + urls: [ + { + fmt: '/_xpack/ml/anomaly_detectors/<%=jobId%>/results/categories/<%=categoryId%>', + req: { + jobId: { + type: 'string', + }, + categoryId: { + type: 'string', + }, + } + }, + { + fmt: '/_xpack/ml/anomaly_detectors/<%=jobId%>/results/categories', + req: { + jobId: { + type: 'string', + } + } + } + ], + method: 'GET' + }); + }; diff --git a/x-pack/legacy/plugins/ml/server/models/job_service/index.js b/x-pack/legacy/plugins/ml/server/models/job_service/index.js index 78d099dad6606..186bcbae84546 100644 --- a/x-pack/legacy/plugins/ml/server/models/job_service/index.js +++ b/x-pack/legacy/plugins/ml/server/models/job_service/index.js @@ -8,7 +8,7 @@ import { datafeedsProvider } from './datafeeds'; import { jobsProvider } from './jobs'; import { groupsProvider } from './groups'; import { newJobCapsProvider } from './new_job_caps'; -import { newJobChartsProvider } from './new_job'; +import { newJobChartsProvider, categorizationExamplesProvider } from './new_job'; export function jobServiceProvider(callWithRequest, request) { return { @@ -17,5 +17,6 @@ export function jobServiceProvider(callWithRequest, request) { ...groupsProvider(callWithRequest), ...newJobCapsProvider(callWithRequest, request), ...newJobChartsProvider(callWithRequest, request), + ...categorizationExamplesProvider(callWithRequest, request), }; } diff --git a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts new file mode 100644 index 0000000000000..26c31f8e05ab2 --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts @@ -0,0 +1,219 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ML_RESULTS_INDEX_PATTERN } from '../../../../common/constants/index_patterns'; + +type callWithRequestType = (action: string, params: any) => Promise; + +interface Token { + token: string; + start_offset: number; + end_offset: number; + type: string; + position: number; +} + +interface Category { + job_id: string; + category_id: number; + terms: string; + regex: string; + max_matching_length: number; + examples: string[]; + grok_pattern: string; +} + +const testCat: Category = { + job_id: 'dasdasdd', + category_id: 100, + terms: 'HEAD projects xdotool HTTP IrssiUrlLog', + regex: '.*?HEAD.+?projects.+?xdotool.+?HTTP.+?IrssiUrlLog.*', + max_matching_length: 119, + examples: [''], + grok_pattern: + '.*?%{IP:ipaddress}.+?%{HTTPDATE:timestamp}.+?HEAD.+?%{PATH:path}.*?projects.*?%{PATH:path2}.*?xdotool.*?%{PATH:path3}.+?HTTP.*?%{PATH:path4}.*?%{QUOTEDSTRING:field}.+?%{QUOTEDSTRING:field2}.*?IrssiUrlLog.*?%{PATH:path5}.*', +}; + +export function categorizationExamplesProvider(callWithRequest: callWithRequestType) { + async function categorizationExamples( + indexPatternTitle: string, + query: object, + size: number, + categorizationFieldName: string, + start: number, + end: number + ) { + const results = await callWithRequest('search', { + index: indexPatternTitle, + size, + body: { + _source: categorizationFieldName, + query, + }, + }); + const examples: string[] = results.hits?.hits + ?.map((doc: any) => doc._source[categorizationFieldName]) + .filter((example: string | undefined) => example !== undefined); + + let tokens: Token[] = []; + try { + const { tokens: tempTokens } = await callWithRequest('indices.analyze', { + body: { + analyzer: 'standard', + text: examples, + }, + }); + tokens = tempTokens; + } catch (error) { + // do somehting with this error + } + + const lengths = examples.map(e => e.length); + const sumLengths = lengths.map((s => (a: number) => (s += a))(0)); + + const tokensPerExample: Token[][] = lengths.map(l => []); + tokens.forEach((t, i) => { + for (let g = 0; g < sumLengths.length; g++) { + if (t.token === 'dashboard2') { + const d = 0; + } + if (t.start_offset <= sumLengths[g]) { + const offset = g > 0 ? sumLengths[g - 1] + g : 0; + tokensPerExample[g].push({ + ...t, + start_offset: t.start_offset - offset, + end_offset: t.end_offset - offset, + }); + break; + } + } + }); + + return examples.map((e, i) => ({ text: e, tokens: tokensPerExample[i] })); + } + + async function getCategories( + jobId: string, + categoryId: number + ): Promise<{ categories: Category[] }> { + return callWithRequest('ml.categories', { jobId, categoryId }); + } + + async function getTotalCategories(jobId: string): Promise<{ total: number }> { + const totalResp = await callWithRequest('search', { + index: ML_RESULTS_INDEX_PATTERN, + size: 0, + body: { + query: { + bool: { + filter: [ + { + term: { + job_id: jobId, + }, + }, + { + term: { + result_type: 'model_plot', + }, + }, + { + term: { + by_field_name: 'mlcategory', + }, + }, + ], + }, + }, + aggs: { + cat_count: { + cardinality: { + field: 'by_field_value', + }, + }, + }, + }, + }); + return totalResp?.aggregations?.cat_count?.value ?? 0; + } + + async function topCategories(jobId: string, numberOfCategories: number) { + const top = await callWithRequest('search', { + index: ML_RESULTS_INDEX_PATTERN, + size: 0, + body: { + query: { + bool: { + filter: [ + { + term: { + job_id: jobId, + }, + }, + { + term: { + result_type: 'model_plot', + }, + }, + { + term: { + by_field_name: 'mlcategory', + }, + }, + ], + }, + }, + aggs: { + cat_count: { + terms: { + field: 'by_field_value', + size: numberOfCategories, + }, + }, + }, + }, + }); + + const catCounts: Array<{ + id: number; + count: number; + }> = top.aggregations?.cat_count?.buckets.map((c: any) => ({ + id: c.key, + count: c.doc_count, + })); + + const tempCategories = await Promise.all(catCounts.map(({ id }) => getCategories(jobId, id))); + const categories = tempCategories + .map((t: any) => { + if (t.categories === undefined || t.categories.length === 0) { + return null; + } + return t.categories[0]; + }) + .filter(t => t !== null); + + const catsById = categories.reduce((p, c) => { + p[c.category_id] = c; + return p; + }, {} as Record); + + const total = await getTotalCategories(jobId); + return { + total, + categories: catCounts.map(({ id, count }) => { + return { + count, + category: catsById[id] ?? null, + }; + }), + }; + } + + return { + categorizationExamples, + topCategories, + }; +} diff --git a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/index.ts b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/index.ts index 758b834ed7b3a..da23efa67d0b5 100644 --- a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/index.ts +++ b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/index.ts @@ -5,3 +5,4 @@ */ export { newJobChartsProvider } from './charts'; +export { categorizationExamplesProvider } from './categorization'; diff --git a/x-pack/legacy/plugins/ml/server/routes/anomaly_detectors.js b/x-pack/legacy/plugins/ml/server/routes/anomaly_detectors.js index 4205027eb5ce2..eb1c01f8a7b06 100644 --- a/x-pack/legacy/plugins/ml/server/routes/anomaly_detectors.js +++ b/x-pack/legacy/plugins/ml/server/routes/anomaly_detectors.js @@ -181,4 +181,23 @@ export function jobRoutes({ commonRouteConfig, elasticsearchPlugin, route }) { ...commonRouteConfig, }, }); + + route({ + method: 'GET', + path: '/api/ml/anomaly_detectors/{jobId}/results/categories/{categoryId}', + handler(request, reply) { + const callWithRequest = callWithRequestFactory(elasticsearchPlugin, request); + const options = { + jobId: request.params.jobId, + categoryId: request.params.categoryId + }; + return callWithRequest('ml.categories', options) + .then(resp => reply(resp)) + .catch(resp => reply(wrapError(resp))); + }, + config: { + ...commonRouteConfig + } + }); + } diff --git a/x-pack/legacy/plugins/ml/server/routes/job_service.js b/x-pack/legacy/plugins/ml/server/routes/job_service.js index 78c57670b7ab3..0505e9841d179 100644 --- a/x-pack/legacy/plugins/ml/server/routes/job_service.js +++ b/x-pack/legacy/plugins/ml/server/routes/job_service.js @@ -270,4 +270,45 @@ export function jobServiceRoutes({ commonRouteConfig, elasticsearchPlugin, route ...commonRouteConfig, }, }); + + route({ + method: 'POST', + path: '/api/ml/jobs/categorization_field_examples', + handler(request) { + const callWithRequest = callWithRequestFactory(elasticsearchPlugin, request); + const { categorizationExamples } = jobServiceProvider(callWithRequest); + const { + indexPatternTitle, + query, + size, + field, + start, + end, + } = request.payload; + return categorizationExamples(indexPatternTitle, query, size, field, start, end) + .catch(resp => wrapError(resp)); + }, + config: { + ...commonRouteConfig + } + }); + + route({ + method: 'POST', + path: '/api/ml/jobs/top_categories', + handler(request) { + const callWithRequest = callWithRequestFactory(elasticsearchPlugin, request); + const { topCategories } = jobServiceProvider(callWithRequest); + const { + jobId, + count, + } = request.payload; + return topCategories(jobId, count) + .catch(resp => wrapError(resp)); + }, + config: { + ...commonRouteConfig + } + }); + } From a058464f013c9a2e71ec7ca60995ec3310eba456 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Mon, 16 Dec 2019 09:42:14 +0000 Subject: [PATCH 02/29] fixing js prettier issues --- .../services/ml_api_service/jobs.js | 13 ++------ .../ml/server/client/elasticsearch_ml.js | 11 +++---- .../ml/server/routes/anomaly_detectors.js | 7 ++--- .../plugins/ml/server/routes/job_service.js | 31 ++++++------------- 4 files changed, 21 insertions(+), 41 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/jobs.js b/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/jobs.js index 7e0058469a196..06f5db0852faa 100644 --- a/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/jobs.js +++ b/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/jobs.js @@ -207,14 +207,7 @@ export const jobs = { }); }, - categorizationFieldExamples( - indexPatternTitle, - query, - size, - field, - start, - end, - ) { + categorizationFieldExamples(indexPatternTitle, query, size, field, start, end) { return http({ url: `${basePath}/jobs/categorization_field_examples`, method: 'POST', @@ -225,7 +218,7 @@ export const jobs = { field, start, end, - } + }, }); }, @@ -236,7 +229,7 @@ export const jobs = { data: { jobId, count, - } + }, }); }, }; diff --git a/x-pack/legacy/plugins/ml/server/client/elasticsearch_ml.js b/x-pack/legacy/plugins/ml/server/client/elasticsearch_ml.js index 7b713b269c31a..cf13d329182ba 100644 --- a/x-pack/legacy/plugins/ml/server/client/elasticsearch_ml.js +++ b/x-pack/legacy/plugins/ml/server/client/elasticsearch_ml.js @@ -765,18 +765,17 @@ export const elasticsearchJsPlugin = (Client, config, components) => { categoryId: { type: 'string', }, - } + }, }, { fmt: '/_xpack/ml/anomaly_detectors/<%=jobId%>/results/categories', req: { jobId: { type: 'string', - } - } - } + }, + }, + }, ], - method: 'GET' + method: 'GET', }); - }; diff --git a/x-pack/legacy/plugins/ml/server/routes/anomaly_detectors.js b/x-pack/legacy/plugins/ml/server/routes/anomaly_detectors.js index eb1c01f8a7b06..54d447c288151 100644 --- a/x-pack/legacy/plugins/ml/server/routes/anomaly_detectors.js +++ b/x-pack/legacy/plugins/ml/server/routes/anomaly_detectors.js @@ -189,15 +189,14 @@ export function jobRoutes({ commonRouteConfig, elasticsearchPlugin, route }) { const callWithRequest = callWithRequestFactory(elasticsearchPlugin, request); const options = { jobId: request.params.jobId, - categoryId: request.params.categoryId + categoryId: request.params.categoryId, }; return callWithRequest('ml.categories', options) .then(resp => reply(resp)) .catch(resp => reply(wrapError(resp))); }, config: { - ...commonRouteConfig - } + ...commonRouteConfig, + }, }); - } diff --git a/x-pack/legacy/plugins/ml/server/routes/job_service.js b/x-pack/legacy/plugins/ml/server/routes/job_service.js index 0505e9841d179..795aeaa5d14ca 100644 --- a/x-pack/legacy/plugins/ml/server/routes/job_service.js +++ b/x-pack/legacy/plugins/ml/server/routes/job_service.js @@ -277,20 +277,14 @@ export function jobServiceRoutes({ commonRouteConfig, elasticsearchPlugin, route handler(request) { const callWithRequest = callWithRequestFactory(elasticsearchPlugin, request); const { categorizationExamples } = jobServiceProvider(callWithRequest); - const { - indexPatternTitle, - query, - size, - field, - start, - end, - } = request.payload; - return categorizationExamples(indexPatternTitle, query, size, field, start, end) - .catch(resp => wrapError(resp)); + const { indexPatternTitle, query, size, field, start, end } = request.payload; + return categorizationExamples(indexPatternTitle, query, size, field, start, end).catch(resp => + wrapError(resp) + ); }, config: { - ...commonRouteConfig - } + ...commonRouteConfig, + }, }); route({ @@ -299,16 +293,11 @@ export function jobServiceRoutes({ commonRouteConfig, elasticsearchPlugin, route handler(request) { const callWithRequest = callWithRequestFactory(elasticsearchPlugin, request); const { topCategories } = jobServiceProvider(callWithRequest); - const { - jobId, - count, - } = request.payload; - return topCategories(jobId, count) - .catch(resp => wrapError(resp)); + const { jobId, count } = request.payload; + return topCategories(jobId, count).catch(resp => wrapError(resp)); }, config: { - ...commonRouteConfig - } + ...commonRouteConfig, + }, }); - } From caadec2a002dc1270bbdfeae058668f70bd5f00f Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Fri, 20 Dec 2019 13:21:47 +0000 Subject: [PATCH 03/29] adding basic category field validation --- .../job_creator/categorization_job_creator.ts | 20 +++ .../common/job_validator/job_validator.ts | 25 ++- .../categorization_examples_loader.ts | 69 ++++++++ .../new_job/common/results_loader/index.ts | 1 + .../examples_valid_callout.tsx | 38 +++++ .../categorization_view/field_examples.tsx | 6 +- .../categorization_view/metric_selection.tsx | 148 +++--------------- .../pick_fields_step/pick_fields.tsx | 4 +- .../services/ml_api_service/index.d.ts | 6 +- .../services/ml_api_service/jobs.js | 13 +- .../application/services/ml_server_info.ts | 7 + .../ml/public/application/util/index_utils.ts | 8 +- .../job_service/new_job/categorization.ts | 99 +++++++++--- .../plugins/ml/server/routes/job_service.js | 26 ++- 14 files changed, 307 insertions(+), 163 deletions(-) create mode 100644 x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts create mode 100644 x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts index aa367f0e0879b..c875ef71e93b1 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts @@ -12,10 +12,14 @@ import { Job, Datafeed, Detector } from './configs'; import { createBasicDetector } from './util/default_configs'; import { JOB_TYPE, CREATED_BY_LABEL } from '../../../../../../common/constants/new_job'; import { getRichDetectors } from './util/general'; +import { CategorizationExamplesLoader, CategoryExample } from '../results_loader'; export class CategorizationJobCreator extends JobCreator { protected _type: JOB_TYPE = JOB_TYPE.CATEGORIZATION; private _createDetector: () => void = () => {}; + private _examplesLoader: CategorizationExamplesLoader; + private _categoryFieldExamples: CategoryExample[] = []; + private _categoryFieldValid: number = 0; constructor( indexPattern: IndexPattern, @@ -24,6 +28,7 @@ export class CategorizationJobCreator extends JobCreator { ) { super(indexPattern, savedSearch, query); this.createdBy = CREATED_BY_LABEL.CATEGORIZATION; + this._examplesLoader = new CategorizationExamplesLoader(this, indexPattern, query); } public setDefaultDetectorProperties(count: Aggregation | null, eventRate: Field | null) { @@ -54,6 +59,21 @@ export class CategorizationJobCreator extends JobCreator { return this._job_config.analysis_config.categorization_field_name || null; } + public async loadCategorizationFieldExamples() { + const { valid, examples } = await this._examplesLoader.loadExamples(); + this._categoryFieldExamples = examples; + this._categoryFieldValid = valid; + return { valid, examples }; + } + + public get categoryFieldExamples() { + return this._categoryFieldExamples; + } + + public get categoryFieldValid() { + return this._categoryFieldValid; + } + public cloneFromExistingJob(job: Job, datafeed: Datafeed) { this._overrideConfigs(job, datafeed); this.createdBy = CREATED_BY_LABEL.CATEGORIZATION; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts index 3c1f767aeaf9c..71a0573e1e6f9 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts @@ -12,7 +12,7 @@ import { basicDatafeedValidation, } from '../../../../../../common/util/job_utils'; import { getNewJobLimits } from '../../../../services/ml_server_info'; -import { JobCreator, JobCreatorType } from '../job_creator'; +import { JobCreator, JobCreatorType, isCategorizationJobCreator } from '../job_creator'; import { populateValidationMessages, checkForExistingJobAndGroupIds } from './util'; import { ExistingJobsAndGroups } from '../../../../services/job_service'; import { cardinalityValidator, CardinalityValidatorResult } from './validators'; @@ -51,6 +51,10 @@ export interface BasicValidations { scrollSize: Validation; } +export interface AdvancedValidations { + categorizationFieldValid: Validation; +} + export class JobValidator { private _jobCreator: JobCreatorType; private _validationSummary: ValidationSummary; @@ -71,6 +75,9 @@ export class JobValidator { frequency: { valid: true }, scrollSize: { valid: true }, }; + private _advancedValidations: AdvancedValidations = { + categorizationFieldValid: { valid: true }, + }; private _validating: boolean = false; private _basicValidationResult$ = new ReplaySubject(2); @@ -141,6 +148,7 @@ export class JobValidator { this._lastDatafeedConfig = formattedDatafeedConfig; this._validateTimeout = setTimeout(() => { this._runBasicValidation(); + this._runAdvancedValidation(); this._jobCreatorSubject$.next(this._jobCreator); @@ -195,6 +203,13 @@ export class JobValidator { this._basicValidationResult$.next(this._basicValidations); } + private _runAdvancedValidation() { + if (isCategorizationJobCreator(this._jobCreator)) { + this._advancedValidations.categorizationFieldValid.valid = + this._jobCreator.categoryFieldValid > 0.2; + } + } + private _isOverallBasicValid() { return Object.values(this._basicValidations).some(v => v.valid === false) === false; } @@ -246,4 +261,12 @@ export class JobValidator { public get validating(): boolean { return this._validating; } + + public get categorizationField() { + return this._advancedValidations.categorizationFieldValid.valid; + } + + public set categorizationField(valid: boolean) { + this._advancedValidations.categorizationFieldValid.valid = valid; + } } diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts new file mode 100644 index 0000000000000..5c534f6b1e6a4 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IndexPattern } from 'ui/index_patterns'; +import { IndexPatternTitle } from '../../../../../../common/types/kibana'; +import { CategorizationJobCreator } from '../job_creator'; +import { ml } from '../../../../services/ml_api_service'; +import { getNewJobDefaults, CategorizationAnalyzer } from '../../../../services/ml_server_info'; + +const NUM_OF_EXAMPLES = 5; + +export interface Token { + token: string; + start_offset: number; + end_offset: number; + type: string; + position: number; +} + +export interface CategoryExample { + text: string; + tokens: Token[]; +} + +export class CategorizationExamplesLoader { + private _jobCreator: CategorizationJobCreator; + private _indexPatternTitle: IndexPatternTitle = ''; + private _timeFieldName: string = ''; + private _query: object = {}; + private _analyzer: CategorizationAnalyzer; + + constructor(jobCreator: CategorizationJobCreator, indexPattern: IndexPattern, query: object) { + this._jobCreator = jobCreator; + this._indexPatternTitle = indexPattern.title; + this._query = query; + + if (typeof indexPattern.timeFieldName === 'string') { + this._timeFieldName = indexPattern.timeFieldName; + } + + const { anomaly_detectors: anomalyDetectors } = getNewJobDefaults(); + this._analyzer = anomalyDetectors.categorization_analyzer!; + } + + async loadExamples() { + const categorizationFieldName = this._jobCreator.categorizationFieldName; + if (categorizationFieldName === null) { + return { valid: 0, examples: [] }; + } + + const start = Math.floor( + this._jobCreator.start + (this._jobCreator.end - this._jobCreator.start) / 2 + ); + const resp = await ml.jobs.categorizationFieldExamples( + this._indexPatternTitle, + this._query, + NUM_OF_EXAMPLES, + categorizationFieldName, + this._timeFieldName, + start, + 0, + this._analyzer + ); + return resp; + } +} diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/index.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/index.ts index ef0b05f73fa31..724c62f22e469 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/index.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/index.ts @@ -5,3 +5,4 @@ */ export { ResultsLoader, Results, ModelItem, Anomaly } from './results_loader'; +export { CategorizationExamplesLoader, CategoryExample } from './categorization_examples_loader'; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx new file mode 100644 index 0000000000000..b12137d6d5ee0 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { Fragment, FC, useEffect, useState } from 'react'; +import { EuiCallOut } from '@elastic/eui'; + +interface Props { + examplesValid: number; +} + +export const ExamplesValidCallout: FC = ({ examplesValid }) => { + function getCallOut() { + if (examplesValid < 0.2) { + return ( + + {Math.floor(examplesValid * 100)}% of field values tested contain valid tokens + + ); + } else if (examplesValid < 0.75) { + return ( + + {Math.floor(examplesValid * 100)}% of field values tested contain valid tokens + + ); + } else { + return ( + + {Math.floor(examplesValid * 100)}% of field values tested contain valid tokens + + ); + } + } + + return {getCallOut()}; +}; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx index b2a47830fc54f..bc13f6f911731 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx @@ -8,7 +8,7 @@ import React, { FC } from 'react'; import { EuiBasicTable, EuiText } from '@elastic/eui'; interface Props { - fieldExamples: Cat[]; + fieldExamples: Cat[] | null; } interface Token { @@ -24,10 +24,10 @@ interface Cat { tokens: Token[]; } -const TOKEN_HIGHLIGHT_COLOR = '#e9f2ff'; +const TOKEN_HIGHLIGHT_COLOR = '#b0ccf7'; export const FieldExamples: FC = ({ fieldExamples }) => { - if (fieldExamples.length === 0) { + if (fieldExamples === null || fieldExamples.length === 0) { return null; } diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx index 1d22f697aaa69..7285d79462592 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx @@ -6,77 +6,34 @@ import React, { Fragment, FC, useContext, useEffect, useState } from 'react'; -import { EuiCode } from '@elastic/eui'; import { JobCreatorContext } from '../../../job_creator_context'; import { CategorizationJobCreator } from '../../../../../common/job_creator'; -import { ml } from '../../../../../../../services/ml_api_service'; -// import { LineChartData } from '../../../../../common/chart_loader'; -// import { DropDownProps, createLabel } from '../agg_select'; -import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service'; -import { mlCategory } from '../../../../../../../../../common/types/fields'; +// import { ml } from '../../../../../../../services/ml_api_service'; import { CategorizationField } from '../categorization_field'; -import { useKibanaContext } from '../../../../../../../contexts/kibana'; +// import { useKibanaContext } from '../../../../../../../contexts/kibana'; import { FieldExamples } from './field_examples'; +import { ExamplesValidCallout } from './examples_valid_callout'; +import { CategoryExample } from '../../../../../common/results_loader'; +// import { getNewJobDefaults } from '../../../../../../../services/ml_server_info'; interface Props { setIsValid: (na: boolean) => void; } -interface Token { - token: string; - start_offset: number; - end_offset: number; - type: string; - position: number; -} - -interface Cat { - text: string; - tokens: Token[]; -} - -const NUM_OF_EXAMPLES = 5; - -// const DTR_IDX = 0; - export const CategorizationDetectors: FC = ({ setIsValid }) => { - const kibanaContext = useKibanaContext(); - const { - jobCreator: jc, - jobCreatorUpdate, - jobCreatorUpdated, - // chartLoader, - // chartInterval, - } = useContext(JobCreatorContext); + const { jobCreator: jc, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext); const jobCreator = jc as CategorizationJobCreator; - // const { fields } = newJobCapsService; - // const [selectedOptions, setSelectedOptions] = useState( - // jobCreator.aggFieldPair !== null ? [{ label: createLabel(jobCreator.aggFieldPair) }] : [] - // ); - // const [aggFieldPair, setAggFieldPair] = useState(jobCreator.aggFieldPair); - // const [lineChartsData, setLineChartData] = useState({}); const [loadingData, setLoadingData] = useState(false); const [start, setStart] = useState(jobCreator.start); const [end, setEnd] = useState(jobCreator.end); - const [fieldExamples, setFieldExamples] = useState([]); - const [tokens, setTokens] = useState([]); - // const [bucketSpanMs, setBucketSpanMs] = useState(jobCreator.bucketSpanMs); + const [fieldExamples, setFieldExamples] = useState(null); + const [examplesValid, setExamplesValid] = useState(0); const [categorizationFieldName, setCategorizationFieldName] = useState( jobCreator.categorizationFieldName ); - // useEffect(() => { - // if (jobCreator.detectors.length !== 1) { - // const agg = newJobCapsService.getAggById('count'); - // if (agg !== null) { - // jobCreator.setDetector(agg, mlCategory); - // jobCreatorUpdate(); - // } - // } - // }, []); - useEffect(() => { if (jobCreator.categorizationFieldName !== categorizationFieldName) { jobCreator.categorizationFieldName = categorizationFieldName; @@ -85,27 +42,6 @@ export const CategorizationDetectors: FC = ({ setIsValid }) => { loadFieldExamples(); }, [categorizationFieldName]); - // function detectorChangeHandler(selectedOptionsIn: DropDownLabel[]) { - // setSelectedOptions(selectedOptionsIn); - // if (selectedOptionsIn.length) { - // const option = selectedOptionsIn[0]; - // if (typeof option !== 'undefined') { - // setAggFieldPair({ agg: option.agg, field: option.field }); - // } else { - // setAggFieldPair(null); - // } - // } - // } - - // useEffect(() => { - // if (aggFieldPair !== null) { - // jobCreator.setDetector(aggFieldPair.agg, aggFieldPair.field); - // jobCreatorUpdate(); - // loadChart(); - // setIsValid(aggFieldPair !== null); - // } - // }, [aggFieldPair]); - useEffect(() => { if (jobCreator.start !== start || jobCreator.end !== end) { setStart(jobCreator.start); @@ -120,74 +56,26 @@ export const CategorizationDetectors: FC = ({ setIsValid }) => { async function loadFieldExamples() { if (categorizationFieldName !== null) { setLoadingData(true); - const resp = await ml.jobs.categorizationFieldExamples( - kibanaContext.currentIndexPattern.title, - kibanaContext.combinedQuery, - NUM_OF_EXAMPLES, - categorizationFieldName, - 0, - 0 - ); - // const examples = resp.map(r => r.text); - // const tkns = resp.map(r => r.tokens); - // console.log(resp); - setFieldExamples(resp); - // const resp = await ml.esSearch({ - // index: kibanaContext.currentIndexPattern.title, - // size: 10, - // body: { - // _source: categorizationFieldName, - // query: kibanaContext.combinedQuery, - // }, - // }); - // try { - // const cs = getChartSettings(jobCreator, chartInterval); - // const resp: LineChartData = await chartLoader.loadLineCharts( - // jobCreator.start, - // jobCreator.end, - // [aggFieldPair], - // null, - // null, - // cs.intervalMs - // ); - // if (resp[DTR_IDX] !== undefined) { - // setLineChartData(resp); - // } - // } catch (error) { - // mlMessageBarService.notify.error(error); - // setLineChartData({}); - // } + const { valid, examples } = await jobCreator.loadCategorizationFieldExamples(); + setFieldExamples(examples); + setExamplesValid(valid); setLoadingData(false); } else { - setFieldExamples([]); + setFieldExamples(null); + setExamplesValid(0); } setIsValid(categorizationFieldName !== null); } + useEffect(() => { + jobCreatorUpdate(); + }, [examplesValid]); + return ( - - {/* */} - {/* {(lineChartsData[DTR_IDX] !== undefined || loadingData === true) && ( - - - - )} */} + {fieldExamples !== null && } ); }; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/pick_fields.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/pick_fields.tsx index c5830ccac82f6..bfec49678bc34 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/pick_fields.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/pick_fields.tsx @@ -31,7 +31,9 @@ export const PickFieldsStep: FC = ({ setCurrentStep, isCurrentStep }) (jobCreator.type === JOB_TYPE.ADVANCED && jobValidator.modelMemoryLimit.valid)) && jobValidator.bucketSpan.valid && jobValidator.duplicateDetectors.valid && - jobValidator.validating === false; + jobValidator.validating === false && + (jobCreator.type !== JOB_TYPE.CATEGORIZATION || + (jobCreator.type === JOB_TYPE.CATEGORIZATION && jobValidator.categorizationField)); setNextActive(active); }, [jobValidatorUpdated]); diff --git a/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/index.d.ts b/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/index.d.ts index c9e1d61b0ac19..4756114024289 100644 --- a/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/index.d.ts +++ b/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/index.d.ts @@ -176,9 +176,11 @@ declare interface Ml { query: object, size: number, field: string, + timeField: string | undefined, start: number, - end: number - ): Promise; + end: number, + analyzer: any + ): Promise<{ valid: number; examples: any[] }>; topCategories(jobId: string, count: number): Promise<{ total: number; categories: any[] }>; }; diff --git a/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/jobs.js b/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/jobs.js index 06f5db0852faa..05d98dc1a1e64 100644 --- a/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/jobs.js +++ b/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/jobs.js @@ -207,7 +207,16 @@ export const jobs = { }); }, - categorizationFieldExamples(indexPatternTitle, query, size, field, start, end) { + categorizationFieldExamples( + indexPatternTitle, + query, + size, + field, + timeField, + start, + end, + analyzer + ) { return http({ url: `${basePath}/jobs/categorization_field_examples`, method: 'POST', @@ -216,8 +225,10 @@ export const jobs = { query, size, field, + timeField, start, end, + analyzer, }, }); }, diff --git a/x-pack/legacy/plugins/ml/public/application/services/ml_server_info.ts b/x-pack/legacy/plugins/ml/public/application/services/ml_server_info.ts index 95d670eda8a4f..0b31187ecae59 100644 --- a/x-pack/legacy/plugins/ml/public/application/services/ml_server_info.ts +++ b/x-pack/legacy/plugins/ml/public/application/services/ml_server_info.ts @@ -11,10 +11,17 @@ export interface MlServerDefaults { categorization_examples_limit?: number; model_memory_limit?: string; model_snapshot_retention_days?: number; + categorization_analyzer?: CategorizationAnalyzer; }; datafeeds: { scroll_size?: number }; } +export interface CategorizationAnalyzer { + char_filter?: any[]; + tokenizer: string; + filter: any[]; +} + export interface MlServerLimits { max_model_memory_limit?: string; } diff --git a/x-pack/legacy/plugins/ml/public/application/util/index_utils.ts b/x-pack/legacy/plugins/ml/public/application/util/index_utils.ts index 2b8838c04cf69..2e176b0044314 100644 --- a/x-pack/legacy/plugins/ml/public/application/util/index_utils.ts +++ b/x-pack/legacy/plugins/ml/public/application/util/index_utils.ts @@ -8,7 +8,11 @@ import { toastNotifications } from 'ui/notify'; import { i18n } from '@kbn/i18n'; import chrome from 'ui/chrome'; import { Query } from 'src/plugins/data/public'; -import { IndexPattern, IndexPatternsContract } from '../../../../../../../src/plugins/data/public'; +import { + IndexPattern, + IIndexPattern, + IndexPatternsContract, +} from '../../../../../../../src/plugins/data/public'; import { IndexPatternSavedObject, SavedSearchSavedObject } from '../../../common/types/kibana'; let indexPatternCache: IndexPatternSavedObject[] = []; @@ -71,7 +75,7 @@ export function getIndexPatternIdFromName(name: string) { } export async function getIndexPatternAndSavedSearch(savedSearchId: string) { - const resp: { savedSearch: SavedSearchSavedObject | null; indexPattern: IndexPattern | null } = { + const resp: { savedSearch: SavedSearchSavedObject | null; indexPattern: IIndexPattern | null } = { savedSearch: null, indexPattern: null, }; diff --git a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts index 26c31f8e05ab2..3ea02eb134c7f 100644 --- a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts +++ b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts @@ -26,26 +26,52 @@ interface Category { grok_pattern: string; } -const testCat: Category = { - job_id: 'dasdasdd', - category_id: 100, - terms: 'HEAD projects xdotool HTTP IrssiUrlLog', - regex: '.*?HEAD.+?projects.+?xdotool.+?HTTP.+?IrssiUrlLog.*', - max_matching_length: 119, - examples: [''], - grok_pattern: - '.*?%{IP:ipaddress}.+?%{HTTPDATE:timestamp}.+?HEAD.+?%{PATH:path}.*?projects.*?%{PATH:path2}.*?xdotool.*?%{PATH:path3}.+?HTTP.*?%{PATH:path4}.*?%{QUOTEDSTRING:field}.+?%{QUOTEDSTRING:field2}.*?IrssiUrlLog.*?%{PATH:path5}.*', -}; +// const testCat: Category = { +// job_id: 'dasdasdd', +// category_id: 100, +// terms: 'HEAD projects xdotool HTTP IrssiUrlLog', +// regex: '.*?HEAD.+?projects.+?xdotool.+?HTTP.+?IrssiUrlLog.*', +// max_matching_length: 119, +// examples: [''], +// grok_pattern: +// '.*?%{IP:ipaddress}.+?%{HTTPDATE:timestamp}.+?HEAD.+?%{PATH:path}.*?projects.*?%{PATH:path2}.*?xdotool.*?%{PATH:path3}.+?HTTP.*?%{PATH:path4}.*?%{QUOTEDSTRING:field}.+?%{QUOTEDSTRING:field2}.*?IrssiUrlLog.*?%{PATH:path5}.*', +// }; export function categorizationExamplesProvider(callWithRequest: callWithRequestType) { async function categorizationExamples( indexPatternTitle: string, - query: object, + query: any, size: number, categorizationFieldName: string, + timeField: string | undefined, start: number, - end: number + end: number, + analyzer?: any ) { + if (timeField !== undefined) { + const range = { + range: { + [timeField]: { + gte: start, + format: 'epoch_millis', + }, + }, + }; + + if (query.bool === undefined) { + query.bool = {}; + } + if (query.bool.filter === undefined) { + query.bool.filter = range; + } else { + if (Array.isArray(query.bool.filter)) { + query.bool.filter.push(range); + } else { + query.bool.filter.range = range; + } + } + } + const results = await callWithRequest('search', { index: indexPatternTitle, size, @@ -62,13 +88,13 @@ export function categorizationExamplesProvider(callWithRequest: callWithRequestT try { const { tokens: tempTokens } = await callWithRequest('indices.analyze', { body: { - analyzer: 'standard', + ...getAnalyzer(analyzer), text: examples, }, }); tokens = tempTokens; } catch (error) { - // do somehting with this error + // do something with this error } const lengths = examples.map(e => e.length); @@ -77,11 +103,8 @@ export function categorizationExamplesProvider(callWithRequest: callWithRequestT const tokensPerExample: Token[][] = lengths.map(l => []); tokens.forEach((t, i) => { for (let g = 0; g < sumLengths.length; g++) { - if (t.token === 'dashboard2') { - const d = 0; - } if (t.start_offset <= sumLengths[g]) { - const offset = g > 0 ? sumLengths[g - 1] + g : 0; + const offset = g > 0 ? sumLengths[g - 1] : 0; tokensPerExample[g].push({ ...t, start_offset: t.start_offset - offset, @@ -95,6 +118,45 @@ export function categorizationExamplesProvider(callWithRequest: callWithRequestT return examples.map((e, i) => ({ text: e, tokens: tokensPerExample[i] })); } + function getAnalyzer(analyzer: any) { + if (typeof analyzer === 'object' && analyzer.tokenizer !== undefined) { + return analyzer; + } else { + return { analyzer: 'standard' }; + } + } + + async function validateCategoryExamples( + indexPatternTitle: string, + query: any, + size: number, + categorizationFieldName: string, + timeField: string | undefined, + start: number, + end: number, + analyzer?: any + ) { + const MULTIPLIER = 20; + const examples = await categorizationExamples( + indexPatternTitle, + query, + size * MULTIPLIER, + categorizationFieldName, + timeField, + start, + end, + analyzer + ); + + const sortedExamples = examples.sort((a, b) => b.tokens.length - a.tokens.length); + const validExamples = sortedExamples.filter(e => e.tokens.length > 1); + + return { + valid: sortedExamples.length === 0 ? 0 : validExamples.length / sortedExamples.length, + examples: examples.filter((e, i) => i / MULTIPLIER - Math.floor(i / MULTIPLIER) === 0), + }; + } + async function getCategories( jobId: string, categoryId: number @@ -214,6 +276,7 @@ export function categorizationExamplesProvider(callWithRequest: callWithRequestT return { categorizationExamples, + validateCategoryExamples, topCategories, }; } diff --git a/x-pack/legacy/plugins/ml/server/routes/job_service.js b/x-pack/legacy/plugins/ml/server/routes/job_service.js index 795aeaa5d14ca..a83b4fa403f65 100644 --- a/x-pack/legacy/plugins/ml/server/routes/job_service.js +++ b/x-pack/legacy/plugins/ml/server/routes/job_service.js @@ -276,11 +276,27 @@ export function jobServiceRoutes({ commonRouteConfig, elasticsearchPlugin, route path: '/api/ml/jobs/categorization_field_examples', handler(request) { const callWithRequest = callWithRequestFactory(elasticsearchPlugin, request); - const { categorizationExamples } = jobServiceProvider(callWithRequest); - const { indexPatternTitle, query, size, field, start, end } = request.payload; - return categorizationExamples(indexPatternTitle, query, size, field, start, end).catch(resp => - wrapError(resp) - ); + const { validateCategoryExamples } = jobServiceProvider(callWithRequest); + const { + indexPatternTitle, + timeField, + query, + size, + field, + start, + end, + analyzer, + } = request.payload; + return validateCategoryExamples( + indexPatternTitle, + query, + size, + field, + timeField, + start, + end, + analyzer + ).catch(resp => wrapError(resp)); }, config: { ...commonRouteConfig, From 4b77ad70f0fd265ba25f406dfaab2160fb12f920 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Fri, 20 Dec 2019 15:41:52 +0000 Subject: [PATCH 04/29] adding rare or count selection --- .../job_creator/categorization_job_creator.ts | 57 +++++++++++++++---- .../categorization_examples_loader.ts | 2 +- .../categorization_detector.tsx | 40 +++++++++++++ .../categorization_detector_select.tsx | 43 ++++++++++++++ .../categorization_detector/description.tsx | 35 ++++++++++++ .../categorization_detector/index.ts | 6 ++ .../categorization_view/metric_selection.tsx | 2 + .../jobs/new_job/pages/new_job/page.tsx | 12 +++- 8 files changed, 181 insertions(+), 16 deletions(-) create mode 100644 x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/categorization_detector.tsx create mode 100644 x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/categorization_detector_select.tsx create mode 100644 x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/description.tsx create mode 100644 x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/index.ts diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts index c875ef71e93b1..d1a41f046ec1e 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts @@ -11,15 +11,21 @@ import { Field, Aggregation, mlCategory } from '../../../../../../common/types/f import { Job, Datafeed, Detector } from './configs'; import { createBasicDetector } from './util/default_configs'; import { JOB_TYPE, CREATED_BY_LABEL } from '../../../../../../common/constants/new_job'; +import { ML_JOB_AGGREGATION } from '../../../../../../common/constants/aggregation_types'; import { getRichDetectors } from './util/general'; import { CategorizationExamplesLoader, CategoryExample } from '../results_loader'; +// type DETECTOR_TYPE = ML_JOB_AGGREGATION.COUNT | ML_JOB_AGGREGATION.RARE; + export class CategorizationJobCreator extends JobCreator { protected _type: JOB_TYPE = JOB_TYPE.CATEGORIZATION; - private _createDetector: () => void = () => {}; + private _createCountDetector: () => void = () => {}; + private _createRareDetector: () => void = () => {}; private _examplesLoader: CategorizationExamplesLoader; private _categoryFieldExamples: CategoryExample[] = []; private _categoryFieldValid: number = 0; + private _detectorType: ML_JOB_AGGREGATION.COUNT | ML_JOB_AGGREGATION.RARE = + ML_JOB_AGGREGATION.COUNT; constructor( indexPattern: IndexPattern, @@ -31,24 +37,44 @@ export class CategorizationJobCreator extends JobCreator { this._examplesLoader = new CategorizationExamplesLoader(this, indexPattern, query); } - public setDefaultDetectorProperties(count: Aggregation | null, eventRate: Field | null) { - if (count === null || eventRate === null) { + public setDefaultDetectorProperties( + count: Aggregation | null, + rare: Aggregation | null, + eventRate: Field | null + ) { + if (count === null || rare === null || eventRate === null) { return; } - const dtr: Detector = createBasicDetector(count, eventRate); - dtr.by_field_name = mlCategory.id; - this._createDetector = () => { - this._addDetector(dtr, count, mlCategory); + + this._createCountDetector = () => { + this._createDetector(count, eventRate); + }; + this._createRareDetector = () => { + this._createDetector(rare, eventRate); }; } - public set categorizationFieldName(fieldName: string | null) { + private _createDetector(agg: Aggregation, field: Field) { + const dtr: Detector = createBasicDetector(agg, field); + dtr.by_field_name = mlCategory.id; + this._addDetector(dtr, agg, mlCategory); + } + + public setDetectorType(type: ML_JOB_AGGREGATION.COUNT | ML_JOB_AGGREGATION.RARE) { + this._detectorType = type; this.removeAllDetectors(); this.removeAllInfluencers(); + if (type === ML_JOB_AGGREGATION.COUNT) { + this._createCountDetector(); + } else { + this._createRareDetector(); + } + } + public set categorizationFieldName(fieldName: string | null) { if (fieldName !== null) { this._job_config.analysis_config.categorization_field_name = fieldName; - this._createDetector(); + this.setDetectorType(this._detectorType); this.addInfluencer(mlCategory.id); } else { delete this._job_config.analysis_config.categorization_field_name; @@ -74,17 +100,24 @@ export class CategorizationJobCreator extends JobCreator { return this._categoryFieldValid; } + public get selectedDetectorType() { + return this._detectorType; + } + public cloneFromExistingJob(job: Job, datafeed: Datafeed) { this._overrideConfigs(job, datafeed); this.createdBy = CREATED_BY_LABEL.CATEGORIZATION; const detectors = getRichDetectors(job, datafeed, this.scriptFields, false); - this.removeAllDetectors(); + // this.removeAllDetectors(); const dtr = detectors[0]; if (detectors.length && dtr.agg !== null && dtr.field !== null) { - this.setDefaultDetectorProperties(dtr.agg, dtr.field); - this._createDetector(); + this._detectorType = + dtr.agg.id === ML_JOB_AGGREGATION.COUNT + ? ML_JOB_AGGREGATION.COUNT + : ML_JOB_AGGREGATION.RARE; + this.setDetectorType(this._detectorType); } } } diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts index 5c534f6b1e6a4..c68d783f19ad3 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts @@ -45,7 +45,7 @@ export class CategorizationExamplesLoader { this._analyzer = anomalyDetectors.categorization_analyzer!; } - async loadExamples() { + public async loadExamples() { const categorizationFieldName = this._jobCreator.categorizationFieldName; if (categorizationFieldName === null) { return { valid: 0, examples: [] }; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/categorization_detector.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/categorization_detector.tsx new file mode 100644 index 0000000000000..18977159cc54b --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/categorization_detector.tsx @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, useContext, useEffect, useState } from 'react'; + +import { CategorizationDetectorSelect } from './categorization_detector_select'; +import { JobCreatorContext } from '../../../job_creator_context'; +import { CategorizationJobCreator } from '../../../../../common/job_creator'; +import { Description } from './description'; + +export const CategorizationDetector: FC = () => { + const { jobCreator: jc, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext); + const jobCreator = jc as CategorizationJobCreator; + const [categorizationDetectorType, setCategorizationDetectorType] = useState( + jobCreator.selectedDetectorType + ); + + useEffect(() => { + if (categorizationDetectorType !== jobCreator.selectedDetectorType) { + jobCreator.setDetectorType(categorizationDetectorType); + jobCreatorUpdate(); + } + }, [categorizationDetectorType]); + + useEffect(() => { + setCategorizationDetectorType(jobCreator.selectedDetectorType); + }, [jobCreatorUpdated]); + + return ( + + + + ); +}; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/categorization_detector_select.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/categorization_detector_select.tsx new file mode 100644 index 0000000000000..2c7998215a486 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/categorization_detector_select.tsx @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC } from 'react'; +import { EuiComboBox, EuiComboBoxOptionProps } from '@elastic/eui'; + +import { ML_JOB_AGGREGATION } from '../../../../../../../../../common/constants/aggregation_types'; + +interface Props { + changeHandler(i: ML_JOB_AGGREGATION.COUNT | ML_JOB_AGGREGATION.RARE): void; + selectedDetectorType: ML_JOB_AGGREGATION.COUNT | ML_JOB_AGGREGATION.RARE; +} + +export const CategorizationDetectorSelect: FC = ({ + changeHandler, + selectedDetectorType, +}) => { + const options: EuiComboBoxOptionProps[] = [ + { label: 'Count', value: ML_JOB_AGGREGATION.COUNT }, + { label: 'Rare', value: ML_JOB_AGGREGATION.RARE }, + ]; + + const selection = [options.find(o => o.value === selectedDetectorType) || options[0]]; + + function onChange(selectedOptions: EuiComboBoxOptionProps[]) { + const option = selectedOptions[0]; + changeHandler(option.value as ML_JOB_AGGREGATION.COUNT | ML_JOB_AGGREGATION.RARE); + } + + return ( + + ); +}; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/description.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/description.tsx new file mode 100644 index 0000000000000..5571977671d53 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/description.tsx @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { memo, FC } from 'react'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { EuiDescribedFormGroup, EuiFormRow } from '@elastic/eui'; + +export const Description: FC = memo(({ children }) => { + const title = i18n.translate( + 'xpack.ml.newJob.wizard.pickFieldsStep.categorizationDetectorSelect.title', + { + defaultMessage: 'Categorization detector', + } + ); + return ( + {title}} + description={ + + } + > + + <>{children} + + + ); +}); diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/index.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/index.ts new file mode 100644 index 0000000000000..6a13d86d0db9a --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/index.ts @@ -0,0 +1,6 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +export { CategorizationDetector } from './categorization_detector'; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx index 7285d79462592..1b72844d7fc6e 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx @@ -10,6 +10,7 @@ import { JobCreatorContext } from '../../../job_creator_context'; import { CategorizationJobCreator } from '../../../../../common/job_creator'; // import { ml } from '../../../../../../../services/ml_api_service'; import { CategorizationField } from '../categorization_field'; +import { CategorizationDetector } from '../categorization_detector'; // import { useKibanaContext } from '../../../../../../../contexts/kibana'; import { FieldExamples } from './field_examples'; import { ExamplesValidCallout } from './examples_valid_callout'; @@ -73,6 +74,7 @@ export const CategorizationDetectors: FC = ({ setIsValid }) => { return ( + {fieldExamples !== null && } diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx index 43f6ebd07fe8e..1804f97e93e71 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx @@ -127,9 +127,6 @@ export const Page: FC = ({ existingJobsAndGroups, jobType }) => { if (isCategorizationJobCreator(jobCreator)) { // categorization job will always use a count agg, so give it // to the job creator now - const count = newJobCapsService.getAggById('count'); - const eventRate = newJobCapsService.getFieldById(EVENT_RATE_FIELD_ID); - jobCreator.setDefaultDetectorProperties(count, eventRate); jobCreator.modelPlot = true; } @@ -137,6 +134,15 @@ export const Page: FC = ({ existingJobsAndGroups, jobType }) => { autoSetTimeRange = isAdvancedJobCreator(jobCreator); } + if (isCategorizationJobCreator(jobCreator)) { + // categorization job will always use a count agg, so give it + // to the job creator now + const count = newJobCapsService.getAggById('count'); + const rare = newJobCapsService.getAggById('rare'); + const eventRate = newJobCapsService.getFieldById(EVENT_RATE_FIELD_ID); + jobCreator.setDefaultDetectorProperties(count, rare, eventRate); + } + if (autoSetTimeRange && isAdvancedJobCreator(jobCreator)) { // for advanced jobs, load the full time range start and end times // so they can be used for job validation and bucket span estimation From 8d0795f953c31788a927306b8f56362d936d9835 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Fri, 20 Dec 2019 17:51:27 +0000 Subject: [PATCH 05/29] fixing types --- .../application/jobs/new_job/utils/new_job_utils.ts | 10 +++------- .../services/new_job_capabilities_service.ts | 3 +-- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.ts index d1e81c1c344de..0f19451b23263 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.ts @@ -4,12 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { - IndexPattern, - esQuery, - Query, - esKuery, -} from '../../../../../../../../../src/plugins/data/public'; +import { esQuery, Query, esKuery } from '../../../../../../../../../src/plugins/data/public'; +import { IIndexPattern } from '../../../../../../../../../src/plugins/data/common/index_patterns'; import { KibanaConfigTypeFix } from '../../../contexts/kibana'; import { SEARCH_QUERY_LANGUAGE } from '../../../../../common/constants/search'; import { SavedSearchSavedObject } from '../../../../../common/types/kibana'; @@ -19,7 +15,7 @@ import { getQueryFromSavedSearch } from '../../../util/index_utils'; export function createSearchItems( kibanaConfig: KibanaConfigTypeFix, - indexPattern: IndexPattern, + indexPattern: IIndexPattern, savedSearch: SavedSearchSavedObject | null ) { // query is only used by the data visualizer as it needs diff --git a/x-pack/legacy/plugins/ml/public/application/services/new_job_capabilities_service.ts b/x-pack/legacy/plugins/ml/public/application/services/new_job_capabilities_service.ts index d78c9298c6073..051973d35d8de 100644 --- a/x-pack/legacy/plugins/ml/public/application/services/new_job_capabilities_service.ts +++ b/x-pack/legacy/plugins/ml/public/application/services/new_job_capabilities_service.ts @@ -15,7 +15,6 @@ import { import { ES_FIELD_TYPES, IIndexPattern, - IndexPattern, IndexPatternsContract, } from '../../../../../../../src/plugins/data/public'; import { ml } from './ml_api_service'; @@ -31,7 +30,7 @@ export function loadNewJobCapabilities( return new Promise(async (resolve, reject) => { if (indexPatternId !== undefined) { // index pattern is being used - const indexPattern: IndexPattern = await indexPatterns.get(indexPatternId); + const indexPattern: IIndexPattern = await indexPatterns.get(indexPatternId); await newJobCapsService.initializeFromIndexPattern(indexPattern); resolve(newJobCapsService.newJobCaps); } else if (savedSearchId !== undefined) { From bf3dafc7a917bfd90df41b289c5ba30cc9989cc3 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Thu, 2 Jan 2020 16:20:50 +0000 Subject: [PATCH 06/29] category examples changes --- .../job_creator/categorization_job_creator.ts | 18 ++++++++++++++++ .../categorization_examples_loader.ts | 8 ++----- .../categorization_view/top_categories.tsx | 21 ++++++++++++------- .../jobs/new_job/pages/new_job/page.tsx | 4 ++++ .../job_service/new_job/categorization.ts | 9 ++++++-- 5 files changed, 44 insertions(+), 16 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts index d1a41f046ec1e..8f1a2dc0c86c7 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts @@ -14,8 +14,10 @@ import { JOB_TYPE, CREATED_BY_LABEL } from '../../../../../../common/constants/n import { ML_JOB_AGGREGATION } from '../../../../../../common/constants/aggregation_types'; import { getRichDetectors } from './util/general'; import { CategorizationExamplesLoader, CategoryExample } from '../results_loader'; +import { CategorizationAnalyzer } from '../../../../services/ml_server_info'; // type DETECTOR_TYPE = ML_JOB_AGGREGATION.COUNT | ML_JOB_AGGREGATION.RARE; +type CategorizationAnalyzerType = string | CategorizationAnalyzer | null; export class CategorizationJobCreator extends JobCreator { protected _type: JOB_TYPE = JOB_TYPE.CATEGORIZATION; @@ -26,6 +28,7 @@ export class CategorizationJobCreator extends JobCreator { private _categoryFieldValid: number = 0; private _detectorType: ML_JOB_AGGREGATION.COUNT | ML_JOB_AGGREGATION.RARE = ML_JOB_AGGREGATION.COUNT; + private _categorization_analyzer: CategorizationAnalyzerType = null; constructor( indexPattern: IndexPattern, @@ -78,6 +81,8 @@ export class CategorizationJobCreator extends JobCreator { this.addInfluencer(mlCategory.id); } else { delete this._job_config.analysis_config.categorization_field_name; + this._categoryFieldExamples = []; + this._categoryFieldValid = 0; } } @@ -104,6 +109,19 @@ export class CategorizationJobCreator extends JobCreator { return this._detectorType; } + public set categorizationAnalyzer(analyzer: CategorizationAnalyzerType) { + this._categorization_analyzer = analyzer; + if (analyzer !== null) { + this._job_config.analysis_config.categorization_analyzer = analyzer; + } else { + delete this._job_config.analysis_config.categorization_analyzer; + } + } + + public get categorizationAnalyzer() { + return this._categorization_analyzer; + } + public cloneFromExistingJob(job: Job, datafeed: Datafeed) { this._overrideConfigs(job, datafeed); this.createdBy = CREATED_BY_LABEL.CATEGORIZATION; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts index c68d783f19ad3..40dc99765f4c3 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts @@ -8,7 +8,6 @@ import { IndexPattern } from 'ui/index_patterns'; import { IndexPatternTitle } from '../../../../../../common/types/kibana'; import { CategorizationJobCreator } from '../job_creator'; import { ml } from '../../../../services/ml_api_service'; -import { getNewJobDefaults, CategorizationAnalyzer } from '../../../../services/ml_server_info'; const NUM_OF_EXAMPLES = 5; @@ -30,7 +29,6 @@ export class CategorizationExamplesLoader { private _indexPatternTitle: IndexPatternTitle = ''; private _timeFieldName: string = ''; private _query: object = {}; - private _analyzer: CategorizationAnalyzer; constructor(jobCreator: CategorizationJobCreator, indexPattern: IndexPattern, query: object) { this._jobCreator = jobCreator; @@ -40,12 +38,10 @@ export class CategorizationExamplesLoader { if (typeof indexPattern.timeFieldName === 'string') { this._timeFieldName = indexPattern.timeFieldName; } - - const { anomaly_detectors: anomalyDetectors } = getNewJobDefaults(); - this._analyzer = anomalyDetectors.categorization_analyzer!; } public async loadExamples() { + const analyzer = this._jobCreator.categorizationAnalyzer; const categorizationFieldName = this._jobCreator.categorizationFieldName; if (categorizationFieldName === null) { return { valid: 0, examples: [] }; @@ -62,7 +58,7 @@ export class CategorizationExamplesLoader { this._timeFieldName, start, 0, - this._analyzer + analyzer ); return resp; } diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx index d82c0c6d36454..9ac273334195d 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx @@ -31,7 +31,7 @@ export const TopCategories: FC = () => { const jobCreator = jc as CategorizationJobCreator; const [categories, setCategories] = useState([]); - const [tableRow, setTableRow] = useState>([]); + const [tableRow, setTableRow] = useState>([]); const [totalCategories, setTotalCategories] = useState(0); function setResultsWrapper(results: Results) { @@ -41,7 +41,12 @@ export const TopCategories: FC = () => { async function loadTopCats() { const results = await ml.jobs.topCategories(jobCreator.jobId, 5); setCategories(results.categories); - setTableRow(results.categories.map(c => ({ count: c.count, regex: c.category.regex }))); + setTableRow( + results.categories.map(c => ({ + count: c.count, + example: c.category.examples?.length ? c.category.examples[0] : '', + })) + ); setTotalCategories(results.total); // console.log(topCats.total); } @@ -59,18 +64,18 @@ export const TopCategories: FC = () => { field: 'count', name: 'count', width: '100px', - render: (regex: any) => ( + render: (count: any) => ( - {regex} + {count} ), }, { - field: 'regex', - name: 'Regex', - render: (regex: any) => ( + field: 'example', + name: 'Example', + render: (example: any) => ( - {regex} + {example} ), }, diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx index 1804f97e93e71..7307d271d8590 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx @@ -40,6 +40,7 @@ import { ExistingJobsAndGroups, mlJobService } from '../../../../services/job_se import { expandCombinedJobConfig } from '../../common/job_creator/configs'; import { newJobCapsService } from '../../../../services/new_job_capabilities_service'; import { EVENT_RATE_FIELD_ID } from '../../../../../../common/types/fields'; +import { getNewJobDefaults } from '../../../../services/ml_server_info'; const PAGE_WIDTH = 1200; // document.querySelector('.single-metric-job-container').width(); const BAR_TARGET = PAGE_WIDTH > 2000 ? 1000 : PAGE_WIDTH / 2; @@ -141,6 +142,9 @@ export const Page: FC = ({ existingJobsAndGroups, jobType }) => { const rare = newJobCapsService.getAggById('rare'); const eventRate = newJobCapsService.getFieldById(EVENT_RATE_FIELD_ID); jobCreator.setDefaultDetectorProperties(count, rare, eventRate); + + const { anomaly_detectors: anomalyDetectors } = getNewJobDefaults(); + jobCreator.categorizationAnalyzer = anomalyDetectors.categorization_analyzer!; } if (autoSetTimeRange && isAdvancedJobCreator(jobCreator)) { diff --git a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts index 3ea02eb134c7f..9d1ee2bc6a8c5 100644 --- a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts +++ b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts @@ -148,12 +148,17 @@ export function categorizationExamplesProvider(callWithRequest: callWithRequestT analyzer ); - const sortedExamples = examples.sort((a, b) => b.tokens.length - a.tokens.length); + const sortedExamples = examples + .map((e, i) => ({ ...e, origIndex: i })) + .sort((a, b) => b.tokens.length - a.tokens.length); const validExamples = sortedExamples.filter(e => e.tokens.length > 1); return { valid: sortedExamples.length === 0 ? 0 : validExamples.length / sortedExamples.length, - examples: examples.filter((e, i) => i / MULTIPLIER - Math.floor(i / MULTIPLIER) === 0), + examples: sortedExamples + .filter((e, i) => i / MULTIPLIER - Math.floor(i / MULTIPLIER) === 0) + .sort((a, b) => a.origIndex - b.origIndex) + .map(e => ({ text: e.text, tokens: e.tokens })), }; } From f2b47127ea4f7a94a45332b622294e79a9e8c080 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Fri, 3 Jan 2020 13:50:09 +0000 Subject: [PATCH 07/29] improving results search --- .../job_service/new_job/categorization.ts | 91 +++++++++---------- 1 file changed, 45 insertions(+), 46 deletions(-) diff --git a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts index 9d1ee2bc6a8c5..32467c97fdb05 100644 --- a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts +++ b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts @@ -16,9 +16,11 @@ interface Token { position: number; } +type CategoryId = number; + interface Category { job_id: string; - category_id: number; + category_id: CategoryId; terms: string; regex: string; max_matching_length: number; @@ -26,17 +28,6 @@ interface Category { grok_pattern: string; } -// const testCat: Category = { -// job_id: 'dasdasdd', -// category_id: 100, -// terms: 'HEAD projects xdotool HTTP IrssiUrlLog', -// regex: '.*?HEAD.+?projects.+?xdotool.+?HTTP.+?IrssiUrlLog.*', -// max_matching_length: 119, -// examples: [''], -// grok_pattern: -// '.*?%{IP:ipaddress}.+?%{HTTPDATE:timestamp}.+?HEAD.+?%{PATH:path}.*?projects.*?%{PATH:path2}.*?xdotool.*?%{PATH:path3}.+?HTTP.*?%{PATH:path4}.*?%{QUOTEDSTRING:field}.+?%{QUOTEDSTRING:field2}.*?IrssiUrlLog.*?%{PATH:path5}.*', -// }; - export function categorizationExamplesProvider(callWithRequest: callWithRequestType) { async function categorizationExamples( indexPatternTitle: string, @@ -162,13 +153,6 @@ export function categorizationExamplesProvider(callWithRequest: callWithRequestT }; } - async function getCategories( - jobId: string, - categoryId: number - ): Promise<{ categories: Category[] }> { - return callWithRequest('ml.categories', { jobId, categoryId }); - } - async function getTotalCategories(jobId: string): Promise<{ total: number }> { const totalResp = await callWithRequest('search', { index: ML_RESULTS_INDEX_PATTERN, @@ -183,31 +167,19 @@ export function categorizationExamplesProvider(callWithRequest: callWithRequestT }, }, { - term: { - result_type: 'model_plot', - }, - }, - { - term: { - by_field_name: 'mlcategory', + exists: { + field: 'category_id', }, }, ], }, }, - aggs: { - cat_count: { - cardinality: { - field: 'by_field_value', - }, - }, - }, }, }); - return totalResp?.aggregations?.cat_count?.value ?? 0; + return totalResp?.hits?.total?.value ?? 0; } - async function topCategories(jobId: string, numberOfCategories: number) { + async function getTopCategoryCounts(jobId: string, numberOfCategories: number) { const top = await callWithRequest('search', { index: ML_RESULTS_INDEX_PATTERN, size: 0, @@ -245,29 +217,56 @@ export function categorizationExamplesProvider(callWithRequest: callWithRequestT }); const catCounts: Array<{ - id: number; + id: CategoryId; count: number; }> = top.aggregations?.cat_count?.buckets.map((c: any) => ({ id: c.key, count: c.doc_count, })); + return catCounts; + } - const tempCategories = await Promise.all(catCounts.map(({ id }) => getCategories(jobId, id))); - const categories = tempCategories - .map((t: any) => { - if (t.categories === undefined || t.categories.length === 0) { - return null; - } - return t.categories[0]; - }) - .filter(t => t !== null); + async function getCategories(jobId: string, catIds: CategoryId[]): Promise { + const result = await callWithRequest('search', { + index: ML_RESULTS_INDEX_PATTERN, + size: catIds.length, + body: { + query: { + bool: { + filter: [ + { + term: { + job_id: jobId, + }, + }, + { + terms: { + category_id: catIds, + }, + }, + ], + }, + }, + }, + }); + + return result.hits.hits?.map((c: { _source: Category }) => c._source) || []; + } + + async function topCategories(jobId: string, numberOfCategories: number) { + const catCounts = await getTopCategoryCounts(jobId, numberOfCategories); + const categories = await getCategories( + jobId, + catCounts.map(c => c.id) + ); const catsById = categories.reduce((p, c) => { p[c.category_id] = c; return p; - }, {} as Record); + }, {} as { [id: number]: Category }); const total = await getTotalCategories(jobId); + return { total, categories: catCounts.map(({ id, count }) => { From 3c0461411ee974cf99dd4e7b74ce5555a6c7b2c8 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Mon, 6 Jan 2020 11:48:58 +0000 Subject: [PATCH 08/29] adding analyzer editing --- .../edit_categorization_analyzer_flyout.tsx | 147 ++++++++++++++++++ .../index.ts | 6 + .../examples_valid_callout.tsx | 2 +- .../categorization_view/metric_selection.tsx | 21 ++- .../job_service/new_job/categorization.ts | 8 +- 5 files changed, 180 insertions(+), 4 deletions(-) create mode 100644 x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx create mode 100644 x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/index.ts diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx new file mode 100644 index 0000000000000..06f34ed99bc9f --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx @@ -0,0 +1,147 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { Fragment, FC, useState, useContext } from 'react'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { + EuiFlyout, + EuiFlyoutFooter, + EuiFlexGroup, + EuiFlexItem, + EuiButton, + EuiButtonEmpty, + EuiTitle, + EuiFlyoutBody, + EuiSpacer, +} from '@elastic/eui'; +import { MLJobEditor } from '../../../../../jobs_list/components/ml_job_editor'; +import { isValidJson } from '../../../../../../../../common/util/validation_utils'; +import { JobCreatorContext } from '../../job_creator_context'; +import { CategorizationJobCreator } from '../../../../common/job_creator'; +import { getNewJobDefaults } from '../../../../../../services/ml_server_info'; + +const EDITOR_HEIGHT = '800px'; + +export const EditCategorizationAnalyzerFlyout: FC = () => { + const { jobCreator: jc, jobCreatorUpdate } = useContext(JobCreatorContext); + const jobCreator = jc as CategorizationJobCreator; + const [showJsonFlyout, setShowJsonFlyout] = useState(false); + + const [categorizationAnalyzerString, setCategorizationAnalyzerString] = useState( + JSON.stringify(jobCreator.categorizationAnalyzer, null, 2) + ); + + const [saveable, setSaveable] = useState(false); + + // useEffect(() => { + // // setJobConfigString(jobCreator.formattedJobJson); + // // setDatafeedConfigString(jobCreator.formattedDatafeedJson); + // }, [jobCreatorUpdated]); + + function toggleJsonFlyout() { + setSaveable(false); + setShowJsonFlyout(!showJsonFlyout); + } + + function onJSONChange(json: string) { + setCategorizationAnalyzerString(json); + const valid = isValidJson(json); + setSaveable(valid); + } + + function onSave() { + jobCreator.categorizationAnalyzer = JSON.parse(categorizationAnalyzerString); + jobCreatorUpdate(); + setShowJsonFlyout(false); + } + + function onUseDefault() { + const { anomaly_detectors: anomalyDetectors } = getNewJobDefaults(); + const analyzerString = JSON.stringify(anomalyDetectors.categorization_analyzer!, null, 2); + onJSONChange(analyzerString); + } + + return ( + + + + {showJsonFlyout === true && ( + setShowJsonFlyout(false)} hideCloseButton size={'m'}> + + + + + + + setShowJsonFlyout(false)} + flush="left" + > + + + + + + + + + + + + + + + + + + )} + + ); +}; + +const FlyoutButton: FC<{ onClick(): void }> = ({ onClick }) => { + return ( + + + + ); +}; + +const Contents: FC<{ + title: string; + value: string; + onChange(s: string): void; +}> = ({ title, value, onChange }) => { + return ( + + +
{title}
+ + + + + ); +}; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/index.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/index.ts new file mode 100644 index 0000000000000..5bc89a695b6c8 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/index.ts @@ -0,0 +1,6 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +export { EditCategorizationAnalyzerFlyout } from './edit_categorization_analyzer_flyout'; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx index b12137d6d5ee0..48026e41de7c2 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { Fragment, FC, useEffect, useState } from 'react'; +import React, { Fragment, FC } from 'react'; import { EuiCallOut } from '@elastic/eui'; interface Props { diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx index 1b72844d7fc6e..28112764bee3d 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx @@ -15,6 +15,7 @@ import { CategorizationDetector } from '../categorization_detector'; import { FieldExamples } from './field_examples'; import { ExamplesValidCallout } from './examples_valid_callout'; import { CategoryExample } from '../../../../../common/results_loader'; +import { EditCategorizationAnalyzerFlyout } from '../../../common/edit_categorization_analyzer_flyout'; // import { getNewJobDefaults } from '../../../../../../../services/ml_server_info'; interface Props { @@ -28,6 +29,9 @@ export const CategorizationDetectors: FC = ({ setIsValid }) => { const [loadingData, setLoadingData] = useState(false); const [start, setStart] = useState(jobCreator.start); const [end, setEnd] = useState(jobCreator.end); + const [categorizationAnalyzerString, setCategorizationAnalyzerString] = useState( + JSON.stringify(jobCreator.categorizationAnalyzer) + ); const [fieldExamples, setFieldExamples] = useState(null); const [examplesValid, setExamplesValid] = useState(0); @@ -44,9 +48,19 @@ export const CategorizationDetectors: FC = ({ setIsValid }) => { }, [categorizationFieldName]); useEffect(() => { + let updateExamples = false; if (jobCreator.start !== start || jobCreator.end !== end) { setStart(jobCreator.start); setEnd(jobCreator.end); + updateExamples = true; + } + const tempCategorizationAnalyzerString = JSON.stringify(jobCreator.categorizationAnalyzer); + if (tempCategorizationAnalyzerString !== categorizationAnalyzerString) { + setCategorizationAnalyzerString(tempCategorizationAnalyzerString); + updateExamples = true; + } + + if (updateExamples) { loadFieldExamples(); } if (jobCreator.categorizationFieldName !== categorizationFieldName) { @@ -77,7 +91,12 @@ export const CategorizationDetectors: FC = ({ setIsValid }) => { - {fieldExamples !== null && } + {fieldExamples !== null && ( + <> + + + + )} ); }; diff --git a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts index 32467c97fdb05..090cf00933ae0 100644 --- a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts +++ b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts @@ -92,10 +92,14 @@ export function categorizationExamplesProvider(callWithRequest: callWithRequestT const sumLengths = lengths.map((s => (a: number) => (s += a))(0)); const tokensPerExample: Token[][] = lengths.map(l => []); + + const { tokenizer } = getAnalyzer(analyzer); // remove this once ml_classic bug is fixed + tokens.forEach((t, i) => { for (let g = 0; g < sumLengths.length; g++) { - if (t.start_offset <= sumLengths[g]) { - const offset = g > 0 ? sumLengths[g - 1] : 0; + const extra = tokenizer === 'ml_classic' ? 0 : g; // remove this once ml_classic bug is fixed + if (t.start_offset <= sumLengths[g] + extra) { + const offset = g > 0 ? sumLengths[g - 1] + extra : 0; tokensPerExample[g].push({ ...t, start_offset: t.start_offset - offset, From b92b087a5e636d9817a24aae5bf42d6c3e7c7349 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Mon, 6 Jan 2020 17:09:57 +0000 Subject: [PATCH 09/29] improving callout --- .../job_creator/categorization_job_creator.ts | 2 +- .../edit_categorization_analyzer_flyout.tsx | 4 +- .../categorization_view.tsx | 14 +-- .../examples_valid_callout.tsx | 111 ++++++++++++++---- .../categorization_view/metric_selection.tsx | 26 ++-- .../metric_selection_summary.tsx | 6 +- .../categorization_view/settings.tsx | 6 +- .../categorization_view/top_categories.tsx | 29 ++--- .../application/services/ml_server_info.ts | 5 +- 9 files changed, 134 insertions(+), 69 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts index 8f1a2dc0c86c7..358064aff62a9 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { IndexPattern } from 'ui/index_patterns'; +import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public'; import { SavedSearchSavedObject } from '../../../../../../common/types/kibana'; import { JobCreator } from './job_creator'; import { Field, Aggregation, mlCategory } from '../../../../../../common/types/fields'; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx index 06f34ed99bc9f..ffc3272ab5dbd 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx @@ -75,7 +75,7 @@ export const EditCategorizationAnalyzerFlyout: FC = () => { @@ -121,7 +121,7 @@ export const EditCategorizationAnalyzerFlyout: FC = () => { const FlyoutButton: FC<{ onClick(): void }> = ({ onClick }) => { return ( - + = ({ isActive, setCanProceed }) => { }, [categoryFieldValid, settingsValid]); return ( - + <> {isActive === false ? ( ) : ( - + <> {categoryFieldValid && ( - + <> - + )} - + )} - + ); }; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx index 48026e41de7c2..2ffa7da4b9fd3 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx @@ -4,35 +4,100 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { Fragment, FC } from 'react'; -import { EuiCallOut } from '@elastic/eui'; +import React, { FC } from 'react'; +import { EuiCallOut, EuiSpacer } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; + +import { CategorizationAnalyzer } from '../../../../../../../services/ml_server_info'; +import { EditCategorizationAnalyzerFlyout } from '../../../common/edit_categorization_analyzer_flyout'; + +type CategorizationAnalyzerType = string | CategorizationAnalyzer | null; interface Props { examplesValid: number; + categorizationAnalyzer: CategorizationAnalyzerType; } -export const ExamplesValidCallout: FC = ({ examplesValid }) => { - function getCallOut() { - if (examplesValid < 0.2) { - return ( - - {Math.floor(examplesValid * 100)}% of field values tested contain valid tokens - - ); - } else if (examplesValid < 0.75) { - return ( - - {Math.floor(examplesValid * 100)}% of field values tested contain valid tokens - - ); - } else { - return ( - - {Math.floor(examplesValid * 100)}% of field values tested contain valid tokens - - ); +export const ExamplesValidCallout: FC = ({ examplesValid, categorizationAnalyzer }) => { + const percentageText = ; + const analyzerUsed = ; + + let color: 'success' | 'danger' | 'warning' = 'success'; + let title = i18n.translate( + 'xpack.ml.newJob.wizard.pickFieldsStep.categorizationFieldCalloutTitle.valid', + { + defaultMessage: 'Selected category field is valid', + } + ); + + if (examplesValid < 0.2) { + color = 'danger'; + title = i18n.translate( + 'xpack.ml.newJob.wizard.pickFieldsStep.categorizationFieldCalloutTitle.invalid', + { + defaultMessage: 'Selected category field is invalid', + } + ); + } else if (examplesValid < 0.75) { + color = 'warning'; + title = i18n.translate( + 'xpack.ml.newJob.wizard.pickFieldsStep.categorizationFieldCalloutTitle.possiblyInvalid', + { + defaultMessage: 'Selected category field is possibly invalid', + } + ); + } + + return ( + + {percentageText} + + {analyzerUsed} + + ); +}; + +const PercentageText: FC<{ examplesValid: number }> = ({ examplesValid }) => ( +
+ +
+); + +const AnalyzerUsed: FC<{ categorizationAnalyzer: CategorizationAnalyzerType }> = ({ + categorizationAnalyzer, +}) => { + let analyzer = ''; + if (typeof categorizationAnalyzer === null) { + return null; + } + + if (typeof categorizationAnalyzer === 'string') { + analyzer = categorizationAnalyzer; + } else { + if (categorizationAnalyzer?.tokenizer !== undefined) { + analyzer = categorizationAnalyzer?.tokenizer!; + } else if (categorizationAnalyzer?.analyzer !== undefined) { + analyzer = categorizationAnalyzer?.analyzer!; } } - return {getCallOut()}; + return ( + <> +
+ +
+
+ +
+ + ); }; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx index 28112764bee3d..02aced53ca262 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { Fragment, FC, useContext, useEffect, useState } from 'react'; +import React, { FC, useContext, useEffect, useState } from 'react'; import { JobCreatorContext } from '../../../job_creator_context'; import { CategorizationJobCreator } from '../../../../../common/job_creator'; @@ -15,9 +15,12 @@ import { CategorizationDetector } from '../categorization_detector'; import { FieldExamples } from './field_examples'; import { ExamplesValidCallout } from './examples_valid_callout'; import { CategoryExample } from '../../../../../common/results_loader'; -import { EditCategorizationAnalyzerFlyout } from '../../../common/edit_categorization_analyzer_flyout'; +import { LoadingWrapper } from '../../../charts/loading_wrapper'; +// import { EditCategorizationAnalyzerFlyout } from '../../../common/edit_categorization_analyzer_flyout'; // import { getNewJobDefaults } from '../../../../../../../services/ml_server_info'; +const LOADING_PLACEHOLDER_HEIGHT = '200x'; + interface Props { setIsValid: (na: boolean) => void; } @@ -87,16 +90,23 @@ export const CategorizationDetectors: FC = ({ setIsValid }) => { }, [examplesValid]); return ( - + <> - - {fieldExamples !== null && ( + {loadingData === true && ( + +
+ + )} + {fieldExamples !== null && loadingData === false && ( <> - - + + )} - + ); }; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection_summary.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection_summary.tsx index 2c91e91ec51d2..768d8c394fb8f 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection_summary.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection_summary.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { Fragment, FC, useContext, useEffect, useState } from 'react'; +import React, { FC, useContext, useEffect, useState } from 'react'; import { JobCreatorContext } from '../../../job_creator_context'; import { CategorizationJobCreator } from '../../../../../common/job_creator'; import { Results, Anomaly } from '../../../../../common/results_loader'; @@ -62,7 +62,7 @@ export const CategorizationDetectorsSummary: FC = () => { } return ( - + <> { fadeChart={jobIsRunning} /> - + ); }; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/settings.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/settings.tsx index 0044a99b72887..c931b57a4f488 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/settings.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/settings.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { Fragment, FC } from 'react'; +import React, { FC } from 'react'; import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import { BucketSpan } from '../bucket_span'; @@ -16,7 +16,7 @@ interface Props { export const CategorizationSettings: FC = ({ setIsValid }) => { return ( - + <> @@ -25,6 +25,6 @@ export const CategorizationSettings: FC = ({ setIsValid }) => { - + ); }; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx index 9ac273334195d..6e3823c8d698c 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx @@ -6,31 +6,16 @@ import React, { FC, useContext, useEffect, useState } from 'react'; import { EuiBasicTable, EuiText } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; import { JobCreatorContext } from '../../../job_creator_context'; import { CategorizationJobCreator } from '../../../../../common/job_creator'; import { Results } from '../../../../../common/results_loader'; import { ml } from '../../../../../../../services/ml_api_service'; -const DTR_IDX = 0; - -interface Category { - // MOVE!!!!!!! - job_id: string; - category_id: number; - terms: string; - regex: string; - max_matching_length: number; - examples: string[]; - grok_pattern: string; -} - export const TopCategories: FC = () => { - const { jobCreator: jc, chartLoader, resultsLoader, chartInterval } = useContext( - JobCreatorContext - ); + const { jobCreator: jc, resultsLoader } = useContext(JobCreatorContext); const jobCreator = jc as CategorizationJobCreator; - const [categories, setCategories] = useState([]); const [tableRow, setTableRow] = useState>([]); const [totalCategories, setTotalCategories] = useState(0); @@ -40,7 +25,6 @@ export const TopCategories: FC = () => { async function loadTopCats() { const results = await ml.jobs.topCategories(jobCreator.jobId, 5); - setCategories(results.categories); setTableRow( results.categories.map(c => ({ count: c.count, @@ -48,7 +32,6 @@ export const TopCategories: FC = () => { })) ); setTotalCategories(results.total); - // console.log(topCats.total); } useEffect(() => { @@ -85,7 +68,13 @@ export const TopCategories: FC = () => { <> {totalCategories > 0 && ( <> -
Total categories: {totalCategories}
+
+ +
)} diff --git a/x-pack/legacy/plugins/ml/public/application/services/ml_server_info.ts b/x-pack/legacy/plugins/ml/public/application/services/ml_server_info.ts index 0b31187ecae59..6bf5a7b0c9743 100644 --- a/x-pack/legacy/plugins/ml/public/application/services/ml_server_info.ts +++ b/x-pack/legacy/plugins/ml/public/application/services/ml_server_info.ts @@ -18,8 +18,9 @@ export interface MlServerDefaults { export interface CategorizationAnalyzer { char_filter?: any[]; - tokenizer: string; - filter: any[]; + tokenizer?: string; + filter?: any[]; + analyzer?: string; } export interface MlServerLimits { From d38050f7a7989e93a6ddef51b9914bbcd7da524f Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Tue, 7 Jan 2020 09:20:18 +0000 Subject: [PATCH 10/29] updating callout text --- x-pack/legacy/plugins/ml/common/constants/new_job.ts | 3 +++ .../categorization_view/examples_valid_callout.tsx | 11 +++++++++-- .../components/categorization_view/top_categories.tsx | 3 ++- .../models/job_service/new_job/categorization.ts | 9 ++++++--- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/x-pack/legacy/plugins/ml/common/constants/new_job.ts b/x-pack/legacy/plugins/ml/common/constants/new_job.ts index d62000fb7f19f..0c53c00c07e13 100644 --- a/x-pack/legacy/plugins/ml/common/constants/new_job.ts +++ b/x-pack/legacy/plugins/ml/common/constants/new_job.ts @@ -24,3 +24,6 @@ export const DEFAULT_BUCKET_SPAN = '15m'; export const DEFAULT_QUERY_DELAY = '60s'; export const SHARED_RESULTS_INDEX_NAME = 'shared'; + +export const NUMBER_OF_CATEGORY_EXAMPLES = 5; +export const CATEGORY_EXAMPLES_MULTIPLIER = 20; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx index 2ffa7da4b9fd3..5d1d15ebd993b 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx @@ -11,6 +11,10 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { CategorizationAnalyzer } from '../../../../../../../services/ml_server_info'; import { EditCategorizationAnalyzerFlyout } from '../../../common/edit_categorization_analyzer_flyout'; +import { + NUMBER_OF_CATEGORY_EXAMPLES, + CATEGORY_EXAMPLES_MULTIPLIER, +} from '../../../../../../../../../common/constants/new_job'; type CategorizationAnalyzerType = string | CategorizationAnalyzer | null; @@ -62,8 +66,11 @@ const PercentageText: FC<{ examplesValid: number }> = ({ examplesValid }) => (
); diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx index 6e3823c8d698c..288730cd41577 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx @@ -11,6 +11,7 @@ import { JobCreatorContext } from '../../../job_creator_context'; import { CategorizationJobCreator } from '../../../../../common/job_creator'; import { Results } from '../../../../../common/results_loader'; import { ml } from '../../../../../../../services/ml_api_service'; +import { NUMBER_OF_CATEGORY_EXAMPLES } from '../../../../../../../../../common/constants/new_job'; export const TopCategories: FC = () => { const { jobCreator: jc, resultsLoader } = useContext(JobCreatorContext); @@ -24,7 +25,7 @@ export const TopCategories: FC = () => { } async function loadTopCats() { - const results = await ml.jobs.topCategories(jobCreator.jobId, 5); + const results = await ml.jobs.topCategories(jobCreator.jobId, NUMBER_OF_CATEGORY_EXAMPLES); setTableRow( results.categories.map(c => ({ count: c.count, diff --git a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts index 090cf00933ae0..d65066b20f023 100644 --- a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts +++ b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts @@ -5,6 +5,7 @@ */ import { ML_RESULTS_INDEX_PATTERN } from '../../../../common/constants/index_patterns'; +import { CATEGORY_EXAMPLES_MULTIPLIER } from '../../../../common/constants/new_job'; type callWithRequestType = (action: string, params: any) => Promise; @@ -131,11 +132,10 @@ export function categorizationExamplesProvider(callWithRequest: callWithRequestT end: number, analyzer?: any ) { - const MULTIPLIER = 20; const examples = await categorizationExamples( indexPatternTitle, query, - size * MULTIPLIER, + size * CATEGORY_EXAMPLES_MULTIPLIER, categorizationFieldName, timeField, start, @@ -151,7 +151,10 @@ export function categorizationExamplesProvider(callWithRequest: callWithRequestT return { valid: sortedExamples.length === 0 ? 0 : validExamples.length / sortedExamples.length, examples: sortedExamples - .filter((e, i) => i / MULTIPLIER - Math.floor(i / MULTIPLIER) === 0) + .filter( + (e, i) => + i / CATEGORY_EXAMPLES_MULTIPLIER - Math.floor(i / CATEGORY_EXAMPLES_MULTIPLIER) === 0 + ) .sort((a, b) => a.origIndex - b.origIndex) .map(e => ({ text: e.text, tokens: e.tokens })), }; From 868da11dd872634e8ed14145d5f23e0c1cfcdaa0 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Tue, 7 Jan 2020 09:29:12 +0000 Subject: [PATCH 11/29] fixing import path --- .../common/results_loader/categorization_examples_loader.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts index 40dc99765f4c3..3f85c7242cb20 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { IndexPattern } from 'ui/index_patterns'; +import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public'; import { IndexPatternTitle } from '../../../../../../common/types/kibana'; import { CategorizationJobCreator } from '../job_creator'; import { ml } from '../../../../services/ml_api_service'; From 6f0f363c33c171ea2cda8a663434827c99dcefc9 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Tue, 7 Jan 2020 10:21:39 +0000 Subject: [PATCH 12/29] resetting cat analyser json on flyout open --- .../edit_categorization_analyzer_flyout.tsx | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx index ffc3272ab5dbd..6d629d88368f0 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { Fragment, FC, useState, useContext } from 'react'; +import React, { Fragment, FC, useEffect, useState, useContext } from 'react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; import { @@ -30,17 +30,17 @@ export const EditCategorizationAnalyzerFlyout: FC = () => { const { jobCreator: jc, jobCreatorUpdate } = useContext(JobCreatorContext); const jobCreator = jc as CategorizationJobCreator; const [showJsonFlyout, setShowJsonFlyout] = useState(false); + const [saveable, setSaveable] = useState(false); const [categorizationAnalyzerString, setCategorizationAnalyzerString] = useState( JSON.stringify(jobCreator.categorizationAnalyzer, null, 2) ); - const [saveable, setSaveable] = useState(false); - - // useEffect(() => { - // // setJobConfigString(jobCreator.formattedJobJson); - // // setDatafeedConfigString(jobCreator.formattedDatafeedJson); - // }, [jobCreatorUpdated]); + useEffect(() => { + if (showJsonFlyout === true) { + setCategorizationAnalyzerString(JSON.stringify(jobCreator.categorizationAnalyzer, null, 2)); + } + }, [showJsonFlyout]); function toggleJsonFlyout() { setSaveable(false); From 84acf3c9e7599453f163174d04e5219302474d8e Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Tue, 7 Jan 2020 13:56:33 +0000 Subject: [PATCH 13/29] disabling model plot by default --- .../plugins/ml/common/types/categories.ts | 17 +++++ .../categorization_examples_loader.ts | 5 +- .../categorization_detector/description.tsx | 2 +- .../categorization_view/top_categories.tsx | 27 ++++--- .../jobs/new_job/pages/job_type/page.tsx | 2 +- .../jobs/new_job/pages/new_job/page.tsx | 6 -- .../services/ml_api_service/index.d.ts | 6 +- .../job_service/new_job/categorization.ts | 70 +++++++++++-------- 8 files changed, 83 insertions(+), 52 deletions(-) create mode 100644 x-pack/legacy/plugins/ml/common/types/categories.ts diff --git a/x-pack/legacy/plugins/ml/common/types/categories.ts b/x-pack/legacy/plugins/ml/common/types/categories.ts new file mode 100644 index 0000000000000..9391fb6952b9a --- /dev/null +++ b/x-pack/legacy/plugins/ml/common/types/categories.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export type CategoryId = number; + +export interface Category { + job_id: string; + category_id: CategoryId; + terms: string; + regex: string; + max_matching_length: number; + examples: string[]; + grok_pattern: string; +} diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts index 3f85c7242cb20..b4412765c79fd 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts @@ -8,8 +8,7 @@ import { IndexPattern } from '../../../../../../../../../../src/plugins/data/pub import { IndexPatternTitle } from '../../../../../../common/types/kibana'; import { CategorizationJobCreator } from '../job_creator'; import { ml } from '../../../../services/ml_api_service'; - -const NUM_OF_EXAMPLES = 5; +import { NUMBER_OF_CATEGORY_EXAMPLES } from '../../../../../../common/constants/new_job'; export interface Token { token: string; @@ -53,7 +52,7 @@ export class CategorizationExamplesLoader { const resp = await ml.jobs.categorizationFieldExamples( this._indexPatternTitle, this._query, - NUM_OF_EXAMPLES, + NUMBER_OF_CATEGORY_EXAMPLES, categorizationFieldName, this._timeFieldName, start, diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/description.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/description.tsx index 5571977671d53..bb530c84e4982 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/description.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/description.tsx @@ -23,7 +23,7 @@ export const Description: FC = memo(({ children }) => { description={ } > diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx index 288730cd41577..3bade07250b46 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx @@ -17,7 +17,7 @@ export const TopCategories: FC = () => { const { jobCreator: jc, resultsLoader } = useContext(JobCreatorContext); const jobCreator = jc as CategorizationJobCreator; - const [tableRow, setTableRow] = useState>([]); + const [tableRow, setTableRow] = useState>([]); const [totalCategories, setTotalCategories] = useState(0); function setResultsWrapper(results: Results) { @@ -44,16 +44,21 @@ export const TopCategories: FC = () => { }, []); const columns = [ - { - field: 'count', - name: 'count', - width: '100px', - render: (count: any) => ( - - {count} - - ), - }, + // only include counts if model plot is enabled + ...(jobCreator.modelPlot + ? [ + { + field: 'count', + name: 'count', + width: '100px', + render: (count: any) => ( + + {count} + + ), + }, + ] + : []), { field: 'example', name: 'Example', diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/job_type/page.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/job_type/page.tsx index 26136bc9ecfda..9a44d561c2d94 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/job_type/page.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/job_type/page.tsx @@ -163,7 +163,7 @@ export const Page: FC = () => { defaultMessage: 'Categorization', }), description: i18n.translate('xpack.ml.newJob.wizard.jobType.categorizationDescription', { - defaultMessage: 'All your categories are belong to us', + defaultMessage: 'Group log messages into categories and detect anomalies within them.', }), id: 'mlJobTypeLinkCategorizationJob', }, diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx index 7307d271d8590..696c053650098 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx @@ -125,12 +125,6 @@ export const Page: FC = ({ existingJobsAndGroups, jobType }) => { jobCreator.createdBy = null; } - if (isCategorizationJobCreator(jobCreator)) { - // categorization job will always use a count agg, so give it - // to the job creator now - jobCreator.modelPlot = true; - } - // auto set the time range if creating a new advanced job autoSetTimeRange = isAdvancedJobCreator(jobCreator); } diff --git a/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/index.d.ts b/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/index.d.ts index 4756114024289..2ad2a148f05d1 100644 --- a/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/index.d.ts +++ b/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/index.d.ts @@ -7,6 +7,7 @@ import { Observable } from 'rxjs'; import { Annotation } from '../../../../common/types/annotations'; import { AggFieldNamePair } from '../../../../common/types/fields'; +import { Category } from '../../../../common/types/categories'; import { ExistingJobsAndGroups } from '../job_service'; import { PrivilegesResponse } from '../../../../common/types/privileges'; import { MlSummaryJobs } from '../../../../common/types/jobs'; @@ -181,7 +182,10 @@ declare interface Ml { end: number, analyzer: any ): Promise<{ valid: number; examples: any[] }>; - topCategories(jobId: string, count: number): Promise<{ total: number; categories: any[] }>; + topCategories( + jobId: string, + count: number + ): Promise<{ total: number; categories: Array<{ count?: number; category: Category }> }>; }; estimateBucketSpan(data: BucketSpanEstimatorData): Promise; diff --git a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts index d65066b20f023..630266e200950 100644 --- a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts +++ b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts @@ -6,6 +6,7 @@ import { ML_RESULTS_INDEX_PATTERN } from '../../../../common/constants/index_patterns'; import { CATEGORY_EXAMPLES_MULTIPLIER } from '../../../../common/constants/new_job'; +import { CategoryId, Category } from '../../../../common/types/categories'; type callWithRequestType = (action: string, params: any) => Promise; @@ -17,18 +18,6 @@ interface Token { position: number; } -type CategoryId = number; - -interface Category { - job_id: string; - category_id: CategoryId; - terms: string; - regex: string; - max_matching_length: number; - examples: string[]; - grok_pattern: string; -} - export function categorizationExamplesProvider(callWithRequest: callWithRequestType) { async function categorizationExamples( indexPatternTitle: string, @@ -233,10 +222,25 @@ export function categorizationExamplesProvider(callWithRequest: callWithRequestT return catCounts; } - async function getCategories(jobId: string, catIds: CategoryId[]): Promise { + async function getCategories( + jobId: string, + catIds: CategoryId[], + size: number + ): Promise { + const categoryFilter = catIds.length + ? { + terms: { + category_id: catIds, + }, + } + : { + exists: { + field: 'category_id', + }, + }; const result = await callWithRequest('search', { index: ML_RESULTS_INDEX_PATTERN, - size: catIds.length, + size, body: { query: { bool: { @@ -246,11 +250,7 @@ export function categorizationExamplesProvider(callWithRequest: callWithRequestT job_id: jobId, }, }, - { - terms: { - category_id: catIds, - }, - }, + categoryFilter, ], }, }, @@ -264,7 +264,8 @@ export function categorizationExamplesProvider(callWithRequest: callWithRequestT const catCounts = await getTopCategoryCounts(jobId, numberOfCategories); const categories = await getCategories( jobId, - catCounts.map(c => c.id) + catCounts.map(c => c.id), + catCounts.length || numberOfCategories ); const catsById = categories.reduce((p, c) => { @@ -274,15 +275,26 @@ export function categorizationExamplesProvider(callWithRequest: callWithRequestT const total = await getTotalCategories(jobId); - return { - total, - categories: catCounts.map(({ id, count }) => { - return { - count, - category: catsById[id] ?? null, - }; - }), - }; + if (catCounts.length) { + return { + total, + categories: catCounts.map(({ id, count }) => { + return { + count, + category: catsById[id] ?? null, + }; + }), + }; + } else { + return { + total, + categories: categories.map(category => { + return { + category, + }; + }), + }; + } } return { From 1955044f3f4a6914202898d4625793d614602a31 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Tue, 7 Jan 2020 15:41:28 +0000 Subject: [PATCH 14/29] minor refactoring --- .../plugins/ml/common/types/categories.ts | 8 ++++++ .../job_creator/categorization_job_creator.ts | 3 -- .../categorization_examples_loader.ts | 9 +----- .../categorization_view/field_examples.tsx | 17 ++--------- .../categorization_view/metric_selection.tsx | 4 --- .../job_service/new_job/categorization.ts | 28 +++++-------------- 6 files changed, 19 insertions(+), 50 deletions(-) diff --git a/x-pack/legacy/plugins/ml/common/types/categories.ts b/x-pack/legacy/plugins/ml/common/types/categories.ts index 9391fb6952b9a..6ccd13ed9a39e 100644 --- a/x-pack/legacy/plugins/ml/common/types/categories.ts +++ b/x-pack/legacy/plugins/ml/common/types/categories.ts @@ -15,3 +15,11 @@ export interface Category { examples: string[]; grok_pattern: string; } + +export interface Token { + token: string; + start_offset: number; + end_offset: number; + type: string; + position: number; +} diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts index 358064aff62a9..29234f70b213c 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts @@ -16,7 +16,6 @@ import { getRichDetectors } from './util/general'; import { CategorizationExamplesLoader, CategoryExample } from '../results_loader'; import { CategorizationAnalyzer } from '../../../../services/ml_server_info'; -// type DETECTOR_TYPE = ML_JOB_AGGREGATION.COUNT | ML_JOB_AGGREGATION.RARE; type CategorizationAnalyzerType = string | CategorizationAnalyzer | null; export class CategorizationJobCreator extends JobCreator { @@ -127,8 +126,6 @@ export class CategorizationJobCreator extends JobCreator { this.createdBy = CREATED_BY_LABEL.CATEGORIZATION; const detectors = getRichDetectors(job, datafeed, this.scriptFields, false); - // this.removeAllDetectors(); - const dtr = detectors[0]; if (detectors.length && dtr.agg !== null && dtr.field !== null) { this._detectorType = diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts index b4412765c79fd..16f127ae3d728 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts @@ -6,18 +6,11 @@ import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public'; import { IndexPatternTitle } from '../../../../../../common/types/kibana'; +import { Token } from '../../../../../../common/types/categories'; import { CategorizationJobCreator } from '../job_creator'; import { ml } from '../../../../services/ml_api_service'; import { NUMBER_OF_CATEGORY_EXAMPLES } from '../../../../../../common/constants/new_job'; -export interface Token { - token: string; - start_offset: number; - end_offset: number; - type: string; - position: number; -} - export interface CategoryExample { text: string; tokens: Token[]; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx index bc13f6f911731..bfe6cc3b905e2 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx @@ -6,22 +6,11 @@ import React, { FC } from 'react'; import { EuiBasicTable, EuiText } from '@elastic/eui'; +import { Token } from '../../../../../../../../../common/types/categories'; +import { CategoryExample } from '../../../../../common/results_loader'; interface Props { - fieldExamples: Cat[] | null; -} - -interface Token { - token: string; - start_offset: number; - end_offset: number; - type: string; - position: number; -} - -interface Cat { - text: string; - tokens: Token[]; + fieldExamples: CategoryExample[] | null; } const TOKEN_HIGHLIGHT_COLOR = '#b0ccf7'; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx index 02aced53ca262..e401150e1aed4 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx @@ -8,16 +8,12 @@ import React, { FC, useContext, useEffect, useState } from 'react'; import { JobCreatorContext } from '../../../job_creator_context'; import { CategorizationJobCreator } from '../../../../../common/job_creator'; -// import { ml } from '../../../../../../../services/ml_api_service'; import { CategorizationField } from '../categorization_field'; import { CategorizationDetector } from '../categorization_detector'; -// import { useKibanaContext } from '../../../../../../../contexts/kibana'; import { FieldExamples } from './field_examples'; import { ExamplesValidCallout } from './examples_valid_callout'; import { CategoryExample } from '../../../../../common/results_loader'; import { LoadingWrapper } from '../../../charts/loading_wrapper'; -// import { EditCategorizationAnalyzerFlyout } from '../../../common/edit_categorization_analyzer_flyout'; -// import { getNewJobDefaults } from '../../../../../../../services/ml_server_info'; const LOADING_PLACEHOLDER_HEIGHT = '200x'; diff --git a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts index 630266e200950..61e8ec67e24e8 100644 --- a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts +++ b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts @@ -6,18 +6,10 @@ import { ML_RESULTS_INDEX_PATTERN } from '../../../../common/constants/index_patterns'; import { CATEGORY_EXAMPLES_MULTIPLIER } from '../../../../common/constants/new_job'; -import { CategoryId, Category } from '../../../../common/types/categories'; +import { CategoryId, Category, Token } from '../../../../common/types/categories'; type callWithRequestType = (action: string, params: any) => Promise; -interface Token { - token: string; - start_offset: number; - end_offset: number; - type: string; - position: number; -} - export function categorizationExamplesProvider(callWithRequest: callWithRequestType) { async function categorizationExamples( indexPatternTitle: string, @@ -65,18 +57,12 @@ export function categorizationExamplesProvider(callWithRequest: callWithRequestT ?.map((doc: any) => doc._source[categorizationFieldName]) .filter((example: string | undefined) => example !== undefined); - let tokens: Token[] = []; - try { - const { tokens: tempTokens } = await callWithRequest('indices.analyze', { - body: { - ...getAnalyzer(analyzer), - text: examples, - }, - }); - tokens = tempTokens; - } catch (error) { - // do something with this error - } + const { tokens }: { tokens: Token[] } = await callWithRequest('indices.analyze', { + body: { + ...getAnalyzer(analyzer), + text: examples, + }, + }); const lengths = examples.map(e => e.length); const sumLengths = lengths.map((s => (a: number) => (s += a))(0)); From 03235c3d6b3dac8b7dc56470379102a2d044d589 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Tue, 7 Jan 2020 16:21:17 +0000 Subject: [PATCH 15/29] fixing types --- .../categorization_view/field_examples.tsx | 1 - .../job_service/new_job/categorization.ts | 21 ++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx index bfe6cc3b905e2..1aa572dd38d12 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx @@ -6,7 +6,6 @@ import React, { FC } from 'react'; import { EuiBasicTable, EuiText } from '@elastic/eui'; -import { Token } from '../../../../../../../../../common/types/categories'; import { CategoryExample } from '../../../../../common/results_loader'; interface Props { diff --git a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts index 61e8ec67e24e8..2d05187ecbf07 100644 --- a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts +++ b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts @@ -57,17 +57,24 @@ export function categorizationExamplesProvider(callWithRequest: callWithRequestT ?.map((doc: any) => doc._source[categorizationFieldName]) .filter((example: string | undefined) => example !== undefined); - const { tokens }: { tokens: Token[] } = await callWithRequest('indices.analyze', { - body: { - ...getAnalyzer(analyzer), - text: examples, - }, - }); + let tokens: Token[] = []; + try { + const { tokens: tempTokens } = await callWithRequest('indices.analyze', { + body: { + ...getAnalyzer(analyzer), + text: examples, + }, + }); + tokens = tempTokens; + } catch (error) { + // fail silently, the tokens could not be loaded + // an empty list of tokens will be returned for each example + } const lengths = examples.map(e => e.length); const sumLengths = lengths.map((s => (a: number) => (s += a))(0)); - const tokensPerExample: Token[][] = lengths.map(l => []); + const tokensPerExample: Token[][] = examples.map(e => []); const { tokenizer } = getAnalyzer(analyzer); // remove this once ml_classic bug is fixed From 42cc293f7644a17268cc4f611bca37f62a8b71c2 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Tue, 7 Jan 2020 19:18:55 +0000 Subject: [PATCH 16/29] hide estimate bucket span --- .../components/bucket_span/bucket_span.tsx | 11 +++++++---- .../components/categorization_view/settings.tsx | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/bucket_span/bucket_span.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/bucket_span/bucket_span.tsx index dfe9272984b81..216561bac2c62 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/bucket_span/bucket_span.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/bucket_span/bucket_span.tsx @@ -14,9 +14,10 @@ import { BucketSpanEstimator } from '../bucket_span_estimator'; interface Props { setIsValid: (proceed: boolean) => void; + hideEstimateButton?: boolean; } -export const BucketSpan: FC = ({ setIsValid }) => { +export const BucketSpan: FC = ({ setIsValid, hideEstimateButton = false }) => { const { jobCreator, jobCreatorUpdate, @@ -56,9 +57,11 @@ export const BucketSpan: FC = ({ setIsValid }) => { disabled={estimating} /> - - - + {hideEstimateButton === false && ( + + + + )} ); diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/settings.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/settings.tsx index c931b57a4f488..9c0d2d682da40 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/settings.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/settings.tsx @@ -19,7 +19,7 @@ export const CategorizationSettings: FC = ({ setIsValid }) => { <> - + From 9b5cce9139a526f7c06db2dd93d47aa19e2821df Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Tue, 7 Jan 2020 19:31:43 +0000 Subject: [PATCH 17/29] setting default bucket span --- .../legacy/plugins/ml/common/constants/new_job.ts | 1 + .../job_creator/categorization_job_creator.ts | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/ml/common/constants/new_job.ts b/x-pack/legacy/plugins/ml/common/constants/new_job.ts index 0c53c00c07e13..b22de2231798e 100644 --- a/x-pack/legacy/plugins/ml/common/constants/new_job.ts +++ b/x-pack/legacy/plugins/ml/common/constants/new_job.ts @@ -21,6 +21,7 @@ export enum CREATED_BY_LABEL { export const DEFAULT_MODEL_MEMORY_LIMIT = '10MB'; export const DEFAULT_BUCKET_SPAN = '15m'; +export const DEFAULT_RARE_BUCKET_SPAN = '1h'; export const DEFAULT_QUERY_DELAY = '60s'; export const SHARED_RESULTS_INDEX_NAME = 'shared'; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts index 29234f70b213c..55319c5b741b5 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts @@ -10,7 +10,12 @@ import { JobCreator } from './job_creator'; import { Field, Aggregation, mlCategory } from '../../../../../../common/types/fields'; import { Job, Datafeed, Detector } from './configs'; import { createBasicDetector } from './util/default_configs'; -import { JOB_TYPE, CREATED_BY_LABEL } from '../../../../../../common/constants/new_job'; +import { + JOB_TYPE, + CREATED_BY_LABEL, + DEFAULT_BUCKET_SPAN, + DEFAULT_RARE_BUCKET_SPAN, +} from '../../../../../../common/constants/new_job'; import { ML_JOB_AGGREGATION } from '../../../../../../common/constants/aggregation_types'; import { getRichDetectors } from './util/general'; import { CategorizationExamplesLoader, CategoryExample } from '../results_loader'; @@ -68,8 +73,10 @@ export class CategorizationJobCreator extends JobCreator { this.removeAllInfluencers(); if (type === ML_JOB_AGGREGATION.COUNT) { this._createCountDetector(); + this.bucketSpan = DEFAULT_BUCKET_SPAN; } else { this._createRareDetector(); + this.bucketSpan = DEFAULT_RARE_BUCKET_SPAN; } } @@ -132,7 +139,12 @@ export class CategorizationJobCreator extends JobCreator { dtr.agg.id === ML_JOB_AGGREGATION.COUNT ? ML_JOB_AGGREGATION.COUNT : ML_JOB_AGGREGATION.RARE; + + const bs = job.analysis_config.bucket_span; this.setDetectorType(this._detectorType); + // set the bucketspan back to the original value + // as setDetectorType applies a default + this.bucketSpan = bs; } } } From f982beab353d220ddb84f946f29ef930f20b229e Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Wed, 8 Jan 2020 11:40:47 +0000 Subject: [PATCH 18/29] removing ml_classic workaround --- .../ml/server/models/job_service/new_job/categorization.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts index 2d05187ecbf07..4db3ada9d2357 100644 --- a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts +++ b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts @@ -76,13 +76,10 @@ export function categorizationExamplesProvider(callWithRequest: callWithRequestT const tokensPerExample: Token[][] = examples.map(e => []); - const { tokenizer } = getAnalyzer(analyzer); // remove this once ml_classic bug is fixed - tokens.forEach((t, i) => { for (let g = 0; g < sumLengths.length; g++) { - const extra = tokenizer === 'ml_classic' ? 0 : g; // remove this once ml_classic bug is fixed - if (t.start_offset <= sumLengths[g] + extra) { - const offset = g > 0 ? sumLengths[g - 1] + extra : 0; + if (t.start_offset <= sumLengths[g] + g) { + const offset = g > 0 ? sumLengths[g - 1] + g : 0; tokensPerExample[g].push({ ...t, start_offset: t.start_offset - offset, From 8130a342a28923b87406e962e449cf8ab0ef564a Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Wed, 8 Jan 2020 11:46:09 +0000 Subject: [PATCH 19/29] changing style of detector selection --- .../categorization_detector.tsx | 40 ++++++++--- .../categorization_detector_select.tsx | 43 ------------ .../categorization_detector/description.tsx | 35 ---------- .../detector_cards.tsx | 69 +++++++++++++++++++ .../categorization_view/metric_selection.tsx | 2 + 5 files changed, 103 insertions(+), 86 deletions(-) delete mode 100644 x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/categorization_detector_select.tsx delete mode 100644 x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/description.tsx create mode 100644 x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/detector_cards.tsx diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/categorization_detector.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/categorization_detector.tsx index 18977159cc54b..96ac1c3f0e325 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/categorization_detector.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/categorization_detector.tsx @@ -5,11 +5,13 @@ */ import React, { FC, useContext, useEffect, useState } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { EuiFlexGroup, EuiSpacer, EuiTitle } from '@elastic/eui'; -import { CategorizationDetectorSelect } from './categorization_detector_select'; +import { ML_JOB_AGGREGATION } from '../../../../../../../../../common/constants/aggregation_types'; import { JobCreatorContext } from '../../../job_creator_context'; import { CategorizationJobCreator } from '../../../../../common/job_creator'; -import { Description } from './description'; +import { CountCard, RareCard } from './detector_cards'; export const CategorizationDetector: FC = () => { const { jobCreator: jc, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext); @@ -29,12 +31,34 @@ export const CategorizationDetector: FC = () => { setCategorizationDetectorType(jobCreator.selectedDetectorType); }, [jobCreatorUpdated]); + function onCountSelection() { + setCategorizationDetectorType(ML_JOB_AGGREGATION.COUNT); + } + function onRareSelection() { + setCategorizationDetectorType(ML_JOB_AGGREGATION.RARE); + } + return ( - - - + <> + +

+ +

+ + + + + + + ); }; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/categorization_detector_select.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/categorization_detector_select.tsx deleted file mode 100644 index 2c7998215a486..0000000000000 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/categorization_detector_select.tsx +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import React, { FC } from 'react'; -import { EuiComboBox, EuiComboBoxOptionProps } from '@elastic/eui'; - -import { ML_JOB_AGGREGATION } from '../../../../../../../../../common/constants/aggregation_types'; - -interface Props { - changeHandler(i: ML_JOB_AGGREGATION.COUNT | ML_JOB_AGGREGATION.RARE): void; - selectedDetectorType: ML_JOB_AGGREGATION.COUNT | ML_JOB_AGGREGATION.RARE; -} - -export const CategorizationDetectorSelect: FC = ({ - changeHandler, - selectedDetectorType, -}) => { - const options: EuiComboBoxOptionProps[] = [ - { label: 'Count', value: ML_JOB_AGGREGATION.COUNT }, - { label: 'Rare', value: ML_JOB_AGGREGATION.RARE }, - ]; - - const selection = [options.find(o => o.value === selectedDetectorType) || options[0]]; - - function onChange(selectedOptions: EuiComboBoxOptionProps[]) { - const option = selectedOptions[0]; - changeHandler(option.value as ML_JOB_AGGREGATION.COUNT | ML_JOB_AGGREGATION.RARE); - } - - return ( - - ); -}; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/description.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/description.tsx deleted file mode 100644 index bb530c84e4982..0000000000000 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/description.tsx +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import React, { memo, FC } from 'react'; -import { i18n } from '@kbn/i18n'; -import { FormattedMessage } from '@kbn/i18n/react'; -import { EuiDescribedFormGroup, EuiFormRow } from '@elastic/eui'; - -export const Description: FC = memo(({ children }) => { - const title = i18n.translate( - 'xpack.ml.newJob.wizard.pickFieldsStep.categorizationDetectorSelect.title', - { - defaultMessage: 'Categorization detector', - } - ); - return ( - {title}} - description={ - - } - > - - <>{children} - - - ); -}); diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/detector_cards.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/detector_cards.tsx new file mode 100644 index 0000000000000..0443331dfb163 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/detector_cards.tsx @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { i18n } from '@kbn/i18n'; +import { EuiFlexItem, EuiCard } from '@elastic/eui'; + +interface CardProps { + onClick: () => void; + isSelected: boolean; +} + +export const CountCard: FC = ({ onClick, isSelected }) => ( + + + + {/*
+ */} + + } + selectable={{ onClick, isSelected }} + /> + +); + +export const RareCard: FC = ({ onClick, isSelected }) => ( + + + + {/*
+ */} + + } + selectable={{ onClick, isSelected }} + /> + +); diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx index e401150e1aed4..2cf43f544ac50 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx @@ -5,6 +5,7 @@ */ import React, { FC, useContext, useEffect, useState } from 'react'; +import { EuiHorizontalRule } from '@elastic/eui'; import { JobCreatorContext } from '../../../job_creator_context'; import { CategorizationJobCreator } from '../../../../../common/job_creator'; @@ -88,6 +89,7 @@ export const CategorizationDetectors: FC = ({ setIsValid }) => { return ( <> + {loadingData === true && ( From 7593215e59a006315cf5da98886d16e3510591d8 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Wed, 8 Jan 2020 12:51:01 +0000 Subject: [PATCH 20/29] fixing convert to advanced issue --- .../jobs/new_job/common/job_creator/util/general.ts | 9 ++++++++- .../charts/loading_wrapper/loading_wrapper.tsx | 4 ++-- .../components/categorization_view/metric_selection.tsx | 6 ++---- .../server/models/job_service/new_job/categorization.ts | 2 +- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts index 29af08b8826ee..22b727452dd8d 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts @@ -19,7 +19,12 @@ import { mlCategory, } from '../../../../../../../common/types/fields'; import { mlJobService } from '../../../../../services/job_service'; -import { JobCreatorType, isMultiMetricJobCreator, isPopulationJobCreator } from '../index'; +import { + JobCreatorType, + isMultiMetricJobCreator, + isPopulationJobCreator, + isCategorizationJobCreator, +} from '../index'; import { CREATED_BY_LABEL, JOB_TYPE } from '../../../../../../../common/constants/new_job'; const getFieldByIdFactory = (scriptFields: Field[]) => (id: string) => { @@ -251,6 +256,8 @@ export function convertToAdvancedJob(jobCreator: JobCreatorType) { jobType = JOB_TYPE.MULTI_METRIC; } else if (isPopulationJobCreator(jobCreator)) { jobType = JOB_TYPE.POPULATION; + } else if (isCategorizationJobCreator(jobCreator)) { + jobType = JOB_TYPE.CATEGORIZATION; } window.location.href = window.location.href.replace(jobType, JOB_TYPE.ADVANCED); diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/loading_wrapper/loading_wrapper.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/loading_wrapper/loading_wrapper.tsx index 1087fdc8d0fce..b8600489a4bd9 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/loading_wrapper/loading_wrapper.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/loading_wrapper/loading_wrapper.tsx @@ -9,7 +9,7 @@ import { EuiFlexGroup, EuiFlexItem, EuiLoadingSpinner } from '@elastic/eui'; interface Props { hasData: boolean; - height: string; + height?: string; loading?: boolean; } @@ -31,7 +31,7 @@ export const LoadingWrapper: FC = ({ hasData, loading = false, height, ch diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx index 2cf43f544ac50..fda0066f9cd37 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx @@ -16,8 +16,6 @@ import { ExamplesValidCallout } from './examples_valid_callout'; import { CategoryExample } from '../../../../../common/results_loader'; import { LoadingWrapper } from '../../../charts/loading_wrapper'; -const LOADING_PLACEHOLDER_HEIGHT = '200x'; - interface Props { setIsValid: (na: boolean) => void; } @@ -92,8 +90,8 @@ export const CategorizationDetectors: FC = ({ setIsValid }) => { {loadingData === true && ( - -
+ +
)} {fieldExamples !== null && loadingData === false && ( diff --git a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts index 4db3ada9d2357..cf07179222051 100644 --- a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts +++ b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts @@ -209,7 +209,7 @@ export function categorizationExamplesProvider(callWithRequest: callWithRequestT id: c.key, count: c.doc_count, })); - return catCounts; + return catCounts || []; } async function getCategories( From 529f109cb1dfd0364d16d7d8824061661ea1a50d Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Wed, 8 Jan 2020 12:57:12 +0000 Subject: [PATCH 21/29] removing sparse data checkbox --- .../components/categorization_view/settings.tsx | 4 ---- 1 file changed, 4 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/settings.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/settings.tsx index 9c0d2d682da40..55db3d495707d 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/settings.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/settings.tsx @@ -8,7 +8,6 @@ import React, { FC } from 'react'; import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import { BucketSpan } from '../bucket_span'; -import { SparseDataSwitch } from '../sparse_data'; interface Props { setIsValid: (proceed: boolean) => void; @@ -21,9 +20,6 @@ export const CategorizationSettings: FC = ({ setIsValid }) => { - - - ); From 2b4f3b8fbc023d14a6c8e1ab982576b0652f0e1e Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Wed, 8 Jan 2020 13:15:36 +0000 Subject: [PATCH 22/29] changes based on review --- x-pack/legacy/plugins/ml/common/constants/new_job.ts | 2 ++ .../jobs/new_job/common/job_validator/job_validator.ts | 3 ++- .../edit_categorization_analyzer_flyout.tsx | 2 +- .../categorization_view/examples_valid_callout.tsx | 6 ++++-- .../components/categorization_view/field_examples.tsx | 8 +++++++- 5 files changed, 16 insertions(+), 5 deletions(-) diff --git a/x-pack/legacy/plugins/ml/common/constants/new_job.ts b/x-pack/legacy/plugins/ml/common/constants/new_job.ts index b22de2231798e..ccd108cd2698f 100644 --- a/x-pack/legacy/plugins/ml/common/constants/new_job.ts +++ b/x-pack/legacy/plugins/ml/common/constants/new_job.ts @@ -28,3 +28,5 @@ export const SHARED_RESULTS_INDEX_NAME = 'shared'; export const NUMBER_OF_CATEGORY_EXAMPLES = 5; export const CATEGORY_EXAMPLES_MULTIPLIER = 20; +export const CATEGORY_EXAMPLES_WARNING_LIMIT = 0.75; +export const CATEGORY_EXAMPLES_ERROR_LIMIT = 0.2; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts index 71a0573e1e6f9..976e94b377ae8 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts @@ -16,6 +16,7 @@ import { JobCreator, JobCreatorType, isCategorizationJobCreator } from '../job_c import { populateValidationMessages, checkForExistingJobAndGroupIds } from './util'; import { ExistingJobsAndGroups } from '../../../../services/job_service'; import { cardinalityValidator, CardinalityValidatorResult } from './validators'; +import { CATEGORY_EXAMPLES_ERROR_LIMIT } from '../../../../../../common/constants/new_job'; // delay start of validation to allow the user to make changes // e.g. if they are typing in a new value, try not to validate @@ -206,7 +207,7 @@ export class JobValidator { private _runAdvancedValidation() { if (isCategorizationJobCreator(this._jobCreator)) { this._advancedValidations.categorizationFieldValid.valid = - this._jobCreator.categoryFieldValid > 0.2; + this._jobCreator.categoryFieldValid > CATEGORY_EXAMPLES_ERROR_LIMIT; } } diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx index 6d629d88368f0..40aa9530d480f 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx @@ -75,7 +75,7 @@ export const EditCategorizationAnalyzerFlyout: FC = () => { diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx index 5d1d15ebd993b..93085607c87c5 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx @@ -14,6 +14,8 @@ import { EditCategorizationAnalyzerFlyout } from '../../../common/edit_categoriz import { NUMBER_OF_CATEGORY_EXAMPLES, CATEGORY_EXAMPLES_MULTIPLIER, + CATEGORY_EXAMPLES_ERROR_LIMIT, + CATEGORY_EXAMPLES_WARNING_LIMIT, } from '../../../../../../../../../common/constants/new_job'; type CategorizationAnalyzerType = string | CategorizationAnalyzer | null; @@ -35,7 +37,7 @@ export const ExamplesValidCallout: FC = ({ examplesValid, categorizationA } ); - if (examplesValid < 0.2) { + if (examplesValid < CATEGORY_EXAMPLES_ERROR_LIMIT) { color = 'danger'; title = i18n.translate( 'xpack.ml.newJob.wizard.pickFieldsStep.categorizationFieldCalloutTitle.invalid', @@ -43,7 +45,7 @@ export const ExamplesValidCallout: FC = ({ examplesValid, categorizationA defaultMessage: 'Selected category field is invalid', } ); - } else if (examplesValid < 0.75) { + } else if (examplesValid < CATEGORY_EXAMPLES_WARNING_LIMIT) { color = 'warning'; title = i18n.translate( 'xpack.ml.newJob.wizard.pickFieldsStep.categorizationFieldCalloutTitle.possiblyInvalid', diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx index 1aa572dd38d12..5850712068cda 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx @@ -5,6 +5,7 @@ */ import React, { FC } from 'react'; +import { i18n } from '@kbn/i18n'; import { EuiBasicTable, EuiText } from '@elastic/eui'; import { CategoryExample } from '../../../../../common/results_loader'; @@ -22,7 +23,12 @@ export const FieldExamples: FC = ({ fieldExamples }) => { const columns = [ { field: 'example', - name: 'Examples', + name: i18n.translate( + 'xpack.ml.newJob.wizard.pickFieldsStep.categorizationFieldExamples.title', + { + defaultMessage: 'Examples', + } + ), render: (example: any) => ( {example} From 74d31c8fb1868ceb83160e7ac92beb8eb4f3b88a Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Wed, 8 Jan 2020 13:37:38 +0000 Subject: [PATCH 23/29] use default mml --- .../common/job_creator/categorization_job_creator.ts | 1 - .../public/application/jobs/new_job/pages/new_job/page.tsx | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts index 55319c5b741b5..fc97f29071de8 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts @@ -70,7 +70,6 @@ export class CategorizationJobCreator extends JobCreator { public setDetectorType(type: ML_JOB_AGGREGATION.COUNT | ML_JOB_AGGREGATION.RARE) { this._detectorType = type; this.removeAllDetectors(); - this.removeAllInfluencers(); if (type === ML_JOB_AGGREGATION.COUNT) { this._createCountDetector(); this.bucketSpan = DEFAULT_BUCKET_SPAN; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx index 696c053650098..b44c1f8373843 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx @@ -110,7 +110,11 @@ export const Page: FC = ({ existingJobsAndGroups, jobType }) => { // creating a new job jobCreator.bucketSpan = DEFAULT_BUCKET_SPAN; - if (jobCreator.type !== JOB_TYPE.POPULATION && jobCreator.type !== JOB_TYPE.ADVANCED) { + if ( + jobCreator.type !== JOB_TYPE.POPULATION && + jobCreator.type !== JOB_TYPE.ADVANCED && + jobCreator.type !== JOB_TYPE.CATEGORIZATION + ) { // for all other than population or advanced, use 10MB jobCreator.modelMemoryLimit = DEFAULT_MODEL_MEMORY_LIMIT; } From 48c18570756886cb231e084b5f87dcd44bd1395e Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Wed, 8 Jan 2020 13:51:10 +0000 Subject: [PATCH 24/29] fixing job cloning --- .../jobs/new_job/pages/new_job/page.tsx | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx index b44c1f8373843..3a37934e6203a 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx @@ -73,6 +73,7 @@ export const Page: FC = ({ existingJobsAndGroups, jobType }) => { // cloning a job const clonedJob = mlJobService.cloneJob(mlJobService.tempJobCloningObjects.job); const { job, datafeed } = expandCombinedJobConfig(clonedJob); + initCategorizationSettings(); jobCreator.cloneFromExistingJob(job, datafeed); // if we're not skipping the time range, this is a standard job clone, so wipe the jobId @@ -131,18 +132,7 @@ export const Page: FC = ({ existingJobsAndGroups, jobType }) => { // auto set the time range if creating a new advanced job autoSetTimeRange = isAdvancedJobCreator(jobCreator); - } - - if (isCategorizationJobCreator(jobCreator)) { - // categorization job will always use a count agg, so give it - // to the job creator now - const count = newJobCapsService.getAggById('count'); - const rare = newJobCapsService.getAggById('rare'); - const eventRate = newJobCapsService.getFieldById(EVENT_RATE_FIELD_ID); - jobCreator.setDefaultDetectorProperties(count, rare, eventRate); - - const { anomaly_detectors: anomalyDetectors } = getNewJobDefaults(); - jobCreator.categorizationAnalyzer = anomalyDetectors.categorization_analyzer!; + initCategorizationSettings(); } if (autoSetTimeRange && isAdvancedJobCreator(jobCreator)) { @@ -160,6 +150,20 @@ export const Page: FC = ({ existingJobsAndGroups, jobType }) => { } } + function initCategorizationSettings() { + if (isCategorizationJobCreator(jobCreator)) { + // categorization job will always use a count agg, so give it + // to the job creator now + const count = newJobCapsService.getAggById('count'); + const rare = newJobCapsService.getAggById('rare'); + const eventRate = newJobCapsService.getFieldById(EVENT_RATE_FIELD_ID); + jobCreator.setDefaultDetectorProperties(count, rare, eventRate); + + const { anomaly_detectors: anomalyDetectors } = getNewJobDefaults(); + jobCreator.categorizationAnalyzer = anomalyDetectors.categorization_analyzer!; + } + } + const chartInterval = new TimeBuckets(); chartInterval.setBarTarget(BAR_TARGET); chartInterval.setMaxBars(MAX_BARS); From 2141d04aef8d5f59e9de624aaa7ed16b652ef216 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Wed, 8 Jan 2020 15:20:30 +0000 Subject: [PATCH 25/29] changes based on review --- .../edit_categorization_analyzer_flyout.tsx | 2 +- .../categorization_detector/detector_cards.tsx | 14 ++------------ .../categorization_view.tsx | 18 +++++++----------- .../examples_valid_callout.tsx | 4 ++-- .../categorization_view/field_examples.tsx | 16 ++++++++-------- .../job_service/new_job/categorization.ts | 3 +-- .../models/job_service/new_job/charts.ts | 2 +- .../models/job_service/new_job/line_chart.ts | 3 +-- .../job_service/new_job/population_chart.ts | 3 +-- 9 files changed, 24 insertions(+), 41 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx index 40aa9530d480f..c2ddfd35f4c13 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx @@ -70,7 +70,7 @@ export const EditCategorizationAnalyzerFlyout: FC = () => { {showJsonFlyout === true && ( - setShowJsonFlyout(false)} hideCloseButton size={'m'}> + setShowJsonFlyout(false)} hideCloseButton size="m"> = ({ onClick, isSelected }) => ( description={ <> - {/*
- */} } selectable={{ onClick, isSelected }} @@ -53,14 +48,9 @@ export const RareCard: FC = ({ onClick, isSelected }) => ( description={ <> - {/*
- */} } selectable={{ onClick, isSelected }} diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/categorization_view.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/categorization_view.tsx index fb0590097d563..5017b0c3239e8 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/categorization_view.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/categorization_view.tsx @@ -26,19 +26,15 @@ export const CategorizationView: FC = ({ isActive, setCanProceed }) => { } }, [categoryFieldValid, settingsValid]); - return ( + return isActive === false ? ( + + ) : ( <> - {isActive === false ? ( - - ) : ( + + {categoryFieldValid && ( <> - - {categoryFieldValid && ( - <> - - - - )} + + )} diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx index 93085607c87c5..15684d2bf5939 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx @@ -5,7 +5,7 @@ */ import React, { FC } from 'react'; -import { EuiCallOut, EuiSpacer } from '@elastic/eui'; +import { EuiCallOut, EuiSpacer, EuiCallOutProps } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -29,7 +29,7 @@ export const ExamplesValidCallout: FC = ({ examplesValid, categorizationA const percentageText = ; const analyzerUsed = ; - let color: 'success' | 'danger' | 'warning' = 'success'; + let color: EuiCallOutProps['color'] = 'success'; let title = i18n.translate( 'xpack.ml.newJob.wizard.pickFieldsStep.categorizationFieldCalloutTitle.valid', { diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx index 5850712068cda..7f9b2e43b9005 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx @@ -36,22 +36,22 @@ export const FieldExamples: FC = ({ fieldExamples }) => { ), }, ]; - const items = fieldExamples.map((example, e) => { + const items = fieldExamples.map((example, i) => { const txt = []; let tokenCounter = 0; let buffer = ''; - let i = 0; - while (i < example.text.length) { + let charCount = 0; + while (charCount < example.text.length) { const token = example.tokens[tokenCounter]; - if (token && i === token.start_offset) { + if (token && charCount === token.start_offset) { txt.push(buffer); buffer = ''; - txt.push({token.token}); - i += token.end_offset - token.start_offset; + txt.push({token.token}); + charCount += token.end_offset - token.start_offset; tokenCounter++; } else { - buffer += example.text[i]; - i++; + buffer += example.text[charCount]; + charCount++; } } txt.push(buffer); diff --git a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts index cf07179222051..34e871a936088 100644 --- a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts +++ b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts @@ -7,8 +7,7 @@ import { ML_RESULTS_INDEX_PATTERN } from '../../../../common/constants/index_patterns'; import { CATEGORY_EXAMPLES_MULTIPLIER } from '../../../../common/constants/new_job'; import { CategoryId, Category, Token } from '../../../../common/types/categories'; - -type callWithRequestType = (action: string, params: any) => Promise; +import { callWithRequestType } from '../../../../common/types/kibana'; export function categorizationExamplesProvider(callWithRequest: callWithRequestType) { async function categorizationExamples( diff --git a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/charts.ts b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/charts.ts index a8b6ba494a070..88ae8caa91e4a 100644 --- a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/charts.ts +++ b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/charts.ts @@ -6,7 +6,7 @@ import { newJobLineChartProvider } from './line_chart'; import { newJobPopulationChartProvider } from './population_chart'; -export type callWithRequestType = (action: string, params: any) => Promise; +import { callWithRequestType } from '../../../../common/types/kibana'; export function newJobChartsProvider(callWithRequest: callWithRequestType) { const { newJobLineChart } = newJobLineChartProvider(callWithRequest); diff --git a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/line_chart.ts b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/line_chart.ts index 5bb0f39982146..c1a5ad5e38ecc 100644 --- a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/line_chart.ts +++ b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/line_chart.ts @@ -6,10 +6,9 @@ import { get } from 'lodash'; import { AggFieldNamePair, EVENT_RATE_FIELD_ID } from '../../../../common/types/fields'; +import { callWithRequestType } from '../../../../common/types/kibana'; import { ML_MEDIAN_PERCENTS } from '../../../../common/util/job_utils'; -export type callWithRequestType = (action: string, params: any) => Promise; - type DtrIndex = number; type TimeStamp = number; type Value = number | undefined | null; diff --git a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/population_chart.ts b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/population_chart.ts index 812a135f6cf08..ee35f13c44ee6 100644 --- a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/population_chart.ts +++ b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/population_chart.ts @@ -6,10 +6,9 @@ import { get } from 'lodash'; import { AggFieldNamePair, EVENT_RATE_FIELD_ID } from '../../../../common/types/fields'; +import { callWithRequestType } from '../../../../common/types/kibana'; import { ML_MEDIAN_PERCENTS } from '../../../../common/util/job_utils'; -export type callWithRequestType = (action: string, params: any) => Promise; - const OVER_FIELD_EXAMPLES_COUNT = 40; type DtrIndex = number; From b5d4fca3c2ce9ccdf625ded22731dac066d49db4 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Wed, 8 Jan 2020 16:12:31 +0000 Subject: [PATCH 26/29] removing categorization_analyzer from job if it is same as default --- .../job_creator/categorization_job_creator.ts | 25 +++++++++++++------ .../edit_categorization_analyzer_flyout.tsx | 2 +- .../detector_cards.tsx | 2 +- .../examples_valid_callout.tsx | 2 +- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts index fc97f29071de8..e92bac9c9a21f 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { isEqual } from 'lodash'; import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public'; import { SavedSearchSavedObject } from '../../../../../../common/types/kibana'; import { JobCreator } from './job_creator'; @@ -19,9 +20,9 @@ import { import { ML_JOB_AGGREGATION } from '../../../../../../common/constants/aggregation_types'; import { getRichDetectors } from './util/general'; import { CategorizationExamplesLoader, CategoryExample } from '../results_loader'; -import { CategorizationAnalyzer } from '../../../../services/ml_server_info'; +import { CategorizationAnalyzer, getNewJobDefaults } from '../../../../services/ml_server_info'; -type CategorizationAnalyzerType = string | CategorizationAnalyzer | null; +type CategorizationAnalyzerType = CategorizationAnalyzer | null; export class CategorizationJobCreator extends JobCreator { protected _type: JOB_TYPE = JOB_TYPE.CATEGORIZATION; @@ -32,7 +33,8 @@ export class CategorizationJobCreator extends JobCreator { private _categoryFieldValid: number = 0; private _detectorType: ML_JOB_AGGREGATION.COUNT | ML_JOB_AGGREGATION.RARE = ML_JOB_AGGREGATION.COUNT; - private _categorization_analyzer: CategorizationAnalyzerType = null; + private _categorizationAnalyzer: CategorizationAnalyzerType = null; + private _defaultCategorizationAnalyzer: CategorizationAnalyzerType; constructor( indexPattern: IndexPattern, @@ -42,6 +44,9 @@ export class CategorizationJobCreator extends JobCreator { super(indexPattern, savedSearch, query); this.createdBy = CREATED_BY_LABEL.CATEGORIZATION; this._examplesLoader = new CategorizationExamplesLoader(this, indexPattern, query); + + const { anomaly_detectors: anomalyDetectors } = getNewJobDefaults(); + this._defaultCategorizationAnalyzer = anomalyDetectors.categorization_analyzer || null; } public setDefaultDetectorProperties( @@ -115,16 +120,20 @@ export class CategorizationJobCreator extends JobCreator { } public set categorizationAnalyzer(analyzer: CategorizationAnalyzerType) { - this._categorization_analyzer = analyzer; - if (analyzer !== null) { - this._job_config.analysis_config.categorization_analyzer = analyzer; - } else { + this._categorizationAnalyzer = analyzer; + + if ( + analyzer === null || + isEqual(this._categorizationAnalyzer, this._defaultCategorizationAnalyzer) + ) { delete this._job_config.analysis_config.categorization_analyzer; + } else { + this._job_config.analysis_config.categorization_analyzer = analyzer; } } public get categorizationAnalyzer() { - return this._categorization_analyzer; + return this._categorizationAnalyzer; } public cloneFromExistingJob(job: Job, datafeed: Datafeed) { diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx index c2ddfd35f4c13..a44cbf3d0c71a 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx @@ -75,7 +75,7 @@ export const EditCategorizationAnalyzerFlyout: FC = () => { diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/detector_cards.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/detector_cards.tsx index b863c69aca49c..bed60f1957bc0 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/detector_cards.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/detector_cards.tsx @@ -27,7 +27,7 @@ export const CountCard: FC = ({ onClick, isSelected }) => ( <> } diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx index 15684d2bf5939..04934d2dc9a36 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx @@ -18,7 +18,7 @@ import { CATEGORY_EXAMPLES_WARNING_LIMIT, } from '../../../../../../../../../common/constants/new_job'; -type CategorizationAnalyzerType = string | CategorizationAnalyzer | null; +type CategorizationAnalyzerType = CategorizationAnalyzer | null; interface Props { examplesValid: number; From 81ca64caf423e56700b0fe72903e86a8adc80cbb Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Wed, 8 Jan 2020 16:50:53 +0000 Subject: [PATCH 27/29] fixing translations --- .../components/categorization_detector/detector_cards.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/detector_cards.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/detector_cards.tsx index bed60f1957bc0..c4c2e4e9b55ff 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/detector_cards.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/detector_cards.tsx @@ -48,7 +48,7 @@ export const RareCard: FC = ({ onClick, isSelected }) => ( description={ <> From e869c4b41e9161bdaf9106019eb0716a7893efc6 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Wed, 8 Jan 2020 17:38:34 +0000 Subject: [PATCH 28/29] disabling model plot for rare jobs --- .../job_creator/categorization_job_creator.ts | 1 + .../components/model_plot/model_plot_switch.tsx | 16 +++++++++++++++- .../categorization_detector/detector_cards.tsx | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts index e92bac9c9a21f..cea99eb5ec64c 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts @@ -81,6 +81,7 @@ export class CategorizationJobCreator extends JobCreator { } else { this._createRareDetector(); this.bucketSpan = DEFAULT_RARE_BUCKET_SPAN; + this.modelPlot = false; } } diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/components/model_plot/model_plot_switch.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/components/model_plot/model_plot_switch.tsx index 64f9f450ae08d..a034bdcc2900f 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/components/model_plot/model_plot_switch.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/components/model_plot/model_plot_switch.tsx @@ -10,16 +10,29 @@ import { EuiSpacer, EuiSwitch } from '@elastic/eui'; import { JobCreatorContext } from '../../../../../job_creator_context'; import { Description } from './description'; import { MMLCallout } from '../mml_callout'; +import { ML_JOB_AGGREGATION } from '../../../../../../../../../../../common/constants/aggregation_types'; +import { isCategorizationJobCreator } from '../../../../../../../common/job_creator'; export const ModelPlotSwitch: FC = () => { - const { jobCreator, jobCreatorUpdate } = useContext(JobCreatorContext); + const { jobCreator, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext); const [modelPlotEnabled, setModelPlotEnabled] = useState(jobCreator.modelPlot); + const [enabled, setEnabled] = useState(false); useEffect(() => { jobCreator.modelPlot = modelPlotEnabled; jobCreatorUpdate(); }, [modelPlotEnabled]); + useEffect(() => { + const aggs = [ML_JOB_AGGREGATION.RARE]; + // disable model plot switch if the wizard is creating a categorization job + // and a rare detector is being used. + const isRareCategoryJob = + isCategorizationJobCreator(jobCreator) && + jobCreator.aggregations.some(agg => aggs.includes(agg.id)); + setEnabled(isRareCategoryJob === false); + }, [jobCreatorUpdated]); + function toggleModelPlot() { setModelPlotEnabled(!modelPlotEnabled); } @@ -29,6 +42,7 @@ export const ModelPlotSwitch: FC = () => { = ({ onClick, isSelected }) => ( <> } From 80b802f501c06aeb2b52e426d36aba4bb123c8f7 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Thu, 9 Jan 2020 11:01:27 +0000 Subject: [PATCH 29/29] removing console.error in useResolver --- .../plugins/ml/public/application/routing/use_resolver.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/application/routing/use_resolver.ts b/x-pack/legacy/plugins/ml/public/application/routing/use_resolver.ts index f74260c06567e..3716b9715bb5b 100644 --- a/x-pack/legacy/plugins/ml/public/application/routing/use_resolver.ts +++ b/x-pack/legacy/plugins/ml/public/application/routing/use_resolver.ts @@ -60,8 +60,6 @@ export const useResolver = ( } } catch (error) { // quietly fail. Let the resolvers handle the redirection if any fail to resolve - // eslint-disable-next-line no-console - console.error('ML page loading resolver', error); } })(); }, []);