Skip to content

feature(slo): Run transforms unattended #167170

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kdelemme merged 6 commits into from
Sep 26, 2023
Merged

feature(slo): Run transforms unattended #167170

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
8cb0892
Add unattended settings to all transforms
kdelemme Sep 25, 2023
0f860c7
Add preview method
kdelemme Sep 25, 2023
8db7f30
Remove error from log
kdelemme Sep 25, 2023
7b2b908
Add preview step in create and update
kdelemme Sep 25, 2023
5f10f2d
Add test
kdelemme Sep 25, 2023
4e0e82f
Add unattended into summary transforms
kdelemme Sep 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions x-pack/plugins/observability/server/assets/constants.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
*/

export const SLO_RESOURCES_VERSION = 2;
export const SLO_SUMMARY_TRANSFORMS_VERSION = 3;

export const SLO_COMPONENT_TEMPLATE_MAPPINGS_NAME = '.slo-observability.sli-mappings';
export const SLO_COMPONENT_TEMPLATE_SETTINGS_NAME = '.slo-observability.sli-settings';
Expand Down
1 change: 1 addition & 0 deletions x-pack/plugins/observability/server/assets/transform_templates/slo_transform_template.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ export const getSLOTransformTemplate = (
dest: destination,
settings: {
deduce_mappings: false,
unattended: true,
},
sync: {
time: {
Expand Down
11 changes: 11 additions & 0 deletions x-pack/plugins/observability/server/services/slo/create_slo.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ describe('CreateSLO', () => {
expect(mockTransformManager.install).toHaveBeenCalledWith(
expect.objectContaining({ ...sloParams, id: 'unique-id' })
);
expect(mockTransformManager.preview).toHaveBeenCalledWith('slo-transform-id');
expect(mockTransformManager.start).toHaveBeenCalledWith('slo-transform-id');
expect(response).toEqual(expect.objectContaining({ id: 'unique-id' }));
expect(esClientMock.index.mock.calls[0]).toMatchSnapshot();
Expand Down Expand Up @@ -100,6 +101,16 @@ describe('CreateSLO', () => {
expect(mockRepository.deleteById).toBeCalled();
});

it('removes the transform and deletes the SLO when transform preview fails', async () => {
mockTransformManager.install.mockResolvedValue('slo-transform-id');
mockTransformManager.preview.mockRejectedValue(new Error('Transform preview error'));
const sloParams = createSLOParams({ indicator: createAPMTransactionErrorRateIndicator() });

await expect(createSLO.execute(sloParams)).rejects.toThrowError('Transform preview error');
expect(mockTransformManager.uninstall).toBeCalledWith('slo-transform-id');
expect(mockRepository.deleteById).toBeCalled();
});

it('removes the transform and deletes the SLO when transform start fails', async () => {
mockTransformManager.install.mockResolvedValue('slo-transform-id');
mockTransformManager.start.mockRejectedValue(new Error('Transform start error'));
Expand Down
1 change: 1 addition & 0 deletions x-pack/plugins/observability/server/services/slo/create_slo.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ export class CreateSLO {
}

try {
await this.transformManager.preview(sloTransformId);
await this.transformManager.start(sloTransformId);
} catch (err) {
await Promise.all([
Expand Down
1 change: 1 addition & 0 deletions x-pack/plugins/observability/server/services/slo/mocks/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ const createSummaryTransformInstallerMock = (): jest.Mocked<SummaryTransformInst
const createTransformManagerMock = (): jest.Mocked<TransformManager> => {
return {
install: jest.fn(),
preview: jest.fn(),
uninstall: jest.fn(),
start: jest.fn(),
stop: jest.fn(),
Expand Down
15 changes: 10 additions & 5 deletions ...ver/services/slo/summary_transform/__snapshots__/summary_transform_installer.test.ts.snap
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 4 additions & 4 deletions ...lugins/observability/server/services/slo/summary_transform/summary_transform_installer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types';
import type { ElasticsearchClient, Logger } from '@kbn/core/server';
import {
SLO_RESOURCES_VERSION,
SLO_SUMMARY_TRANSFORMS_VERSION,
SLO_SUMMARY_TRANSFORM_NAME_PREFIX,
} from '../../../assets/constants';
import { retryTransientEsErrors } from '../../../utils/retry';
Expand All @@ -32,7 +32,7 @@ export class DefaultSummaryTransformInstaller implements SummaryTransformInstall
const alreadyInstalled =
summaryTransforms.count === allTransformIds.length &&
summaryTransforms.transforms.every(
(transform) => transform._meta?.version === SLO_RESOURCES_VERSION
(transform) => transform._meta?.version === SLO_SUMMARY_TRANSFORMS_VERSION
) &&
summaryTransforms.transforms.every((transform) => allTransformIds.includes(transform.id));

Expand All @@ -46,9 +46,9 @@ export class DefaultSummaryTransformInstaller implements SummaryTransformInstall
const transform = summaryTransforms.transforms.find((t) => t.id === transformId);

const transformAlreadyInstalled =
!!transform && transform._meta?.version === SLO_RESOURCES_VERSION;
!!transform && transform._meta?.version === SLO_SUMMARY_TRANSFORMS_VERSION;
const previousTransformAlreadyInstalled =
!!transform && transform._meta?.version !== SLO_RESOURCES_VERSION;
!!transform && transform._meta?.version !== SLO_SUMMARY_TRANSFORMS_VERSION;

if (transformAlreadyInstalled) {
this.logger.info(`SLO summary transform [${transformId}] already installed - skipping`);
Expand Down
6 changes: 3 additions & 3 deletions ...bility/server/services/slo/summary_transform/templates/summary_occurrences_30d_rolling.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@
import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types';
import {
SLO_DESTINATION_INDEX_PATTERN,
SLO_RESOURCES_VERSION,
SLO_SUMMARY_DESTINATION_INDEX_NAME,
SLO_SUMMARY_INGEST_PIPELINE_NAME,
SLO_SUMMARY_TRANSFORMS_VERSION,
SLO_SUMMARY_TRANSFORM_NAME_PREFIX,
} from '../../../../assets/constants';
import { groupBy } from './common';
Expand Down Expand Up @@ -143,10 +143,10 @@ export const SUMMARY_OCCURRENCES_30D_ROLLING: TransformPutTransformRequest = {
},
settings: {
deduce_mappings: false,
max_page_search_size: 8000,
unattended: true,
},
_meta: {
version: SLO_RESOURCES_VERSION,
version: SLO_SUMMARY_TRANSFORMS_VERSION,
managed: true,
managed_by: 'observability',
},
Expand Down
5 changes: 3 additions & 2 deletions ...ability/server/services/slo/summary_transform/templates/summary_occurrences_7d_rolling.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@
import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types';
import {
SLO_DESTINATION_INDEX_PATTERN,
SLO_RESOURCES_VERSION,
SLO_SUMMARY_DESTINATION_INDEX_NAME,
SLO_SUMMARY_INGEST_PIPELINE_NAME,
SLO_SUMMARY_TRANSFORMS_VERSION,
SLO_SUMMARY_TRANSFORM_NAME_PREFIX,
} from '../../../../assets/constants';
import { groupBy } from './common';
Expand Down Expand Up @@ -143,9 +143,10 @@ export const SUMMARY_OCCURRENCES_7D_ROLLING: TransformPutTransformRequest = {
},
settings: {
deduce_mappings: false,
unattended: true,
},
_meta: {
version: SLO_RESOURCES_VERSION,
version: SLO_SUMMARY_TRANSFORMS_VERSION,
managed: true,
managed_by: 'observability',
},
Expand Down
5 changes: 3 additions & 2 deletions ...bility/server/services/slo/summary_transform/templates/summary_occurrences_90d_rolling.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@
import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types';
import {
SLO_DESTINATION_INDEX_PATTERN,
SLO_RESOURCES_VERSION,
SLO_SUMMARY_DESTINATION_INDEX_NAME,
SLO_SUMMARY_INGEST_PIPELINE_NAME,
SLO_SUMMARY_TRANSFORMS_VERSION,
SLO_SUMMARY_TRANSFORM_NAME_PREFIX,
} from '../../../../assets/constants';
import { groupBy } from './common';
Expand Down Expand Up @@ -143,9 +143,10 @@ export const SUMMARY_OCCURRENCES_90D_ROLLING: TransformPutTransformRequest = {
},
settings: {
deduce_mappings: false,
unattended: true,
},
_meta: {
version: SLO_RESOURCES_VERSION,
version: SLO_SUMMARY_TRANSFORMS_VERSION,
managed: true,
managed_by: 'observability',
},
Expand Down
5 changes: 3 additions & 2 deletions ...ty/server/services/slo/summary_transform/templates/summary_occurrences_monthly_aligned.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@
import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types';
import {
SLO_DESTINATION_INDEX_PATTERN,
SLO_RESOURCES_VERSION,
SLO_SUMMARY_DESTINATION_INDEX_NAME,
SLO_SUMMARY_INGEST_PIPELINE_NAME,
SLO_SUMMARY_TRANSFORMS_VERSION,
SLO_SUMMARY_TRANSFORM_NAME_PREFIX,
} from '../../../../assets/constants';
import { groupBy } from './common';
Expand Down Expand Up @@ -141,9 +141,10 @@ export const SUMMARY_OCCURRENCES_MONTHLY_ALIGNED: TransformPutTransformRequest =
},
settings: {
deduce_mappings: false,
unattended: true,
},
_meta: {
version: SLO_RESOURCES_VERSION,
version: SLO_SUMMARY_TRANSFORMS_VERSION,
managed: true,
managed_by: 'observability',
},
Expand Down
5 changes: 3 additions & 2 deletions ...ity/server/services/slo/summary_transform/templates/summary_occurrences_weekly_aligned.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@
import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types';
import {
SLO_DESTINATION_INDEX_PATTERN,
SLO_RESOURCES_VERSION,
SLO_SUMMARY_DESTINATION_INDEX_NAME,
SLO_SUMMARY_INGEST_PIPELINE_NAME,
SLO_SUMMARY_TRANSFORMS_VERSION,
SLO_SUMMARY_TRANSFORM_NAME_PREFIX,
} from '../../../../assets/constants';
import { groupBy } from './common';
Expand Down Expand Up @@ -141,9 +141,10 @@ export const SUMMARY_OCCURRENCES_WEEKLY_ALIGNED: TransformPutTransformRequest =
},
settings: {
deduce_mappings: false,
unattended: true,
},
_meta: {
version: SLO_RESOURCES_VERSION,
version: SLO_SUMMARY_TRANSFORMS_VERSION,
managed: true,
managed_by: 'observability',
},
Expand Down
5 changes: 3 additions & 2 deletions ...ability/server/services/slo/summary_transform/templates/summary_timeslices_30d_rolling.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@
import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types';
import {
SLO_DESTINATION_INDEX_PATTERN,
SLO_RESOURCES_VERSION,
SLO_SUMMARY_DESTINATION_INDEX_NAME,
SLO_SUMMARY_INGEST_PIPELINE_NAME,
SLO_SUMMARY_TRANSFORMS_VERSION,
SLO_SUMMARY_TRANSFORM_NAME_PREFIX,
} from '../../../../assets/constants';
import { groupBy } from './common';
Expand Down Expand Up @@ -143,9 +143,10 @@ export const SUMMARY_TIMESLICES_30D_ROLLING: TransformPutTransformRequest = {
},
settings: {
deduce_mappings: false,
unattended: true,
},
_meta: {
version: SLO_RESOURCES_VERSION,
version: SLO_SUMMARY_TRANSFORMS_VERSION,
managed: true,
managed_by: 'observability',
},
Expand Down
5 changes: 3 additions & 2 deletions ...vability/server/services/slo/summary_transform/templates/summary_timeslices_7d_rolling.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@
import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types';
import {
SLO_DESTINATION_INDEX_PATTERN,
SLO_RESOURCES_VERSION,
SLO_SUMMARY_DESTINATION_INDEX_NAME,
SLO_SUMMARY_INGEST_PIPELINE_NAME,
SLO_SUMMARY_TRANSFORMS_VERSION,
SLO_SUMMARY_TRANSFORM_NAME_PREFIX,
} from '../../../../assets/constants';
import { groupBy } from './common';
Expand Down Expand Up @@ -143,9 +143,10 @@ export const SUMMARY_TIMESLICES_7D_ROLLING: TransformPutTransformRequest = {
},
settings: {
deduce_mappings: false,
unattended: true,
},
_meta: {
version: SLO_RESOURCES_VERSION,
version: SLO_SUMMARY_TRANSFORMS_VERSION,
managed: true,
managed_by: 'observability',
},
Expand Down
5 changes: 3 additions & 2 deletions ...ability/server/services/slo/summary_transform/templates/summary_timeslices_90d_rolling.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@
import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types';
import {
SLO_DESTINATION_INDEX_PATTERN,
SLO_RESOURCES_VERSION,
SLO_SUMMARY_DESTINATION_INDEX_NAME,
SLO_SUMMARY_INGEST_PIPELINE_NAME,
SLO_SUMMARY_TRANSFORMS_VERSION,
SLO_SUMMARY_TRANSFORM_NAME_PREFIX,
} from '../../../../assets/constants';
import { groupBy } from './common';
Expand Down Expand Up @@ -143,9 +143,10 @@ export const SUMMARY_TIMESLICES_90D_ROLLING: TransformPutTransformRequest = {
},
settings: {
deduce_mappings: false,
unattended: true,
},
_meta: {
version: SLO_RESOURCES_VERSION,
version: SLO_SUMMARY_TRANSFORMS_VERSION,
managed: true,
managed_by: 'observability',
},
Expand Down
5 changes: 3 additions & 2 deletions ...ity/server/services/slo/summary_transform/templates/summary_timeslices_monthly_aligned.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@
import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types';
import {
SLO_DESTINATION_INDEX_PATTERN,
SLO_RESOURCES_VERSION,
SLO_SUMMARY_DESTINATION_INDEX_NAME,
SLO_SUMMARY_INGEST_PIPELINE_NAME,
SLO_SUMMARY_TRANSFORMS_VERSION,
SLO_SUMMARY_TRANSFORM_NAME_PREFIX,
} from '../../../../assets/constants';
import { groupBy } from './common';
Expand Down Expand Up @@ -171,9 +171,10 @@ export const SUMMARY_TIMESLICES_MONTHLY_ALIGNED: TransformPutTransformRequest =
},
settings: {
deduce_mappings: false,
unattended: true,
},
_meta: {
version: SLO_RESOURCES_VERSION,
version: SLO_SUMMARY_TRANSFORMS_VERSION,
managed: true,
managed_by: 'observability',
},
Expand Down
Loading