Skip to content

[ML] Skip invalid modules when data recognizer lists matches #33703

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
peteharverson merged 2 commits into from
Mar 22, 2019
Merged

[ML] Skip invalid modules when data recognizer lists matches #33703

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
29dbe9b
[ML] Skip invalid modules when data recognizer lists matches
peteharverson Mar 22, 2019
dfd463e
[ML] Append error to data recognizer module load console logs
peteharverson Mar 22, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions x-pack/plugins/ml/public/jobs/new_job/simple/recognize/create_job/create_job_controller.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import { CreateRecognizerJobsServiceProvider } from './create_job_service';
import { mlMessageBarService } from 'plugins/ml/components/messagebar/messagebar_service';
import { ml } from 'plugins/ml/services/ml_api_service';
import template from './create_job.html';
import { toastNotifications } from 'ui/notify';
import { timefilter } from 'ui/timefilter';

uiRoutes
Expand Down Expand Up @@ -358,6 +359,23 @@ module
});
}
resolve();
})
.catch((err) => {
console.log('Error setting up module', err);
toastNotifications.addWarning({
title: i18n('xpack.ml.newJob.simple.recognize.moduleSetupFailedWarningTitle', {
defaultMessage: 'Error setting up module {moduleId}',
values: { moduleId }
}),
text: i18n('xpack.ml.newJob.simple.recognize.moduleSetupFailedWarningDescription', {
defaultMessage: 'An error occurred trying to create the {count, plural, one {job} other {jobs}} in the module.',
values: {
count: $scope.formConfig.jobs.length
}
})
});
$scope.overallState = SAVE_STATE.FAILED;
$scope.$applyAsync();
});
});
}
Expand Down
93 changes: 61 additions & 32 deletions x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,15 +63,24 @@ export class DataRecognizer {
const configs = [];
const dirs = await this.listDirs(this.modulesDir);
await Promise.all(dirs.map(async (dir) => {
const file = await this.readFile(`${this.modulesDir}/${dir}/manifest.json`);
let file;
try {
configs.push({
dirName: dir,
json: JSON.parse(file)
});
file = await this.readFile(`${this.modulesDir}/${dir}/manifest.json`);
} catch (error) {
mlLog('warning', `Error parsing ${dir}/manifest.json`);
mlLog('warning', `Data recognizer skipping folder ${dir} as manifest.json cannot be read`);
}

if (file !== undefined) {
try {
configs.push({
dirName: dir,
json: JSON.parse(file)
});
} catch (error) {
mlLog('warning', `Data recognizer error parsing ${dir}/manifest.json. ${error}`);
}
}

}));

return configs;
Expand All @@ -90,8 +99,14 @@ export class DataRecognizer {

await Promise.all(manifestFiles.map(async (i) => {
const moduleConfig = i.json;
const match = await this.searchForFields(moduleConfig, indexPattern);
if (match) {
let match = false;
try {
match = await this.searchForFields(moduleConfig, indexPattern);
} catch (error) {
mlLog('warning', `Data recognizer error running query defined for module ${moduleConfig.id}. ${error}`);
}

if (match === true) {
let logo = null;
if (moduleConfig.logoFile) {
try {
Expand Down Expand Up @@ -131,6 +146,7 @@ export class DataRecognizer {
size,
body
});

return (resp.hits.total !== 0);
}

Expand All @@ -155,25 +171,33 @@ export class DataRecognizer {
const kibana = {};
// load all of the job configs
await Promise.all(manifestJSON.jobs.map(async (job) => {
const jobConfig = await this.readFile(`${this.modulesDir}/${dirName}/${ML_DIR}/${job.file}`);
// use the file name for the id
jobs.push({
id: `${prefix}${job.id}`,
config: JSON.parse(jobConfig)
});
try {
const jobConfig = await this.readFile(`${this.modulesDir}/${dirName}/${ML_DIR}/${job.file}`);
// use the file name for the id
jobs.push({
id: `${prefix}${job.id}`,
config: JSON.parse(jobConfig)
});
} catch (error) {

@alvarezmelissa87 alvarezmelissa87 Mar 22, 2019

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would it be useful to log out the actual error being caught somewhere?

@peteharverson peteharverson Mar 22, 2019

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, good point. I'll append the error to these log messages.

mlLog('warning', `Data recognizer error loading config for job ${job.id} for module ${id}. ${error}`);
}
}));

// load all of the datafeed configs
await Promise.all(manifestJSON.datafeeds.map(async (datafeed) => {
const datafeedConfig = await this.readFile(`${this.modulesDir}/${dirName}/${ML_DIR}/${datafeed.file}`);
const config = JSON.parse(datafeedConfig);
// use the job id from the manifestFile
config.job_id = `${prefix}${datafeed.job_id}`;

datafeeds.push({
id: prefixDatafeedId(datafeed.id, prefix),
config
});
try {
const datafeedConfig = await this.readFile(`${this.modulesDir}/${dirName}/${ML_DIR}/${datafeed.file}`);
const config = JSON.parse(datafeedConfig);
// use the job id from the manifestFile
config.job_id = `${prefix}${datafeed.job_id}`;

datafeeds.push({
id: prefixDatafeedId(datafeed.id, prefix),
config
});
} catch (error) {
mlLog('warning', `Data recognizer error loading config for datafeed ${datafeed.id} for module ${id}. ${error}`);
}
}));

// load all of the kibana saved objects
Expand All @@ -182,15 +206,19 @@ export class DataRecognizer {
await Promise.all(kKeys.map(async (key) => {
kibana[key] = [];
await Promise.all(manifestJSON.kibana[key].map(async (obj) => {
const kConfig = await this.readFile(`${this.modulesDir}/${dirName}/${KIBANA_DIR}/${key}/${obj.file}`);
// use the file name for the id
const kId = obj.file.replace('.json', '');
const config = JSON.parse(kConfig);
kibana[key].push({
id: kId,
title: config.title,
config
});
try {
const kConfig = await this.readFile(`${this.modulesDir}/${dirName}/${KIBANA_DIR}/${key}/${obj.file}`);
// use the file name for the id
const kId = obj.file.replace('.json', '');
const config = JSON.parse(kConfig);
kibana[key].push({
id: kId,
title: config.title,
config
});
} catch (error) {
mlLog('warning', `Data recognizer error loading config for ${key} ${obj.id} for module ${id}. ${error}`);
}
}));
}));
}
Expand Down Expand Up @@ -218,6 +246,7 @@ export class DataRecognizer {
end,
request
) {

this.savedObjectsClient = request.getSavedObjectsClient();
this.indexPatterns = await this.loadIndexPatterns();

Expand Down