Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -460,6 +460,16 @@ The level of issue which can fail the test (and cause it to exit with code 2) wh

Defaults to `error`. Note this configuration is only available when using Pa11y on the command line, not via the JavaScript Interface.

### `includeIncomplete` (boolean)

Whether to include axe-core's "incomplete" results (items that require manual review) alongside definitive violations. Defaults to `true` for backward compatibility. Set to `false` to exclude incomplete results, which can produce false positives on pages with rich content (math symbols, pseudo-elements, SVG foreignObject, etc.). Only used by the axe runner.

```json
{
"includeIncomplete": false
}
```

### `levelCapWhenNeedsReview` (string)

Cap any issue requiring manual review to this level. This should be one of `error` (the default), `warning`, or `notice`. Only used by the axe runner.
Expand Down
5 changes: 5 additions & 0 deletions bin/pa11y.js
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,10 @@ function configureProgram() {
'--include-warnings',
'include warnings in the report'
)
.option(
'--no-include-incomplete',
'(axe-only) exclude incomplete results that require manual review'
)
.option(
'--level-cap-when-needs-review <level>',
'(axe-only) cap severity of any issue requiring manual review to: ' +
Expand Down Expand Up @@ -168,6 +172,7 @@ function processOptions() {
}, loadConfig(programOptions.config), {
hideElements: programOptions.hideElements,
ignore: (programOptions.ignore.length ? programOptions.ignore : undefined),
includeIncomplete: programOptions.includeIncomplete,
includeNotices: programOptions.includeNotices,
includeWarnings: programOptions.includeWarnings,
level: programOptions.level,
Expand Down
3 changes: 3 additions & 0 deletions lib/pa11y.js
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,7 @@ async function injectRunners(options, state) {
const runPa11yWithOptions = ({
hideElements,
ignore,
includeIncomplete,
levelCapWhenNeedsReview,
rootElement,
rules,
Expand All @@ -344,6 +345,7 @@ const runPa11yWithOptions = ({
pa11yVersion,
hideElements,
ignore,
includeIncomplete,
levelCapWhenNeedsReview,
rootElement,
rules,
Expand Down Expand Up @@ -484,6 +486,7 @@ pa11y.defaults = {
hideElements: null,
ignore: [],
ignoreUrl: false,
includeIncomplete: true,
includeNotices: false,
includeWarnings: false,
log: {
Expand Down
14 changes: 9 additions & 5 deletions lib/runners/axe.js
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,10 @@ const run = async options => {
* @param {Pa11yConfiguration} options Pa11y's configuration
* @returns {Promise<Array<Pa11yResult>>} A promise of axe issues translated for Pa11y
*/
async function runAxeCore({standard, rules, ignore, rootElement, levelCapWhenNeedsReview}) {
async function runAxeCore({
standard, rules, ignore, rootElement,
includeIncomplete, levelCapWhenNeedsReview
}) {
const {
violations = [],
incomplete = []
Expand All @@ -197,10 +200,11 @@ const run = async options => {
})
);

return [
...violations.flatMap(issue => processIssue(issue, false, levelCapWhenNeedsReview)),
...incomplete.flatMap(issue => processIssue(issue, true, levelCapWhenNeedsReview))
];
const issues = [...violations.flatMap(issue => processIssue(issue, false, levelCapWhenNeedsReview))];
if (includeIncomplete !== false) {
issues.push(...incomplete.flatMap(issue => processIssue(issue, true, levelCapWhenNeedsReview)));
}
return issues;
}

return await runAxeCore(options);
Expand Down
5 changes: 5 additions & 0 deletions test/unit/lib/pa11y.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ describe('lib/pa11y', function() {
'notice',
'warning'
],
includeIncomplete: true,
levelCapWhenNeedsReview: 'error',
pa11yVersion: pkg.version,
rootElement: pa11y.defaults.rootElement,
Expand Down Expand Up @@ -1192,6 +1193,10 @@ describe('lib/pa11y', function() {
assert.isFalse(pa11y.defaults.ignoreUrl);
});

it('has an `includeIncomplete` property', function() {
assert.isTrue(pa11y.defaults.includeIncomplete);
});

it('has an `includeNotices` property', function() {
assert.isFalse(pa11y.defaults.includeNotices);
});
Expand Down
50 changes: 50 additions & 0 deletions test/unit/lib/runners/axe.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,56 @@ describe('lib/runners/axe', function() {
});
});

describe('includeIncomplete', function() {
describe('when set to false', function() {
beforeEach(async function() {
options.includeIncomplete = false;
resolvedValue = await runner.run(options, pa11y);
});

it('excludes incomplete issues from the results', function() {
const incompleteIssues = resolvedValue.filter(
issue => issue.runnerExtras.needsFurtherReview === true
);
assert.strictEqual(incompleteIssues.length, 0);
});

it('still includes violation issues', function() {
const violationIssues = resolvedValue.filter(
issue => issue.runnerExtras.needsFurtherReview === false
);
assert.greaterThan(violationIssues.length, 0);
});
});

describe('when set to true (explicit)', function() {
beforeEach(async function() {
options.includeIncomplete = true;
resolvedValue = await runner.run(options, pa11y);
});

it('includes incomplete issues in the results', function() {
const incompleteIssues = resolvedValue.filter(
issue => issue.runnerExtras.needsFurtherReview === true
);
assert.greaterThan(incompleteIssues.length, 0);
});
});

describe('when left unset, defaults to including incomplete', function() {
beforeEach(async function() {
resolvedValue = await runner.run(options, pa11y);
});

it('includes incomplete issues in the results', function() {
const incompleteIssues = resolvedValue.filter(
issue => issue.runnerExtras.needsFurtherReview === true
);
assert.greaterThan(incompleteIssues.length, 0);
});
});
});

describe('levelCapWhenNeedsReview', function() {
describe('when set to "warning"', function() {
beforeEach(async function() {
Expand Down