diff --git a/axe.d.ts b/axe.d.ts index 5940029b8e..3aea04a38e 100644 --- a/axe.d.ts +++ b/axe.d.ts @@ -155,7 +155,7 @@ declare namespace axe { toolOptions: RunOptions; passes: Result[]; violations: Result[]; - incomplete: Result[]; + incomplete: IncompleteResult[]; inapplicable: Result[]; } interface Result { @@ -167,6 +167,9 @@ declare namespace axe { tags: TagValue[]; nodes: NodeResult[]; } + interface IncompleteResult extends Result { + error: Omit; + } interface NodeResult { html: string; impact?: ImpactValue; @@ -204,6 +207,21 @@ declare namespace axe { fail: string | { [key: string]: string }; incomplete?: string | { [key: string]: string }; } + interface SupportError { + name: string; + message: string; + stack: string; + ruleId?: string; + method?: string; + cause?: SerialError; + errorNode?: DqElement; + } + interface SerialError { + message: string; + stack: string; + name: string; + cause?: SerialError; + } interface CheckLocale { [key: string]: CheckMessages; } @@ -461,7 +479,13 @@ declare namespace axe { isLabelledShadowDomSelector: ( selector: unknown ) => selector is LabelledShadowDomSelector; - + SupportError: ( + error: Error, + ruleId?: string, + method?: string, + errorNode?: DqElement + ) => SupportError; + serializeError: (error: Error) => SerialError; DqElement: DqElementConstructor; uuid: ( options?: { random?: Uint8Array | Array }, diff --git a/lib/checks/shared/error-occurred.json b/lib/checks/shared/error-occurred.json new file mode 100644 index 0000000000..7d8c614463 --- /dev/null +++ b/lib/checks/shared/error-occurred.json @@ -0,0 +1,10 @@ +{ + "id": "error-occurred", + "evaluate": "exists-evaluate", + "metadata": { + "messages": { + "pass": "", + "incomplete": "Axe encountered an error; test the page for this type of problem manually" + } + } +} diff --git a/lib/core/base/audit.js b/lib/core/base/audit.js index 36bcb325ff..65d22c4167 100644 --- a/lib/core/base/audit.js +++ b/lib/core/base/audit.js @@ -9,7 +9,8 @@ import { preload, findBy, ruleShouldRun, - performanceTimer + performanceTimer, + serializeError } from '../utils'; import doT from '@deque/dot'; import constants from '../constants'; @@ -368,6 +369,9 @@ export default class Audit { after(results, options) { const rules = this.rules; return results.map(ruleResult => { + if (ruleResult.error) { + return ruleResult; + } const rule = findBy(rules, 'id', ruleResult.id); if (!rule) { // If you see this, you're probably running the Mocha tests with the axe extension installed @@ -375,7 +379,14 @@ export default class Audit { 'Result for unknown rule. You may be running mismatch axe-core versions' ); } - return rule.after(ruleResult, options); + try { + return rule.after(ruleResult, options); + } catch (err) { + if (options.debug) { + throw err; + } + return createIncompleteErrorResult(rule, err); + } }); } /** @@ -732,36 +743,37 @@ function getDefferedRule(rule, context, options) { rule.run( context, options, - // resolve callback for rule `run` - ruleResult => { - // resolve - resolve(ruleResult); - }, - // reject callback for rule `run` + ruleResult => resolve(ruleResult), err => { - // if debug - construct error details - if (!options.debug) { - const errResult = Object.assign(new RuleResult(rule), { - result: constants.CANTTELL, - description: 'An error occured while running this rule', - message: err.message, - stack: err.stack, - error: err, - // Add a serialized reference to the node the rule failed on for easier debugging. - // See https://github.com/dequelabs/axe-core/issues/1317. - errorNode: err.errorNode - }); - // resolve - resolve(errResult); - } else { - // reject + if (options.debug) { reject(err); + } else { + resolve(createIncompleteErrorResult(rule, err)); } } ); }; } +function createIncompleteErrorResult(rule, error) { + const { errorNode } = error; + const serialError = serializeError(error); + const none = [ + { + id: 'error-occurred', + result: undefined, + data: serialError, + relatedNodes: [] + } + ]; + const node = errorNode || new DqElement(document.documentElement); + return Object.assign(new RuleResult(rule), { + error: serialError, + result: constants.CANTTELL, + nodes: [{ any: [], all: [], none, node }] + }); +} + /** * For all the rules, create the helpUrl and add it to the data for that rule */ diff --git a/lib/core/base/rule.js b/lib/core/base/rule.js index 95d68ec2d5..de8ec71936 100644 --- a/lib/core/base/rule.js +++ b/lib/core/base/rule.js @@ -1,4 +1,3 @@ -/*global SupportError */ import { createExecutionContext } from './check'; import RuleResult from './rule-result'; import { @@ -8,7 +7,8 @@ import { queue, DqElement, select, - assert + assert, + SupportError } from '../utils'; import { isVisibleToScreenReaders } from '../../commons/dom'; import constants from '../constants'; @@ -181,7 +181,16 @@ Rule.prototype.runChecks = function runChecks( const check = self._audit.checks[c.id || c]; const option = getCheckOption(check, self.id, options); checkQueue.defer((res, rej) => { - check.run(node, option, context, res, rej); + check.run(node, option, context, res, error => { + rej( + new SupportError({ + ruleId: self.id, + method: `${check.id}#evaluate`, + errorNode: new DqElement(node), + error + }) + ); + }); }); }); @@ -235,8 +244,7 @@ Rule.prototype.run = function run(context, options = {}, resolve, reject) { // Matches throws an error when it lacks support for document methods nodes = this.gatherAndMatchNodes(context, options); } catch (error) { - // Exit the rule execution if matches fails - reject(new SupportError({ cause: error, ruleId: this.id })); + reject(error); return; } @@ -312,15 +320,7 @@ Rule.prototype.runSync = function runSync(context, options = {}) { } const ruleResult = new RuleResult(this); - let nodes; - - try { - nodes = this.gatherAndMatchNodes(context, options); - } catch (error) { - // Exit the rule execution if matches fails - throw new SupportError({ cause: error, ruleId: this.id }); - } - + const nodes = this.gatherAndMatchNodes(context, options); if (options.performanceTimer) { this._logGatherPerformance(nodes); } @@ -451,7 +451,18 @@ Rule.prototype.gatherAndMatchNodes = function gatherAndMatchNodes( performanceTimer.mark(markMatchesStart); } - nodes = nodes.filter(node => this.matches(node.actualNode, node, context)); + nodes = nodes.filter(node => { + try { + return this.matches(node.actualNode, node, context); + } catch (error) { + throw new SupportError({ + ruleId: this.id, + method: `#matches`, + errorNode: new DqElement(node.actualNode), + error + }); + } + }); if (options.performanceTimer) { performanceTimer.mark(markMatchesEnd); @@ -542,12 +553,20 @@ function sanitizeNodes(result) { */ Rule.prototype.after = function after(result, options) { const afterChecks = findAfterChecks(this); - const ruleID = this.id; afterChecks.forEach(check => { const beforeResults = findCheckResults(result.nodes, check.id); - const checkOption = getCheckOption(check, ruleID, options); - - const afterResults = check.after(beforeResults, checkOption.options); + const checkOption = getCheckOption(check, this.id, options); + let afterResults; + try { + afterResults = check.after(beforeResults, checkOption.options); + } catch (error) { + throw new SupportError({ + ruleId: this.id, + method: `${check.id}#after`, + errorNode: result.nodes?.[0].node, + error + }); + } if (this.reviewOnFail) { afterResults.forEach(checkResult => { diff --git a/lib/core/index.js b/lib/core/index.js index 2d15d52824..9529707189 100644 --- a/lib/core/index.js +++ b/lib/core/index.js @@ -25,16 +25,3 @@ if (typeof window.getComputedStyle === 'function') { } // local namespace for common functions let commons; - -function SupportError(error) { - this.name = 'SupportError'; - this.cause = error.cause; - this.message = `\`${error.cause}\` - feature unsupported in your environment.`; - if (error.ruleId) { - this.ruleId = error.ruleId; - this.message += ` Skipping ${this.ruleId} rule.`; - } - this.stack = new Error().stack; -} -SupportError.prototype = Object.create(Error.prototype); -SupportError.prototype.constructor = SupportError; diff --git a/lib/core/utils/index.js b/lib/core/utils/index.js index 81164587e2..0e228b4c73 100644 --- a/lib/core/utils/index.js +++ b/lib/core/utils/index.js @@ -87,6 +87,8 @@ export { default as ruleShouldRun } from './rule-should-run'; export { default as filterHtmlAttrs } from './filter-html-attrs'; export { default as select } from './select'; export { default as sendCommandToFrame } from './send-command-to-frame'; +export { default as serializeError } from './serialize-error'; +export { default as SupportError } from './support-error'; export { default as setScrollState } from './set-scroll-state'; export { default as shadowSelect } from './shadow-select'; export { default as shadowSelectAll } from './shadow-select-all'; diff --git a/lib/core/utils/merge-results.js b/lib/core/utils/merge-results.js index 717eeb198a..f12f413479 100644 --- a/lib/core/utils/merge-results.js +++ b/lib/core/utils/merge-results.js @@ -95,6 +95,10 @@ function mergeResults(frameResults, options) { if (ruleResult.nodes.length) { spliceNodes(res.nodes, ruleResult.nodes); } + if (ruleResult.error) { + // TODO: Test me + res.error ??= ruleResult.error; + } } }); }); diff --git a/lib/core/utils/serialize-error.js b/lib/core/utils/serialize-error.js new file mode 100644 index 0000000000..14f647c54a --- /dev/null +++ b/lib/core/utils/serialize-error.js @@ -0,0 +1,23 @@ +/** + * Serializes an error to a JSON object + * @param e - The error to serialize + * @returns A JSON object representing the error + */ +export default function serializeError(err, iteration = 0) { + if (typeof err !== 'object' || err === null) { + return { message: String(err) }; + } + const serial = { ...err }; // Copy all "own" properties + // Copy error.message / name / stack, these don't serialize otherwise + for (const prop of ['message', 'stack', 'name']) { + if (typeof err[prop] === 'string') { + serial[prop] = err[prop]; + } + } + // Recursively serialize cause up to 10 levels deep + if (err.cause) { + serial.cause = + iteration < 10 ? serializeError(err.cause, iteration + 1) : '...'; + } + return serial; +} diff --git a/lib/core/utils/support-error.js b/lib/core/utils/support-error.js new file mode 100644 index 0000000000..66b2adad6b --- /dev/null +++ b/lib/core/utils/support-error.js @@ -0,0 +1,23 @@ +import serializeError from './serialize-error'; + +export default class SupportError extends Error { + constructor({ error, ruleId, method, errorNode }) { + super(); + this.name = error.name ?? 'SupportError'; + this.message = error.message; + this.stack = error.stack; + if (error.cause) { + this.cause = serializeError(error.cause); + } + if (ruleId) { + this.ruleId = ruleId; + this.message += ` Skipping ${this.ruleId} rule.`; + } + if (method) { + this.method = method; + } + if (errorNode) { + this.errorNode = errorNode; + } + } +} diff --git a/locales/_template.json b/locales/_template.json index 05ff09b929..f6239a1f92 100644 --- a/locales/_template.json +++ b/locales/_template.json @@ -984,6 +984,10 @@ "pass": "Document has a non-empty element", "fail": "Document does not have a non-empty <title> element" }, + "error-occurred": { + "pass": "", + "incomplete": "Axe encountered an error; test the page for this type of problem manually" + }, "exists": { "pass": "Element does not exist", "incomplete": "Element exists" diff --git a/test/core/base/audit.js b/test/core/base/audit.js index b04002874d..3c08242b25 100644 --- a/test/core/base/audit.js +++ b/test/core/base/audit.js @@ -2,37 +2,54 @@ describe('Audit', () => { const Audit = axe._thisWillBeDeletedDoNotUse.base.Audit; const Rule = axe._thisWillBeDeletedDoNotUse.base.Rule; const ver = axe.version.substring(0, axe.version.lastIndexOf('.')); - const { fixtureSetup } = axe.testUtils; + const { fixtureSetup, captureError } = axe.testUtils; let audit; - const isNotCalled = function (err) { + const isNotCalled = err => { throw err || new Error('Reject should not be called'); }; const noop = () => {}; + const assertEqualSupportError = (actual, expect) => { + assert.include(actual.message, expect.message); + assert.equal(actual.stack, expect.stack); + assert.equal(actual.name, expect.name); + }; + + const assertErrorResults = (result, error, selector) => { + assert.equal(result.result, 'cantTell'); + assertEqualSupportError(result.error, error); + + assert.lengthOf(result.nodes, 1); + const node1 = result.nodes[0]; + assert.isEmpty(node1.any); + assert.isEmpty(node1.all); + assert.include(node1.node.selector, selector); + + assert.lengthOf(node1.none, 1); + const none = node1.none[0]; + assert.equal(none.id, 'error-occurred'); + assert.equal(none.result, undefined); + assert.isDefined(none.data); + assertEqualSupportError(none.data, error); + assert.lengthOf(none.relatedNodes, 0); + }; + const mockChecks = [ { id: 'positive1-check1', - evaluate: () => { - return true; - } + evaluate: () => true }, { id: 'positive2-check1', - evaluate: () => { - return true; - } + evaluate: () => true }, { id: 'negative1-check1', - evaluate: () => { - return true; - } + evaluate: () => true }, { id: 'positive3-check1', - evaluate: () => { - return true; - } + evaluate: () => true } ]; @@ -68,9 +85,7 @@ describe('Audit', () => { ]; const fixture = document.getElementById('fixture'); - let origAuditRun; - beforeEach(() => { audit = new Audit(); mockRules.forEach(function (r) { @@ -1139,45 +1154,6 @@ describe('Audit', () => { ); }); - it('catches errors and passes them as a cantTell result', done => { - const err = new Error('Launch the super sheep!'); - audit.addRule({ - id: 'throw1', - selector: '*', - any: [ - { - id: 'throw1-check1' - } - ] - }); - audit.addCheck({ - id: 'throw1-check1', - evaluate: () => { - throw err; - } - }); - axe._tree = axe.utils.getFlattenedTree(fixture); - axe._selectorData = axe.utils.getSelectorData(axe._tree); - audit.run( - { include: [axe._tree[0]] }, - { - runOnly: { - type: 'rule', - values: ['throw1'] - } - }, - function (results) { - assert.lengthOf(results, 1); - assert.equal(results[0].result, 'cantTell'); - assert.equal(results[0].message, err.message); - assert.equal(results[0].stack, err.stack); - assert.equal(results[0].error, err); - done(); - }, - isNotCalled - ); - }); - it('should not halt if errors occur', done => { audit.addRule({ id: 'throw1', @@ -1230,43 +1206,6 @@ describe('Audit', () => { assert.equal(checked, 'options validated'); }); - it('should halt if an error occurs when debug is set', done => { - audit.addRule({ - id: 'throw1', - selector: '*', - any: [ - { - id: 'throw1-check1' - } - ] - }); - audit.addCheck({ - id: 'throw1-check1', - evaluate: () => { - throw new Error('Launch the super sheep!'); - } - }); - - // check error node requires _selectorCache to be setup - axe.setup(); - - audit.run( - { include: [axe.utils.getFlattenedTree(fixture)[0]] }, - { - debug: true, - runOnly: { - type: 'rule', - values: ['throw1'] - } - }, - noop, - function (err) { - assert.equal(err.message, 'Launch the super sheep!'); - done(); - } - ); - }); - it('propagates DqElement options', async () => { fixtureSetup('<input id="input">'); const results = await new Promise((resolve, reject) => { @@ -1281,6 +1220,59 @@ describe('Audit', () => { assert.equal(node.element, fixture.firstChild); assert.equal(node.selector, 'html > body > #fixture > #input'); }); + + describe('when an error occurs', () => { + let err; + beforeEach(() => { + err = new Error('Launch the super sheep!'); + audit.addRule({ + id: 'throw1', + selector: '#fixture', + any: [ + { + id: 'throw1-check1' + } + ] + }); + audit.addCheck({ + id: 'throw1-check1', + evaluate: () => { + throw err; + } + }); + axe.setup(); + }); + + it('catches errors and resolves them as a cantTell result', done => { + audit.run( + { include: [axe._tree[0]] }, + { runOnly: { type: 'rule', values: ['throw1'] } }, + captureError(results => { + assert.lengthOf(results, 1); + assertErrorResults(results[0], err, '#fixture'); + done(); + }, done), + isNotCalled + ); + }); + + it('should halt if an error occurs when debug is set', done => { + const context = { include: [axe.utils.getFlattenedTree(fixture)[0]] }; + const options = { + debug: true, + runOnly: { type: 'rule', values: ['throw1'] } + }; + audit.run( + context, + options, + noop, + captureError(reject => { + assert.include(reject.message, err.message); + done(); + }, done) + ); + }); + }); }); describe('Audit#after', () => { @@ -1288,7 +1280,7 @@ describe('Audit', () => { /*eslint no-unused-vars:0*/ audit = new Audit(); let success = false; - const options = [{ id: 'hehe', enabled: true, monkeys: 'bananas' }]; + const options = { runOnly: 'hehe' }; const results = [ { id: 'hehe', @@ -1310,6 +1302,91 @@ describe('Audit', () => { }; audit.after(results, options); + assert.isTrue(success); + }); + + it('does not run Rule#after if the result has an error', () => { + audit = new Audit(); + const results = [{ id: 'throw1', error: new Error('La la la!') }]; + let success = true; + audit.rules.push(new Rule({ id: 'throw1' })); + audit.rules[0].after = () => (success = false); + audit.after(results, {}); + assert.lengthOf(results, 1); + assert.equal(results[0].error.message, 'La la la!'); + assert.isTrue(success, 'Rule#after should not be called'); + }); + + it('catches errors and passes them as a cantTell result', () => { + audit = new Audit(); + const err = new SyntaxError('La la la!'); + const results = [ + { + id: 'throw1', + nodes: [ + { + id: 'throw1-check1-after', + node: new axe.utils.DqElement(fixture), + any: [{ id: 'throw1-check1-after', result: false }], + all: [], + none: [] + } + ] + } + ]; + audit.addRule({ + id: 'throw1', + selector: '#fixture', + any: [{ id: 'throw1-check1-after' }] + }); + audit.addCheck({ + id: 'throw1-check1-after', + after: () => { + throw err; + } + }); + axe.setup(); + const result = audit.after(results, {}); + assert.lengthOf(result, 1); + assertErrorResults(result[0], err, '#fixture'); + }); + + it('throws errors when debug is set', () => { + audit = new Audit(); + const err = new SyntaxError('La la la!'); + const options = { debug: true }; + const results = [ + { + id: 'throw1', + nodes: [ + { + id: 'throw1-check1-after', + node: new axe.utils.DqElement(fixture), + any: [{ id: 'throw1-check1-after', result: false }], + all: [], + none: [] + } + ] + } + ]; + audit.addRule({ + id: 'throw1', + selector: '#fixture', + any: [{ id: 'throw1-check1-after' }] + }); + audit.addCheck({ + id: 'throw1-check1-after', + after: () => { + throw err; + } + }); + axe.setup(); + try { + audit.after(results, options); + assert.fail('Should have thrown'); + } catch (actual) { + assertEqualSupportError(actual, err); + } }); }); diff --git a/test/core/base/rule.js b/test/core/base/rule.js index 893fa92da2..af265d93d2 100644 --- a/test/core/base/rule.js +++ b/test/core/base/rule.js @@ -4,7 +4,7 @@ describe('Rule', () => { const metadataFunctionMap = axe._thisWillBeDeletedDoNotUse.base.metadataFunctionMap; const fixture = document.getElementById('fixture'); - const { fixtureSetup } = axe.testUtils; + const { fixtureSetup, captureError } = axe.testUtils; const noop = () => {}; const isNotCalled = function (err) { throw err || new Error('Reject should not be called'); @@ -233,30 +233,6 @@ describe('Rule', () => { ); }); - it('should handle an error in #matches', done => { - const div = document.createElement('div'); - div.setAttribute('style', '#fff'); - fixture.appendChild(div); - let success = false, - rule = new Rule({ - matches: () => { - throw new Error('this is an error'); - } - }); - - rule.run( - { - include: [axe.utils.getFlattenedTree(div)[0]] - }, - {}, - isNotCalled, - () => { - assert.isFalse(success); - done(); - } - ); - }); - it('should execute Check#run on its child checks - any', done => { fixtureSetup('<blink>Hi</blink>'); let success = false; @@ -618,9 +594,7 @@ describe('Rule', () => { it('should pass thrown errors to the reject param', done => { fixtureSetup('<blink>Hi</blink>'); const rule = new Rule( - { - none: ['cats'] - }, + { none: ['cats'] }, { checks: { cats: { @@ -719,6 +693,62 @@ describe('Rule', () => { ); }); + describe('error handling', () => { + it('should return a SupportError if #matches throws', done => { + const rule = new Rule({ + id: 'fizz', + matches: () => { + throw new Error('this is an error'); + } + }); + axe.setup(); + rule.run( + { include: [axe.utils.getFlattenedTree(fixture)[0]] }, + {}, + isNotCalled, + captureError(err => { + assert.instanceOf(err, axe.utils.SupportError); + assert.include(err.message, 'this is an error'); + assert.equal(err.ruleId, 'fizz'); + assert.equal(err.method, '#matches'); + assert.deepEqual(err.errorNode.selector, ['#fixture']); + done(); + }, done) + ); + }); + + it('should return a SupportError if check.evaluate throws', done => { + const rule = new Rule( + { id: 'garden', any: ['plants'] }, + { + checks: { + plants: new Check({ + id: 'plants', + enabled: true, + evaluate: () => { + throw new Error('zombies ate my pants'); + } + }) + } + } + ); + axe.setup(); + rule.run( + { include: axe.utils.getFlattenedTree(fixture) }, + {}, + isNotCalled, + captureError(err => { + assert.instanceOf(err, axe.utils.SupportError); + assert.include(err.message, 'zombies ate my pants'); + assert.equal(err.ruleId, 'garden'); + assert.equal(err.method, 'plants#evaluate'); + assert.deepEqual(err.errorNode.selector, ['#fixture']); + done(); + }, done) + ); + }); + }); + describe('NODE rule', () => { it('should create a RuleResult', () => { axe.setup(); @@ -1660,9 +1690,50 @@ describe('Rule', () => { assert.lengthOf(result.nodes, 1); }); + + it('should throw a SupportError if check.after throws', () => { + const rule = new Rule( + { id: 'dogs', any: ['cats'] }, + { + checks: { + cats: { + id: 'cats', + enabled: true, + after: () => { + throw new Error('this is an error'); + } + } + } + } + ); + axe.setup(); + try { + rule.after( + { + id: 'cats', + nodes: [ + { + all: [], + none: [], + any: [{ id: 'cats', result: true }], + node: new axe.utils.DqElement(fixture) + } + ] + }, + {} + ); + assert.fail('Should have thrown'); + } catch (err) { + assert.instanceOf(err, axe.utils.SupportError); + assert.include(err.message, 'this is an error'); + assert.equal(err.ruleId, 'dogs'); + assert.equal(err.method, 'cats#after'); + assert.deepEqual(err.errorNode.selector, ['#fixture']); + } + }); }); - describe('after', () => { + describe('reviewOnFail', () => { it('should mark checks as incomplete if reviewOnFail is set to true for all', () => { axe.setup(); const rule = new Rule( diff --git a/test/core/public/run-rules.js b/test/core/public/run-rules.js index d04b853a80..ec9007c22f 100644 --- a/test/core/public/run-rules.js +++ b/test/core/public/run-rules.js @@ -1,10 +1,10 @@ -describe('runRules', function () { - 'use strict'; - var ver = axe.version.substring(0, axe.version.lastIndexOf('.')); +describe('runRules', () => { + const ver = axe.version.substring(0, axe.version.lastIndexOf('.')); + const { captureError } = axe.testUtils; function iframeReady(src, context, id, cb) { - var i = document.createElement('iframe'); - i.addEventListener('load', function () { + let i = document.createElement('iframe'); + i.addEventListener('load', () => { cb(); }); i.src = src; @@ -13,9 +13,9 @@ describe('runRules', function () { } function createFrames(url, callback) { - var frame, + let frame, num = 2; - var loaded = 0; + let loaded = 0; if (typeof url === 'function') { callback = url; @@ -42,22 +42,22 @@ describe('runRules', function () { return frame; } - var fixture = document.getElementById('fixture'); + let fixture = document.getElementById('fixture'); - var isNotCalled; - beforeEach(function () { + let isNotCalled; + beforeEach(() => { isNotCalled = function (err) { throw err || new Error('Reject should not be called'); }; }); - afterEach(function () { + afterEach(() => { fixture.innerHTML = ''; axe._audit = null; axe.teardown(); }); - it('should work', function (done) { + it('should work', done => { axe._load({ rules: [ { @@ -69,7 +69,7 @@ describe('runRules', function () { checks: [ { id: 'html', - evaluate: function () { + evaluate: () => { return true; } } @@ -80,8 +80,8 @@ describe('runRules', function () { var frame = document.createElement('iframe'); frame.src = '../mock/frames/frame-frame.html'; - frame.addEventListener('load', function () { - setTimeout(function () { + frame.addEventListener('load', () => { + setTimeout(() => { axe._runRules( document, {}, @@ -96,7 +96,7 @@ describe('runRules', function () { fixture.appendChild(frame); }); - it('should properly order iframes', function (done) { + it('should properly order iframes', done => { axe._load({ rules: [ { @@ -105,39 +105,26 @@ describe('runRules', function () { any: ['iframe'] } ], - checks: [ - { - id: 'iframe', - evaluate: function () { - return true; - } - } - ], + checks: [{ id: 'iframe', evaluate: () => true }], messages: {} }); var frame = document.createElement('iframe'); - frame.addEventListener('load', function () { - setTimeout(function () { + frame.addEventListener('load', () => { + setTimeout(() => { axe._runRules( document, {}, - function (r) { - var nodes = r[0].passes.map(function (detail) { - return detail.node.selector; - }); - try { - assert.deepEqual(nodes, [ - ['#level0'], - ['#level0', '#level1'], - ['#level0', '#level1', '#level2a'], - ['#level0', '#level1', '#level2b'] - ]); - done(); - } catch (e) { - done(e); - } - }, + captureError(r => { + const nodes = r[0].passes.map(detail => detail.node.selector); + assert.deepEqual(nodes, [ + ['#level0'], + ['#level0', '#level1'], + ['#level0', '#level1', '#level2a'], + ['#level0', '#level1', '#level2b'] + ]); + done(); + }, done), isNotCalled ); }, 500); @@ -147,7 +134,7 @@ describe('runRules', function () { fixture.appendChild(frame); }); - it('should properly calculate context and return results from matching frames', function (done) { + it('should properly calculate context and return results from matching frames', done => { axe._load({ rules: [ { @@ -164,17 +151,15 @@ describe('runRules', function () { checks: [ { id: 'has-target', - evaluate: function () { - return true; - } + evaluate: () => true }, { id: 'first-div', - evaluate: function (node) { + evaluate(node) { this.relatedNodes([node]); return false; }, - after: function (results) { + after(results) { if (results.length) { results[0].result = true; } @@ -185,136 +170,124 @@ describe('runRules', function () { messages: {} }); - iframeReady( - '../mock/frames/context.html', - fixture, - 'context-test', - function () { - var div = document.createElement('div'); - fixture.appendChild(div); + iframeReady('../mock/frames/context.html', fixture, 'context-test', () => { + var div = document.createElement('div'); + fixture.appendChild(div); - axe._runRules( - '#fixture', - {}, - function (results) { - try { - assert.deepEqual(JSON.parse(JSON.stringify(results)), [ + axe._runRules( + '#fixture', + {}, + captureError(results => { + assert.deepEqual(JSON.parse(JSON.stringify(results)), [ + { + id: 'div#target', + helpUrl: + 'https://dequeuniversity.com/rules/axe/' + + ver + + '/div#target?application=axeAPI', + pageLevel: false, + impact: null, + inapplicable: [], + incomplete: [], + violations: [], + passes: [ { - id: 'div#target', - helpUrl: - 'https://dequeuniversity.com/rules/axe/' + - ver + - '/div#target?application=axeAPI', - pageLevel: false, + result: 'passed', impact: null, - inapplicable: [], - incomplete: [], - violations: [], - passes: [ + node: { + selector: ['#context-test', '#target'], + ancestry: [ + 'html > body > div:nth-child(1) > iframe:nth-child(1)', + 'html > body > div:nth-child(2)' + ], + xpath: [ + "/iframe[@id='context-test']", + "/div[@id='target']" + ], + source: '<div id="target"></div>', + nodeIndexes: [12, 14], + fromFrame: true + }, + any: [ { - result: 'passed', - impact: null, - node: { - selector: ['#context-test', '#target'], - ancestry: [ - 'html > body > div:nth-child(1) > iframe:nth-child(1)', - 'html > body > div:nth-child(2)' - ], - xpath: [ - "/iframe[@id='context-test']", - "/div[@id='target']" - ], - source: '<div id="target"></div>', - nodeIndexes: [12, 14], - fromFrame: true - }, - any: [ - { - id: 'has-target', - data: null, - relatedNodes: [] - } - ], - all: [], - none: [] + id: 'has-target', + data: null, + relatedNodes: [] } ], - result: 'passed', - tags: [] - }, + all: [], + none: [] + } + ], + result: 'passed', + tags: [] + }, + { + id: 'first-div', + helpUrl: + 'https://dequeuniversity.com/rules/axe/' + + ver + + '/first-div?application=axeAPI', + pageLevel: false, + impact: null, + inapplicable: [], + incomplete: [], + violations: [], + passes: [ { - id: 'first-div', - helpUrl: - 'https://dequeuniversity.com/rules/axe/' + - ver + - '/first-div?application=axeAPI', - pageLevel: false, + result: 'passed', impact: null, - inapplicable: [], - incomplete: [], - violations: [], - passes: [ + node: { + selector: ['#context-test', '#foo'], + ancestry: [ + 'html > body > div:nth-child(1) > iframe:nth-child(1)', + 'html > body > div:nth-child(1)' + ], + xpath: ["/iframe[@id='context-test']", "/div[@id='foo']"], + source: + '<div id="foo">\n <div id="bar"></div>\n </div>', + nodeIndexes: [12, 9], + fromFrame: true + }, + any: [ { - result: 'passed', - impact: null, - node: { - selector: ['#context-test', '#foo'], - ancestry: [ - 'html > body > div:nth-child(1) > iframe:nth-child(1)', - 'html > body > div:nth-child(1)' - ], - xpath: [ - "/iframe[@id='context-test']", - "/div[@id='foo']" - ], - source: - '<div id="foo">\n <div id="bar"></div>\n </div>', - nodeIndexes: [12, 9], - fromFrame: true - }, - any: [ + id: 'first-div', + data: null, + relatedNodes: [ { - id: 'first-div', - data: null, - relatedNodes: [ - { - selector: ['#context-test', '#foo'], - ancestry: [ - 'html > body > div:nth-child(1) > iframe:nth-child(1)', - 'html > body > div:nth-child(1)' - ], - xpath: [ - "/iframe[@id='context-test']", - "/div[@id='foo']" - ], - source: - '<div id="foo">\n <div id="bar"></div>\n </div>', - nodeIndexes: [12, 9], - fromFrame: true - } - ] + selector: ['#context-test', '#foo'], + ancestry: [ + 'html > body > div:nth-child(1) > iframe:nth-child(1)', + 'html > body > div:nth-child(1)' + ], + xpath: [ + "/iframe[@id='context-test']", + "/div[@id='foo']" + ], + source: + '<div id="foo">\n <div id="bar"></div>\n </div>', + nodeIndexes: [12, 9], + fromFrame: true } - ], - all: [], - none: [] + ] } ], - result: 'passed', - tags: [] + all: [], + none: [] } - ]); - done(); - } catch (e) { - done(e); + ], + result: 'passed', + tags: [] } - }, - isNotCalled - ); - } - ); + ]); + done(); + }, done), + isNotCalled + ); + }); }); - it('should reject if the context is invalid', function (done) { + it('should reject if the context is invalid', done => { axe._load({ rules: [ { @@ -326,32 +299,26 @@ describe('runRules', function () { messages: {} }); - iframeReady( - '../mock/frames/context.html', - fixture, - 'context-test', - function () { - axe._runRules( - '#not-happening', - {}, - function () { - assert.fail('This selector should not exist.'); - }, - function (error) { - assert.isOk(error); - assert.equal( - error.message, - 'No elements found for include in page Context' - ); - - done(); - } - ); - } - ); + iframeReady('../mock/frames/context.html', fixture, 'context-test', () => { + axe._runRules( + '#not-happening', + {}, + () => { + assert.fail('This selector should not exist.'); + }, + captureError(error => { + assert.isOk(error); + assert.equal( + error.message, + 'No elements found for include in page Context' + ); + done(); + }, done) + ); + }); }); - it('should accept a jQuery-like object', function (done) { + it('should accept a jQuery-like object', done => { axe._load({ rules: [ { @@ -363,7 +330,7 @@ describe('runRules', function () { checks: [ { id: 'bob', - evaluate: function () { + evaluate: () => { return true; } } @@ -379,19 +346,22 @@ describe('runRules', function () { length: 2 }; - axe.run($test, function (err, results) { - assert.isNull(err); - assert.lengthOf(results.violations, 1); - assert.lengthOf(results.violations[0].nodes, 4); - assert.deepEqual(results.violations[0].nodes[0].target, ['#t1']); - // assert.deepEqual(results.violations[0].nodes[1].target, ['span']); - assert.deepEqual(results.violations[0].nodes[2].target, ['#t2']); - // assert.deepEqual(results.violations[0].nodes[3].target, ['em']); - done(); - }); + axe.run( + $test, + captureError((err, results) => { + assert.isNull(err); + assert.lengthOf(results.violations, 1); + assert.lengthOf(results.violations[0].nodes, 4); + assert.deepEqual(results.violations[0].nodes[0].target, ['#t1']); + // assert.deepEqual(results.violations[0].nodes[1].target, ['span']); + assert.deepEqual(results.violations[0].nodes[2].target, ['#t2']); + // assert.deepEqual(results.violations[0].nodes[3].target, ['em']); + done(); + }, done) + ); }); - it('should accept a NodeList', function (done) { + it('should accept a NodeList', done => { axe._load({ rules: [ { @@ -403,7 +373,7 @@ describe('runRules', function () { checks: [ { id: 'fred', - evaluate: function () { + evaluate: () => { return true; } } @@ -414,19 +384,22 @@ describe('runRules', function () { '<div class="foo" id="t1"><span></span></div><div class="foo" id="t2"><em></em></div>'; var test = fixture.querySelectorAll('.foo'); - axe.run(test, function (err, results) { - assert.isNull(err); - assert.lengthOf(results.violations, 1); - assert.lengthOf(results.violations[0].nodes, 4); - assert.deepEqual(results.violations[0].nodes[0].target, ['#t1']); - // assert.deepEqual(results.violations[0].nodes[1].target, ['span']); - assert.deepEqual(results.violations[0].nodes[2].target, ['#t2']); - // assert.deepEqual(results.violations[0].nodes[3].target, ['em']); - done(); - }); + axe.run( + test, + captureError((err, results) => { + assert.isNull(err); + assert.lengthOf(results.violations, 1); + assert.lengthOf(results.violations[0].nodes, 4); + assert.deepEqual(results.violations[0].nodes[0].target, ['#t1']); + // assert.deepEqual(results.violations[0].nodes[1].target, ['span']); + assert.deepEqual(results.violations[0].nodes[2].target, ['#t2']); + // assert.deepEqual(results.violations[0].nodes[3].target, ['em']); + done(); + }, done) + ); }); - it('should pull metadata from configuration', function (done) { + it('should pull metadata from configuration', done => { axe._load({ rules: [ { @@ -443,7 +416,7 @@ describe('runRules', function () { checks: [ { id: 'has-target', - evaluate: function () { + evaluate: () => { return false; } }, @@ -512,119 +485,115 @@ describe('runRules', function () { axe._runRules( '#fixture', {}, - function (results) { - try { - assert.deepEqual(JSON.parse(JSON.stringify(results)), [ - { - id: 'div#target', - helpUrl: - 'https://dequeuniversity.com/rules/axe/' + - ver + - '/div#target?application=axeAPI', - pageLevel: false, - foo: 'bar', - stuff: 'blah', - impact: 'moderate', - passes: [], - inapplicable: [], - incomplete: [], - violations: [ - { - result: 'failed', - node: { - selector: ['#target'], - ancestry: [ - 'html > body > div:nth-child(1) > div:nth-child(1)' - ], - xpath: ["/div[@id='target']"], - source: '<div id="target">Target!</div>', - nodeIndexes: [12], - fromFrame: false - }, - impact: 'moderate', - any: [ - { - impact: 'moderate', - otherThingy: true, - message: 'failing is not good', - id: 'has-target', - data: null, - relatedNodes: [] - } + captureError(results => { + assert.deepEqual(JSON.parse(JSON.stringify(results)), [ + { + id: 'div#target', + helpUrl: + 'https://dequeuniversity.com/rules/axe/' + + ver + + '/div#target?application=axeAPI', + pageLevel: false, + foo: 'bar', + stuff: 'blah', + impact: 'moderate', + passes: [], + inapplicable: [], + incomplete: [], + violations: [ + { + result: 'failed', + node: { + selector: ['#target'], + ancestry: [ + 'html > body > div:nth-child(1) > div:nth-child(1)' ], - all: [], - none: [] - } - ], - result: 'failed', - tags: [] - }, - { - id: 'first-div', - helpUrl: - 'https://dequeuniversity.com/rules/axe/' + - ver + - '/first-div?application=axeAPI', - pageLevel: false, - bar: 'foo', - stuff: 'no', - impact: null, - inapplicable: [], - incomplete: [], - violations: [], - passes: [ - { - result: 'passed', - impact: null, - node: { - selector: ['#target'], - xpath: ["/div[@id='target']"], - ancestry: [ - 'html > body > div:nth-child(1) > div:nth-child(1)' - ], - source: '<div id="target">Target!</div>', - nodeIndexes: [12], - fromFrame: false - }, - any: [ - { - impact: 'serious', - id: 'first-div', - thingy: true, - message: 'passing is good', - data: null, - relatedNodes: [ - { - selector: ['#target'], - ancestry: [ - 'html > body > div:nth-child(1) > div:nth-child(1)' - ], - xpath: ["/div[@id='target']"], - source: '<div id="target">Target!</div>', - nodeIndexes: [12], - fromFrame: false - } - ] - } + xpath: ["/div[@id='target']"], + source: '<div id="target">Target!</div>', + nodeIndexes: [12], + fromFrame: false + }, + impact: 'moderate', + any: [ + { + impact: 'moderate', + otherThingy: true, + message: 'failing is not good', + id: 'has-target', + data: null, + relatedNodes: [] + } + ], + all: [], + none: [] + } + ], + result: 'failed', + tags: [] + }, + { + id: 'first-div', + helpUrl: + 'https://dequeuniversity.com/rules/axe/' + + ver + + '/first-div?application=axeAPI', + pageLevel: false, + bar: 'foo', + stuff: 'no', + impact: null, + inapplicable: [], + incomplete: [], + violations: [], + passes: [ + { + result: 'passed', + impact: null, + node: { + selector: ['#target'], + xpath: ["/div[@id='target']"], + ancestry: [ + 'html > body > div:nth-child(1) > div:nth-child(1)' ], - all: [], - none: [] - } - ], - result: 'passed', - tags: [] - } - ]); - done(); - } catch (e) { - done(e); - } - }, + source: '<div id="target">Target!</div>', + nodeIndexes: [12], + fromFrame: false + }, + any: [ + { + impact: 'serious', + id: 'first-div', + thingy: true, + message: 'passing is good', + data: null, + relatedNodes: [ + { + selector: ['#target'], + ancestry: [ + 'html > body > div:nth-child(1) > div:nth-child(1)' + ], + xpath: ["/div[@id='target']"], + source: '<div id="target">Target!</div>', + nodeIndexes: [12], + fromFrame: false + } + ] + } + ], + all: [], + none: [] + } + ], + result: 'passed', + tags: [] + } + ]); + done(); + }, done), isNotCalled ); }); - it('should call the reject argument if an error occurs', function (done) { + it('should call the reject argument if an error occurs', done => { axe._load({ rules: [ { @@ -635,25 +604,24 @@ describe('runRules', function () { messages: {} }); - createFrames(function () { - setTimeout(function () { + createFrames(() => { + setTimeout(() => { axe._runRules( document, {}, - function () { + () => { assert.ok(false, 'You shall not pass!'); - done(); }, - function (err) { + captureError(err => { assert.instanceOf(err, Error); done(); - } + }, done) ); }, 100); }); }); - it('should resolve to cantTell when a rule fails', function (done) { + it('should resolve to cantTell when a rule fails', done => { axe._load({ rules: [ { @@ -670,36 +638,34 @@ describe('runRules', function () { checks: [ { id: 'undeffed', - evaluate: function () { + evaluate: () => { return undefined; } }, { id: 'thrower', - evaluate: function () { + evaluate: () => { throw new Error('Check failed to complete'); } } ] }); - fixture.innerHTML = '<div></div>'; - axe.run('#fixture', function (err, results) { - assert.isNull(err); - assert.lengthOf(results.incomplete, 2); - assert.equal(results.incomplete[0].id, 'incomplete-1'); - assert.equal(results.incomplete[1].id, 'incomplete-2'); - - assert.include( - results.incomplete[1].description, - 'An error occured while running this rule' - ); - done(); - }); + axe.run( + '#fixture', + captureError((err, results) => { + assert.isNull(err); + assert.lengthOf(results.incomplete, 2); + assert.equal(results.incomplete[0].id, 'incomplete-1'); + assert.equal(results.incomplete[1].id, 'incomplete-2'); + assert.isNotNull(results.incomplete[1].error); + done(); + }, done) + ); }); - it('should resolve to cantTell if an error occurs inside frame rules', function (done) { + it('should resolve to cantTell if an error occurs inside frame rules', done => { axe._load({ rules: [ { @@ -716,13 +682,13 @@ describe('runRules', function () { checks: [ { id: 'undeffed', - evaluate: function () { + evaluate: () => { return false; } }, { id: 'thrower', - evaluate: function () { + evaluate: () => { return false; } } @@ -733,24 +699,23 @@ describe('runRules', function () { '../mock/frames/rule-error.html', fixture, 'context-test', - function () { - axe.run('#fixture', function (err, results) { - assert.isNull(err); - assert.lengthOf(results.incomplete, 2); - assert.equal(results.incomplete[0].id, 'incomplete-1'); - assert.equal(results.incomplete[1].id, 'incomplete-2'); - - assert.include( - results.incomplete[1].description, - 'An error occured while running this rule' - ); - done(); - }); + () => { + axe.run( + '#fixture', + captureError((err, results) => { + assert.isNull(err); + assert.lengthOf(results.incomplete, 2); + assert.equal(results.incomplete[0].id, 'incomplete-1'); + assert.equal(results.incomplete[1].id, 'incomplete-2'); + assert.isNotNull(results.incomplete[1].error); + done(); + }, done) + ); } ); }); - it('should cascade `no elements found` errors in frames to reject run_rules', function (done) { + it('should cascade `no elements found` errors in frames to reject run_rules', done => { axe._load({ rules: [ { @@ -763,26 +728,24 @@ describe('runRules', function () { fixture.innerHTML = '<div id="outer"></div>'; var outer = document.getElementById('outer'); - iframeReady('../mock/frames/context.html', outer, 'target', function () { + iframeReady('../mock/frames/context.html', outer, 'target', () => { axe._runRules( [['#target', '#elementNotFound']], {}, - function resolve() { - assert.ok(false, 'frame should have thrown an error'); - }, - function reject(err) { + () => assert.ok(false, 'frame should have thrown an error'), + captureError(err => { assert.instanceOf(err, Error); assert.include( err.message, 'No elements found for include in frame Context' ); done(); - } + }, done) ); }); }); - it('should not call reject when the resolve throws', function (done) { + it('should not call reject when the resolve throws', done => { var rejectCalled = false; axe._load({ rules: [ @@ -795,7 +758,7 @@ describe('runRules', function () { checks: [ { id: 'html', - evaluate: function () { + evaluate: () => { return true; } } @@ -804,7 +767,7 @@ describe('runRules', function () { }); function resolve() { - setTimeout(function () { + setTimeout(() => { assert.isFalse(rejectCalled); axe.log = log; done(); @@ -823,7 +786,7 @@ describe('runRules', function () { axe._runRules(document, {}, resolve, reject); }); - it('should ignore iframes if `iframes` === false', function (done) { + it('should ignore iframes if `iframes` === false', done => { axe._load({ rules: [ { @@ -835,7 +798,7 @@ describe('runRules', function () { checks: [ { id: 'html', - evaluate: function () { + evaluate: () => { return true; } } @@ -846,12 +809,12 @@ describe('runRules', function () { var frame = document.createElement('iframe'); frame.src = '../mock/frames/frame-frame.html'; - frame.addEventListener('load', function () { - setTimeout(function () { + frame.addEventListener('load', () => { + setTimeout(() => { axe._runRules( document, { iframes: false, elementRef: true }, - function (r) { + captureError(r => { assert.lengthOf(r[0].passes, 1); assert.equal( r[0].passes[0].node.element.ownerDocument, @@ -859,15 +822,14 @@ describe('runRules', function () { 'Result should not be in an iframe' ); done(); - }, - isNotCalled + }, isNotCalled) ); }, 500); }); fixture.appendChild(frame); }); - it('should not fail if `include` / `exclude` is overwritten', function (done) { + it('should not fail if `include` / `exclude` is overwritten', done => { function invalid() { throw new Error('nope!'); } @@ -885,7 +847,7 @@ describe('runRules', function () { checks: [ { id: 'html', - evaluate: function () { + evaluate: () => { return true; } } @@ -897,17 +859,21 @@ describe('runRules', function () { [document], {}, function (r) { - assert.lengthOf(r[0].passes, 1); - - delete Array.prototype.include; - delete Array.prototype.exclude; - done(); + try { + assert.lengthOf(r[0].passes, 1); + done(); + } catch (e) { + done(e); + } finally { + delete Array.prototype.include; + delete Array.prototype.exclude; + } }, isNotCalled ); }); - it('should return a cleanup method', function (done) { + it('should return a cleanup method', done => { axe._load({ rules: [ { @@ -919,7 +885,7 @@ describe('runRules', function () { checks: [ { id: 'html', - evaluate: function () { + evaluate: () => { return true; } } @@ -930,7 +896,7 @@ describe('runRules', function () { axe._runRules( document, {}, - function resolve(out, cleanup) { + captureError((out, cleanup) => { assert.isDefined(axe._tree); assert.isDefined(axe._selectorData); @@ -938,12 +904,12 @@ describe('runRules', function () { assert.isUndefined(axe._tree); assert.isUndefined(axe._selectorData); done(); - }, + }, done), isNotCalled ); }); - it('should clear up axe._tree / axe._selectorData after an error', function (done) { + it('should clear up axe._tree / axe._selectorData after an error', done => { axe._load({ rules: [ { @@ -954,19 +920,24 @@ describe('runRules', function () { messages: {} }); - createFrames(function () { - setTimeout(function () { - axe._runRules(document, {}, isNotCalled, function () { - assert.isUndefined(axe._tree); - assert.isUndefined(axe._selectorData); - done(); - }); + createFrames(() => { + setTimeout(() => { + axe._runRules( + document, + {}, + isNotCalled, + captureError(() => { + assert.isUndefined(axe._tree); + assert.isUndefined(axe._selectorData); + done(); + }, done) + ); }, 100); }); }); // todo: see issue - https://github.com/dequelabs/axe-core/issues/2168 - it.skip('should clear the memoized cache for each function', function (done) { + it.skip('should clear the memoized cache for each function', done => { axe._load({ rules: [ { @@ -978,7 +949,7 @@ describe('runRules', function () { checks: [ { id: 'html', - evaluate: function () { + evaluate: () => { return true; } } @@ -989,11 +960,11 @@ describe('runRules', function () { axe._runRules( document, {}, - function resolve(out, cleanup) { + captureError((out, cleanup) => { var called = false; axe._memoizedFns = [ { - clear: function () { + clear: () => { called = true; } } @@ -1003,7 +974,7 @@ describe('runRules', function () { assert.isTrue(called); done(); - }, + }, done), isNotCalled ); }); diff --git a/test/core/utils/merge-results.js b/test/core/utils/merge-results.js index 85c15760fc..fd3ff5ea74 100644 --- a/test/core/utils/merge-results.js +++ b/test/core/utils/merge-results.js @@ -1,8 +1,9 @@ -describe('axe.utils.mergeResults', function () { +describe('axe.utils.mergeResults', () => { 'use strict'; var queryFixture = axe.testUtils.queryFixture; + var SupportError = axe.utils.SupportError; - it('should normalize empty results', function () { + it('should normalize empty results', () => { var result = axe.utils.mergeResults([ { results: [] }, { results: [{ id: 'a', result: 'b' }] } @@ -15,7 +16,7 @@ describe('axe.utils.mergeResults', function () { ]); }); - it('merges frame content, including all selector types', function () { + it('merges frame content, including all selector types', () => { var iframe = queryFixture('<iframe id="target"></iframe>').actualNode; var node = { selector: ['#foo'], @@ -49,7 +50,7 @@ describe('axe.utils.mergeResults', function () { assert.deepEqual(node.nodeIndexes, [1, 123]); }); - it('merges frame specs', function () { + it('merges frame specs', () => { var iframe = queryFixture('<iframe id="target"></iframe>').actualNode; var frameSpec = new axe.utils.DqElement(iframe).toJSON(); var node = { @@ -84,7 +85,7 @@ describe('axe.utils.mergeResults', function () { assert.deepEqual(node.nodeIndexes, [1, 123]); }); - it('sorts results from iframes into their correct DOM position', function () { + it('sorts results from iframes into their correct DOM position', () => { var result = axe.utils.mergeResults([ { results: [ @@ -148,7 +149,7 @@ describe('axe.utils.mergeResults', function () { assert.deepEqual(ids, ['h1', 'iframe1 >> h2', 'iframe1 >> h3', 'h4']); }); - it('sorts nested iframes', function () { + it('sorts nested iframes', () => { var result = axe.utils.mergeResults([ { results: [ @@ -219,7 +220,7 @@ describe('axe.utils.mergeResults', function () { ]); }); - it('sorts results even if nodeIndexes are empty', function () { + it('sorts results even if nodeIndexes are empty', () => { var result = axe.utils.mergeResults([ { results: [ @@ -296,7 +297,7 @@ describe('axe.utils.mergeResults', function () { ]); }); - it('sorts results even if nodeIndexes are undefined', function () { + it('sorts results even if nodeIndexes are undefined', () => { var result = axe.utils.mergeResults([ { results: [ @@ -370,7 +371,7 @@ describe('axe.utils.mergeResults', function () { ]); }); - it('sorts nodes all placed on the same result', function () { + it('sorts nodes all placed on the same result', () => { var result = axe.utils.mergeResults([ { results: [ @@ -419,4 +420,57 @@ describe('axe.utils.mergeResults', function () { '#level0 >> #level1 >> #level2b' ]); }); + + describe('errors', () => { + it('sets error if it is present', () => { + const result = axe.utils.mergeResults([ + { results: [{ id: 'a', result: 'b', error: new Error('test') }] } + ]); + assert.equal(result[0].error.message, 'test'); + }); + + it('picks the first error if there are multiple', () => { + const result = axe.utils.mergeResults([ + { + results: [ + { + id: 'error-occurred', + result: undefined, + nodes: [{ node: { selector: ['h1'], nodeIndexes: [1] } }] + }, + { + id: 'error-occurred', + result: undefined, + error: new SupportError({ error: new Error('test 1') }), + nodes: [ + { + node: { + selector: ['iframe1', 'h2'], + nodeIndexes: [2, 1], + fromFrame: true + } + } + ] + }, + { + id: 'error-occurred', + result: undefined, + error: new SupportError({ error: new Error('test 2') }), + nodes: [ + { + node: { + selector: ['iframe2', 'h3'], + nodeIndexes: [3, 1], + fromFrame: true + } + } + ] + } + ] + } + ]); + + assert.equal(result[0].error.message, 'test 1'); + }); + }); }); diff --git a/test/core/utils/serialize-error.js b/test/core/utils/serialize-error.js new file mode 100644 index 0000000000..bd36c7c3a8 --- /dev/null +++ b/test/core/utils/serialize-error.js @@ -0,0 +1,45 @@ +describe('utils.serializeError', function () { + const serializeError = axe.utils.serializeError; + + it('should serialize an error', () => { + const error = new Error('test'); + const serialized = serializeError(error); + assert.ownInclude(serialized, { + message: error.message, + stack: error.stack, + name: error.name + }); + }); + + it('should serialize an error with a cause', () => { + const error = new Error('test'); + error.cause = new Error('cause'); + const serialized = serializeError(error); + assert.ownInclude(serialized.cause, { + message: error.cause.message, + stack: error.cause.stack, + name: error.cause.name + }); + }); + + it('should serialize recursively', () => { + const error = new Error('test'); + error.cause = new Error('cause'); + error.cause.cause = new Error('cause2'); + const serialized = serializeError(error); + assert.ownInclude(serialized.cause.cause, { + message: error.cause.cause.message, + stack: error.cause.cause.stack, + name: error.cause.cause.name + }); + }); + + it('should not serialize the cause if the stack exceeds 10 levels', () => { + const error = new Error('test'); + error.cause = new Error('cause'); + error.cause.cause = new Error('cause2'); + error.cause.cause.cause = new Error('cause3'); + const serialized = serializeError(error, 9); + assert.equal(serialized.cause.cause, '...'); + }); +}); diff --git a/test/core/utils/support-error.js b/test/core/utils/support-error.js new file mode 100644 index 0000000000..ab47870bcc --- /dev/null +++ b/test/core/utils/support-error.js @@ -0,0 +1,45 @@ +describe('utils.SupportError', () => { + const SupportError = axe.utils.SupportError; + + it('returns a serializable error', () => { + const error = new Error('test'); + const supportError = new SupportError({ error }); + assert.ownInclude(supportError, { + message: error.message, + stack: error.stack, + name: error.name + }); + }); + + it('returns a instanceof Error', () => { + const error = new Error('test'); + const supportError = new SupportError({ error }); + assert.instanceOf(supportError, Error); + }); + + it('includes the ruleId if provided', () => { + const error = new Error('test'); + const supportError = new SupportError({ error, ruleId: 'aria' }); + assert.equal(supportError.ruleId, 'aria'); + assert.include(supportError.message, 'Skipping aria rule.'); + }); + + it('includes the method if provided', () => { + const error = new Error('test'); + const supportError = new SupportError({ error, method: '#matches' }); + assert.equal(supportError.method, '#matches'); + }); + + it('includes the errorNode if provided', () => { + const error = new Error('test'); + const supportError = new SupportError({ error, errorNode: 'err' }); + assert.equal(supportError.errorNode, 'err'); + }); + + it('includes a serialized cause if provided', () => { + const error = new Error('test'); + error.cause = new Error('cause'); + const supportError = new SupportError({ error }); + assert.deepEqual(supportError.cause, axe.utils.serializeError(error.cause)); + }); +}); diff --git a/test/integration/full/error-occurred/error-frame.html b/test/integration/full/error-occurred/error-frame.html new file mode 100644 index 0000000000..dbe5eafb48 --- /dev/null +++ b/test/integration/full/error-occurred/error-frame.html @@ -0,0 +1,30 @@ +<!doctype html> +<html lang="en"> + <head> + <title>error-occurred in frame test + + + + + + + + + +
+ + + + + + diff --git a/test/integration/full/error-occurred/error-frame.js b/test/integration/full/error-occurred/error-frame.js new file mode 100644 index 0000000000..f94b3476b5 --- /dev/null +++ b/test/integration/full/error-occurred/error-frame.js @@ -0,0 +1,118 @@ +describe('error-occurred test', () => { + const { runPartialRecursive } = axe.testUtils; + let results; + + describe('axe.run()', () => { + before(done => { + axe.testUtils.awaitNestedLoad(() => { + axe.run( + { + runOnly: ['matches-error', 'evaluate-error', 'after-error'] + }, + function (err, r) { + assert.isNull(err); + results = r; + done(); + } + ); + }); + }); + + it('should find 0 violations', () => { + assert.lengthOf(results.violations, 0); + }); + + it('should find 0 passes', () => { + assert.lengthOf(results.passes, 0); + }); + + describe('incomplete', () => { + it('should find matches-error', () => { + const matchesError = results.incomplete.find( + result => result.id === 'matches-error' + ); + window.assertIsErrorOccurred(matchesError, { + message: 'matches error', + target: ['#frame', '#target'] + }); + }); + + it('should find evaluate-error', () => { + const evaluateError = results.incomplete.find( + result => result.id === 'evaluate-error' + ); + window.assertIsErrorOccurred(evaluateError, { + message: 'evaluate error', + target: ['#frame', '#target'] + }); + }); + + it('should find after-error', () => { + const afterError = results.incomplete.find( + result => result.id === 'after-error' + ); + window.assertIsErrorOccurred(afterError, { + message: 'after error', + target: ['#frame', '#target'] + }); + }); + }); + }); + + describe('axe.runPartial() + axe.finishRun()', () => { + before(() => { + return new Promise(resolve => { + axe.testUtils.awaitNestedLoad(async () => { + const runOptions = { + runOnly: ['matches-error', 'evaluate-error', 'after-error'] + }; + const partialResults = await Promise.all( + runPartialRecursive(document, runOptions) + ); + results = await axe.finishRun(partialResults, runOptions); + resolve(); + }); + }); + }); + + it('should find 0 violations', () => { + assert.lengthOf(results.violations, 0); + }); + + it('should find 0 passes', () => { + assert.lengthOf(results.passes, 0); + }); + + describe('incomplete', () => { + it('should find matches-error', () => { + const matchesError = results.incomplete.find( + result => result.id === 'matches-error' + ); + window.assertIsErrorOccurred(matchesError, { + message: 'matches error', + target: ['#frame', '#target'] + }); + }); + + it('should find evaluate-error', () => { + const evaluateError = results.incomplete.find( + result => result.id === 'evaluate-error' + ); + window.assertIsErrorOccurred(evaluateError, { + message: 'evaluate error', + target: ['#frame', '#target'] + }); + }); + + it('should find after-error', () => { + const afterError = results.incomplete.find( + result => result.id === 'after-error' + ); + window.assertIsErrorOccurred(afterError, { + message: 'after error', + target: ['#frame', '#target'] + }); + }); + }); + }); +}); diff --git a/test/integration/full/error-occurred/error-occurred.html b/test/integration/full/error-occurred/error-occurred.html new file mode 100644 index 0000000000..bbec76bb3b --- /dev/null +++ b/test/integration/full/error-occurred/error-occurred.html @@ -0,0 +1,30 @@ + + + + error-occurred test + + + + + + + + +
+
+ + + + + + diff --git a/test/integration/full/error-occurred/error-occurred.js b/test/integration/full/error-occurred/error-occurred.js new file mode 100644 index 0000000000..9f87380942 --- /dev/null +++ b/test/integration/full/error-occurred/error-occurred.js @@ -0,0 +1,118 @@ +describe('error-occurred test', () => { + const { runPartialRecursive } = axe.testUtils; + let results; + + describe('axe.run()', () => { + before(done => { + axe.testUtils.awaitNestedLoad(() => { + axe.run( + { + runOnly: ['matches-error', 'evaluate-error', 'after-error'] + }, + function (err, r) { + assert.isNull(err); + results = r; + done(); + } + ); + }); + }); + + it('should find 0 violations', () => { + assert.lengthOf(results.violations, 0); + }); + + it('should find 0 passes', () => { + assert.lengthOf(results.passes, 0); + }); + + describe('incomplete', () => { + it('should find matches-error', () => { + const matchesError = results.incomplete.find( + result => result.id === 'matches-error' + ); + window.assertIsErrorOccurred(matchesError, { + message: 'matches error', + target: ['#target'] + }); + }); + + it('should find evaluate-error', () => { + const evaluateError = results.incomplete.find( + result => result.id === 'evaluate-error' + ); + window.assertIsErrorOccurred(evaluateError, { + message: 'evaluate error', + target: ['#target'] + }); + }); + + it('should find after-error', () => { + const afterError = results.incomplete.find( + result => result.id === 'after-error' + ); + window.assertIsErrorOccurred(afterError, { + message: 'after error', + target: ['#target'] + }); + }); + }); + }); + + describe('axe.runPartial() + axe.finishRun()', () => { + before(() => { + return new Promise(resolve => { + axe.testUtils.awaitNestedLoad(async () => { + const runOptions = { + runOnly: ['matches-error', 'evaluate-error', 'after-error'] + }; + const partialResults = await Promise.all( + runPartialRecursive(document, runOptions) + ); + results = await axe.finishRun(partialResults, runOptions); + resolve(); + }); + }); + }); + + it('should find 0 violations', () => { + assert.lengthOf(results.violations, 0); + }); + + it('should find 0 passes', () => { + assert.lengthOf(results.passes, 0); + }); + + describe('incomplete', () => { + it('should find matches-error', () => { + const matchesError = results.incomplete.find( + result => result.id === 'matches-error' + ); + window.assertIsErrorOccurred(matchesError, { + message: 'matches error', + target: ['#target'] + }); + }); + + it('should find evaluate-error', () => { + const evaluateError = results.incomplete.find( + result => result.id === 'evaluate-error' + ); + window.assertIsErrorOccurred(evaluateError, { + message: 'evaluate error', + target: ['#target'] + }); + }); + + it('should find after-error', () => { + const afterError = results.incomplete.find( + result => result.id === 'after-error' + ); + window.assertIsErrorOccurred(afterError, { + message: 'after error', + target: ['#target'] + }); + }); + }); + }); +}); diff --git a/test/integration/full/error-occurred/error-ruleset.js b/test/integration/full/error-occurred/error-ruleset.js new file mode 100644 index 0000000000..83860070af --- /dev/null +++ b/test/integration/full/error-occurred/error-ruleset.js @@ -0,0 +1,55 @@ +window.assertIsErrorOccurred = function (result, { message, target }) { + assert.isDefined(result); + assert.isDefined(result.error); + assert.include(result.error.message, message); + + assert.lengthOf(result.nodes, 1); + const node = result.nodes[0]; + assert.lengthOf(node.any, 0); + assert.lengthOf(node.all, 0); + assert.lengthOf(node.none, 1); + assert.equal(node.none[0].id, 'error-occurred'); + assert.include(node.none[0].message, 'Axe encountered an error'); + assert.deepEqual(node.target, target); +}; + +axe.configure({ + rules: [ + { + id: 'matches-error', + selector: '#target', + matches: () => { + throw new Error('matches error'); + }, + any: ['exists'] + }, + { + id: 'evaluate-error', + selector: '#target', + any: ['check-evaluate-error'] + }, + { + id: 'after-error', + selector: '#target', + any: ['check-after-error'] + } + ], + checks: [ + { + id: 'check-evaluate-error', + evaluate: () => { + throw new Error('evaluate error'); + }, + after: () => { + throw new Error('I should not be seen'); + } + }, + { + id: 'check-after-error', + evaluate: () => true, + after: () => { + throw new Error('after error'); + } + } + ] +}); diff --git a/test/integration/full/error-occurred/frames/error.html b/test/integration/full/error-occurred/frames/error.html new file mode 100644 index 0000000000..81a004af01 --- /dev/null +++ b/test/integration/full/error-occurred/frames/error.html @@ -0,0 +1,4 @@ +
+ + + diff --git a/typings/axe-core/axe-core-tests.ts b/typings/axe-core/axe-core-tests.ts index 34cba1e8dc..68c9b087fd 100644 --- a/typings/axe-core/axe-core-tests.ts +++ b/typings/axe-core/axe-core-tests.ts @@ -16,6 +16,13 @@ axe.run(context, {}, (error: Error, results: axe.AxeResults) => { } console.log(results.passes.length); console.log(results.incomplete.length); + const errors = results.incomplete.map(result => result.error); + console.log( + errors.map( + ({ message, stack, ruleId, method }) => + `${message} ${ruleId} ${method}\n\n${stack}` + ) + ); console.log(results.inapplicable.length); console.log(results.violations.length); console.log(results.violations[0].nodes[0].failureSummary);