diff --git a/doc/config.md b/doc/config.md index 0d2f7dd02..69781478d 100644 --- a/doc/config.md +++ b/doc/config.md @@ -226,6 +226,11 @@ Settings list: specified, the fallen tests will not be relaunched (by default it's 0). Note that the same test never be performed in the same browser session. + +* `shouldRetry` — function which defines whether to make a retry. Should returns `Boolean` value. First argument is `data` which is the result of the test run, it's an Object with following fields: + * `attempt {Number}` number of retries performed for the test + * `retriesLeft {Number}` number of retries left + * `[equal] {Boolean}` notice that if the test fails with critical error equal can be absent * `screenshotMode` — image capture mode. There are 3 allowed values for this option: * `auto` (default). Mode will be obtained automatically. diff --git a/lib/runner/suite-runner/insistent-suite-runner.js b/lib/runner/suite-runner/insistent-suite-runner.js index 53d2a65f4..960b7438a 100644 --- a/lib/runner/suite-runner/insistent-suite-runner.js +++ b/lib/runner/suite-runner/insistent-suite-runner.js @@ -72,15 +72,27 @@ module.exports = class InsistentSuiteRunner extends SuiteRunner { this._statesToRetry.forEach((state) => collection.enable(suite, {state, browser})); } - _submitForRetry(e) { - const retriesLeft = this._config.retry - this._retriesPerformed; - if (retriesLeft <= 0) { + _shouldRetry(data) { + if (typeof this._config.shouldRetry === 'function') { + return this._config.shouldRetry(data); + } + + return data.retriesLeft > 0; + } + + _submitForRetry(data) { + Object.assign(data, { + attempt: this._retriesPerformed, + retriesLeft: this._config.retry - this._retriesPerformed + }); + + if (!this._shouldRetry(data)) { return false; } - this._statesToRetry = this._statesToRetry.concat(e.state.name); + this._statesToRetry = this._statesToRetry.concat(data.state.name); - this._emit(Events.RETRY, _.extend(e, {attempt: this._retriesPerformed, retriesLeft})); + this._emit(Events.RETRY, data); return true; } }; diff --git a/test/unit/runner/suite-runner/insistent-suite-runner.js b/test/unit/runner/suite-runner/insistent-suite-runner.js index e882ca925..8ba986b0a 100644 --- a/test/unit/runner/suite-runner/insistent-suite-runner.js +++ b/test/unit/runner/suite-runner/insistent-suite-runner.js @@ -275,6 +275,37 @@ describe('runner/suite-runner/insistent-suite-runner', () => { assert.calledOnce(RegularSuiteRunner.prototype.run); }); }); + + it('should not retry if config shouldRetry() told so', () => { + stubWrappedRun_((runner) => runner.emit(Events.ERROR, {state: makeStateStub()})); + const config = mkConfigStub_({ + retry: 100, + shouldRetry: () => false + }); + + return mkInsistentRunner_({config}) + .run() + .then(() => assert(RegularSuiteRunner.prototype.run.calledOnce)); + }); + + it('shouldRetry() should have proper argument', () => { + stubWrappedRun_((runner) => runner.emit(Events.ERROR, {state: makeStateStub()})); + const rule = sandbox.spy(); + const config = mkConfigStub_({ + retry: 1, + shouldRetry: rule + }); + + return mkInsistentRunner_({config}) + .run() + .then(() => { + const firstCallArgs = rule.firstCall.args; + const arg = firstCallArgs[0]; + assert.equal(firstCallArgs.length, 1, 'shouldRetry() expect one argument'); + assert.typeOf(arg.retriesLeft, 'number', 'No "retriesLeft" property in shouldRetry argument'); + assert.typeOf(arg.attempt, 'number', 'No "attempt" property in shouldRetry argument'); + }); + }); }); describe('on TEST_RESULT without diff', () => { @@ -387,6 +418,38 @@ describe('runner/suite-runner/insistent-suite-runner', () => { assert.calledOnce(RegularSuiteRunner.prototype.run); }); }); + + it('should not retry if config shouldRetry() told so', () => { + stubWrappedRun_((runner) => runner.emit(Events.TEST_RESULT, {state: makeStateStub()})); + const config = mkConfigStub_({ + retry: 100, + shouldRetry: () => false + }); + + return mkInsistentRunner_({config}) + .run() + .then(() => assert(RegularSuiteRunner.prototype.run.calledOnce)); + }); + + it('shouldRetry() should have proper argument', () => { + stubWrappedRun_((runner) => runner.emit(Events.TEST_RESULT, {equal: false, state: makeStateStub()})); + const rule = sandbox.spy(); + const config = mkConfigStub_({ + retry: 1, + shouldRetry: rule + }); + + return mkInsistentRunner_({config}) + .run() + .then(() => { + const firstCallArgs = rule.firstCall.args; + const arg = firstCallArgs[0]; + assert.equal(firstCallArgs.length, 1, 'shouldRetry() expect one argument'); + assert.typeOf(arg.retriesLeft, 'number', 'No "retriesLeft" property in shouldRetry argument'); + assert.typeOf(arg.attempt, 'number', 'No "attempt" property in shouldRetry argument'); + assert.typeOf(arg.equal, 'boolean', 'No "equal" property in shouldRetry argument'); + }); + }); }); it('should retry only failed states', () => {