microsoft · muhqu · May 22, 2024 · May 22, 2024 · May 22, 2024 · May 22, 2024
diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md
@@ -469,7 +469,7 @@ export default defineConfig({
 
 Shard tests and execute only the selected shard. Specify in the one-based form like `{ total: 5, current: 2 }`.
 
-Learn more about [parallelism and sharding](../test-parallel.md) with Playwright Test.
+Learn more about [parallelism](../test-parallel.md) and [sharding](../test-sharding.md) with Playwright Test.
 
 **Usage**
 
@@ -482,6 +482,27 @@ export default defineConfig({
 ```
 
 
+## property: TestConfig.shardingMode
+
+* since: v1.48
+- type: ?<[ShardingMode]<"partition"|"round-robin"|"duration-round-robin">>
+
+Defines the algorithm to be used for sharding. Defaults to `'partition'`.
+* `'partition'` - divide the set of test groups by number of shards. e.g. first
+  half goes to shard 1/2 and seconds half to shard 2/2.
+* `'round-robin'` - spread test groups to shards in a round-robin way. e.g. loop
+  over test groups and always assign to the shard that has the lowest number of
+  tests.
+* `'duration-round-robin'` - use duration info from `.last-run.json` to spread
+  test groups to shards in a round-robin way. e.g. loop over test groups and
+  always assign to the shard that has the lowest duration of tests. new tests
+  which were not present in the last run will use an average duration time. When
+  no `.last-run.json` could be found the behavior is identical to
+  `'round-robin'`.
+
+Learn more about [sharding](../test-sharding.md) with Playwright Test.
+
+
 ## property: TestConfig.testDir
 * since: v1.10
 - type: ?<[string]>

diff --git a/docs/src/test-sharding-js.md b/docs/src/test-sharding-js.md
@@ -180,3 +180,73 @@ Supported options:
     reporter: [['html', { open: 'never' }]],
   };
   ```
+
+## Sharding Modes
+
+Playwright offers different sharding modes to alter the behavior how test groups are assigned to shards, which can have an impact on overall execution time and resource utilization.
+
+### `shardingMode: 'partition'`
+
+This is the _default_. Test groups are ordered in the way they are discovered. Test groups are assigned the current shard until it has equal or more than 1/Nth of the overall number of tests. Then the next shard is filled, etc.
+
+This has the effect that tests which share a common prefix are likely to execute on the same shard.
+
+```yaml
+         [  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12]
+Shard 1:  ^---------^                                      : [  1, 2, 3 ]
+Shard 2:              ^---------^                          : [  4, 5, 6 ]
+Shard 3:                          ^---------^              : [  7, 8, 9 ]
+Shard 4:                                      ^---------^  : [ 10,11,12 ]
+```
+
+### `shardingMode: 'round-robin'`
+
+Spreads test groups evenly across shards. It sorts test groups by number of tests in descending order, then loops through the test groups and assigns them to the shard with the lowest number of tests.
+
+Below is an example where every test group represents a single test (e.g. `--fully-parallel`).
+
+```yaml
+         [  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12]
+Shard 1:    ^               ^               ^              : [  1, 5, 9 ]
+Shard 2:        ^               ^               ^          : [  2, 6,10 ]
+Shard 3:            ^               ^               ^      : [  3, 7,11 ]
+Shard 4:                ^               ^               ^  : [  4, 8,12 ]
+```
+
+<details>
+<summary>More complex scenario</summary>
+
+Below is a scenario where tests [ 2 and 3 ], [ 4, 5 and 6 ] and [ 9 and 10 ] are executed in a test group which affects how tests are spread across the shards.
+
+```yaml
+Original Order: [ [1], [2, 3], [4, 5, 6], [7], [8], [9, 10], [11], [12] ]
+Sorted Order:   [ [4, 5, 6], [2, 3], [9, 10], [1], [7], [8], [11], [12] ]
+Shard 1:           ^-----^                                                : [ [ 4,   5,   6] ]
+Shard 2:                      ^--^                       ^                : [ [ 2,  3],  [8] ]
+Shard 3:                              ^---^                    ^          : [ [ 9, 10], [11] ]
+Shard 4:                                       ^    ^                ^    : [ [1], [7], [12] ]
+```
+
+</details>
+
+## `shardingMode: 'duration-round-robin'`
+
+Very similar to `round-robin`, but uses the duration of a tests previous run as cost factor. The duration will be read from `.last-run.json` when available. When a test can not be found in `.last-run.json` it will use the average duration of available tests. When no last run info is available, the behavior is identical to `round-robin`.
+
+As an example, consider we have 12 tests and test 7 and 8 take 5 seconds, test 10 and 11 takes 3 seconds and all other tests take 1 second to execute.
+
+```yaml
+Original Order: [  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12]
+Sorted Order:   [  7,  8, 10, 11,  1,  2,  3,  4,  5,  6,  9, 12]
+Shard 1:           ^                               ^              : [  7, 5 ]
+Shard 2:               ^                               ^          : [  8, 6 ]
+Shard 3:                   ^       ^       ^               ^      : [ 10, 1, 3, 9 ]
+Shard 4:                       ^       ^       ^               ^  : [ 11, 2, 4, 12 ]
+```
+
+All shards would have an execution time of around 6 seconds...
+* Shard 1 would execute tests 5 <sup>(1s)</sup> and 7 <sup>(5s)</sup> in 6 seconds.
+* Shard 2 would execute tests 6 <sup>(1s)</sup> and 8 <sup>(5s)</sup> in 6 seconds.
+* Shard 3 would execute tests 1 <sup>(1s)</sup>, 3 <sup>(1s)</sup>, 9 <sup>(1s)</sup> and 10 <sup>(3s)</sup> in 6 seconds.
+* Shard 4 would execute tests 2 <sup>(1s)</sup>, 4 <sup>(1s)</sup>, 11 <sup>(3s)</sup> and 12 <sup>(1s)</sup> in 6 seconds.
+
diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts
@@ -17,7 +17,7 @@
 import fs from 'fs';
 import path from 'path';
 import os from 'os';
-import type { Config, Fixtures, Project, ReporterDescription } from '../../types/test';
+import type { Config, Fixtures, ShardingMode, Project, ReporterDescription } from '../../types/test';
 import type { Location } from '../../types/testReporter';
 import type { TestRunnerPluginRegistration } from '../plugins';
 import { getPackageJsonPath, mergeObjects } from '../util';
@@ -56,6 +56,8 @@ export class FullConfigInternal {
   cliLastFailed?: boolean;
   testIdMatcher?: Matcher;
   defineConfigWasUsed = false;
+  shardingMode: ShardingMode;
+  lastRunFile: string | undefined;
 
   constructor(location: ConfigLocation, userConfig: Config, configCLIOverrides: ConfigCLIOverrides) {
     if (configCLIOverrides.projects && userConfig.projects)
@@ -93,6 +95,8 @@ export class FullConfigInternal {
       workers: 0,
       webServer: null,
     };
+    this.shardingMode = takeFirst(configCLIOverrides.shardingMode, userConfig.shardingMode, 'partition');
+    this.lastRunFile = configCLIOverrides.lastRunFile;
     for (const key in userConfig) {
       if (key.startsWith('@'))
         (this.config as any)[key] = (userConfig as any)[key];
@@ -289,4 +293,4 @@ const configInternalSymbol = Symbol('configInternalSymbol');
 
 export function getProjectId(project: FullProject): string {
   return (project as any).__projectId!;
-}
+}
diff --git a/packages/playwright/src/common/configLoader.ts b/packages/playwright/src/common/configLoader.ts
@@ -225,6 +225,11 @@ function validateConfig(file: string, config: Config) {
       throw errorWithFile(file, `config.shard.current must be a positive number, not greater than config.shard.total`);
   }
 
+  if ('shardingMode' in config && config.shardingMode !== undefined) {
+    if (typeof config.shardingMode !== 'string' || !['partition', 'round-robin', 'duration-round-robin'].includes(config.shardingMode))
+      throw errorWithFile(file, `config.shardingMode must be one of "partition", "round-robin" or "duration-round-robin"`);
+  }
+
   if ('updateSnapshots' in config && config.updateSnapshots !== undefined) {
     if (typeof config.updateSnapshots !== 'string' || !['all', 'none', 'missing'].includes(config.updateSnapshots))
       throw errorWithFile(file, `config.updateSnapshots must be one of "all", "none" or "missing"`);
@@ -341,9 +346,9 @@ export async function loadConfigFromFileRestartIfNeeded(configFile: string | und
   return await loadConfig(location, overrides, ignoreDeps);
 }
 
-export async function loadEmptyConfigForMergeReports() {
+export async function loadEmptyConfigForMergeReports(overrides?: ConfigCLIOverrides) {
   // Merge reports is "different" for no good reason. It should not pick up local config from the cwd.
-  return await loadConfig({ configDir: process.cwd() });
+  return await loadConfig({ configDir: process.cwd() }, overrides);
 }
 
 export function restartWithExperimentalTsEsm(configFile: string | undefined, force: boolean = false): boolean {

diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts
@@ -17,7 +17,7 @@
 import util from 'util';
 import { type SerializedCompilationCache, serializeCompilationCache } from '../transform/compilationCache';
 import type { ConfigLocation, FullConfigInternal } from './config';
-import type { ReporterDescription, TestInfoError, TestStatus } from '../../types/test';
+import type { PlaywrightTestConfig, ReporterDescription, TestInfoError, TestStatus } from '../../types/test';
 
 export type ConfigCLIOverrides = {
   debug?: boolean;
@@ -33,6 +33,8 @@ export type ConfigCLIOverrides = {
   reporter?: ReporterDescription[];
   additionalReporters?: ReporterDescription[];
   shard?: { current: number, total: number };
+  shardingMode?: PlaywrightTestConfig['shardingMode'];
+  lastRunFile?: string;
   timeout?: number;
   tsconfig?: string;
   ignoreSnapshots?: boolean;

diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts
@@ -27,7 +27,7 @@ import { createMergedReport } from './reporters/merge';
 import { loadConfigFromFileRestartIfNeeded, loadEmptyConfigForMergeReports, resolveConfigLocation } from './common/configLoader';
 import type { ConfigCLIOverrides } from './common/ipc';
 import type { TestError } from '../types/testReporter';
-import type { TraceMode } from '../types/test';
+import type { TraceMode, ShardingMode } from '../types/test';
 import { builtInReporters, defaultReporter, defaultTimeout } from './common/config';
 import { program } from 'playwright-core/lib/cli/program';
 export { program } from 'playwright-core/lib/cli/program';
@@ -145,6 +145,7 @@ function addMergeReportsCommand(program: Command) {
   });
   command.option('-c, --config <file>', `Configuration file. Can be used to specify additional configuration for the output report.`);
   command.option('--reporter <reporter>', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`);
+  command.option('--last-run-file <file>', `Path to a json file where the last run information is written to (default: test-results/.last-run.json)`);
   command.addHelpText('afterAll', `
 Arguments [dir]:
   Directory containing blob reports.
@@ -264,7 +265,8 @@ async function listTestFiles(opts: { [key: string]: any }) {
 
 async function mergeReports(reportDir: string | undefined, opts: { [key: string]: any }) {
   const configFile = opts.config;
-  const config = configFile ? await loadConfigFromFileRestartIfNeeded(configFile) : await loadEmptyConfigForMergeReports();
+  const cliOverrides = overridesFromOptions(opts);
+  const config = configFile ? await loadConfigFromFileRestartIfNeeded(configFile, cliOverrides) : await loadEmptyConfigForMergeReports(cliOverrides);
   if (!config)
     return;
 
@@ -297,6 +299,8 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid
     retries: options.retries ? parseInt(options.retries, 10) : undefined,
     reporter: resolveReporterOption(options.reporter),
     shard: shardPair ? { current: shardPair[0], total: shardPair[1] } : undefined,
+    shardingMode: options.shardingMode ? resolveShardingModeOption(options.shardingMode) : undefined,
+    lastRunFile: options.lastRunFile ? path.resolve(process.cwd(), options.lastRunFile) : undefined,
     timeout: options.timeout ? parseInt(options.timeout, 10) : undefined,
     tsconfig: options.tsconfig ? path.resolve(process.cwd(), options.tsconfig) : undefined,
     ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined,
@@ -332,6 +336,16 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid
   return overrides;
 }
 
+const shardingModes: ShardingMode[] = ['partition', 'round-robin', 'duration-round-robin'];
+
+function resolveShardingModeOption(shardingMode?: string): ShardingMode | undefined {
+  if (!shardingMode)
+    return undefined;
+  if (!shardingModes.includes(shardingMode as ShardingMode))
+    throw new Error(`Unsupported sharding mode "${shardingMode}", must be one of: ${shardingModes.map(mode => `"${mode}"`).join(', ')}`);
+  return shardingMode as ShardingMode;
+}
+
 function resolveReporterOption(reporter?: string): ReporterDescription[] | undefined {
   if (!reporter || !reporter.length)
     return undefined;
@@ -362,6 +376,7 @@ const testOptions: [string, string][] = [
   ['--headed', `Run tests in headed browsers (default: headless)`],
   ['--ignore-snapshots', `Ignore screenshot and snapshot expectations`],
   ['--last-failed', `Only re-run the failures`],
+  ['--last-run-file <file>', `Path to a json file where the last run information is read from and written to (default: test-results/.last-run.json)`],
   ['--list', `Collect all the tests and report them, but do not run`],
   ['--max-failures <N>', `Stop after the first N failures`],
   ['--no-deps', 'Do not run project dependencies'],
@@ -374,6 +389,7 @@ const testOptions: [string, string][] = [
   ['--reporter <reporter>', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`],
   ['--retries <retries>', `Maximum retry count for flaky tests, zero for no retries (default: no retries)`],
   ['--shard <shard>', `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"`],
+  ['--sharding-mode <mode>', `Sharding algorithm to use; "partition", "round-robin" or "duration-round-robin". Defaults to "partition".`],
   ['--timeout <timeout>', `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${defaultTimeout})`],
   ['--trace <mode>', `Force tracing mode, can be ${kTraceModes.map(mode => `"${mode}"`).join(', ')}`],
   ['--tsconfig <path>', `Path to a single tsconfig applicable to all imported files (default: look up tsconfig for each imported file separately)`],

diff --git a/packages/playwright/src/reporters/DEPS.list b/packages/playwright/src/reporters/DEPS.list
@@ -6,6 +6,7 @@
 
 [merge.ts]
 ../runner/reporters.ts
+../runner/lastRun.ts
 
 [internalReporter.ts]
 ../transform/babelBundle.ts

diff --git a/packages/playwright/src/reporters/merge.ts b/packages/playwright/src/reporters/merge.ts
@@ -28,6 +28,7 @@ import { currentBlobReportVersion, type BlobReportMetadata } from './blob';
 import { relativeFilePath } from '../util';
 import type { TestError } from '../../types/testReporter';
 import type * as blobV1 from './versions/blobV1';
+import { LastRunReporter } from '../runner/lastRun';
 
 type StatusCallback = (message: string) => void;
 
@@ -39,7 +40,8 @@ type ReportData = {
 
 export async function createMergedReport(config: FullConfigInternal, dir: string, reporterDescriptions: ReporterDescription[], rootDirOverride: string | undefined) {
   const reporters = await createReporters(config, 'merge', false, reporterDescriptions);
-  const multiplexer = new Multiplexer(reporters);
+  const lastRun = new LastRunReporter(config);
+  const multiplexer = new Multiplexer([...reporters, lastRun]);
   const stringPool = new StringInternPool();
 
   let printStatus: StatusCallback = () => {};

diff --git a/packages/playwright/src/runner/lastRun.ts b/packages/playwright/src/runner/lastRun.ts
@@ -21,9 +21,10 @@ import { filterProjects } from './projectUtils';
 import type { FullConfigInternal } from '../common/config';
 import type { ReporterV2 } from '../reporters/reporterV2';
 
-type LastRunInfo = {
+export type LastRunInfo = {
   status: FullResult['status'];
   failedTests: string[];
+  testDurations?: { [testId: string]: number };
 };
 
 export class LastRunReporter implements ReporterV2 {
@@ -33,21 +34,32 @@ export class LastRunReporter implements ReporterV2 {
 
   constructor(config: FullConfigInternal) {
     this._config = config;
-    const [project] = filterProjects(config.projects, config.cliProjectFilter);
-    if (project)
-      this._lastRunFile = path.join(project.project.outputDir, '.last-run.json');
+    if (config.lastRunFile) {
+      // specified via command line argument
+      this._lastRunFile = config.lastRunFile;
+    } else {
+      const [project] = filterProjects(config.projects, config.cliProjectFilter);
+      if (project)
+        this._lastRunFile = path.join(project.project.outputDir, '.last-run.json');
+    }
   }
 
-  async filterLastFailed() {
+  async lastRunInfo(): Promise<LastRunInfo | undefined> {
     if (!this._lastRunFile)
       return;
     try {
-      const lastRunInfo = JSON.parse(await fs.promises.readFile(this._lastRunFile, 'utf8')) as LastRunInfo;
-      this._config.testIdMatcher = id => lastRunInfo.failedTests.includes(id);
+      return JSON.parse(await fs.promises.readFile(this._lastRunFile, 'utf8'));
     } catch {
     }
   }
 
+  async filterLastFailed() {
+    const lastRunInfo = await this.lastRunInfo();
+    if (!lastRunInfo)
+      return;
+    this._config.testIdMatcher = id => lastRunInfo.failedTests.includes(id);
+  }
+
   version(): 'v2' {
     return 'v2';
   }
@@ -65,7 +77,11 @@ export class LastRunReporter implements ReporterV2 {
       return;
     await fs.promises.mkdir(path.dirname(this._lastRunFile), { recursive: true });
     const failedTests = this._suite?.allTests().filter(t => !t.ok()).map(t => t.id);
-    const lastRunReport = JSON.stringify({ status: result.status, failedTests }, undefined, 2);
+    const testDurations = this._suite?.allTests().reduce((map, t) => {
+      map[t.id] = t.results.map(r => r.duration).reduce((a, b) => a + b, 0);
+      return map;
+    }, {} as { [key: string]: number });
+    const lastRunReport = JSON.stringify({ status: result.status, failedTests, testDurations }, undefined, 2);
     await fs.promises.writeFile(this._lastRunFile, lastRunReport);
   }
 }
diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts
@@ -180,7 +180,7 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
       testGroups.push(...createTestGroups(projectSuite, config.config.workers));
 
     // Shard test groups.
-    const testGroupsInThisShard = filterForShard(config.config.shard, testGroups);
+    const testGroupsInThisShard = await filterForShard(config, testGroups);
     const testsInThisShard = new Set<TestCase>();
     for (const group of testGroupsInThisShard) {
       for (const test of group.tests)