From 58c330f9892532650b73bda9fdfafc224b135f71 Mon Sep 17 00:00:00 2001 From: Javier Calvarro Nelson Date: Fri, 29 May 2026 17:19:10 +0200 Subject: [PATCH 1/7] Add MCP servers to dotnet-blazor plugin Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/allowed-external-deps.txt | 6 ++++++ plugins/dotnet-blazor/plugin.json | 16 +++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/eng/allowed-external-deps.txt b/eng/allowed-external-deps.txt index a231c8b91e..ecb17a7f99 100644 --- a/eng/allowed-external-deps.txt +++ b/eng/allowed-external-deps.txt @@ -25,3 +25,9 @@ tool-ref:msbuild:#tool:agent/runSubagent # dotnet-msbuild plugin: Microsoft.AITools.BinlogMcp server (binlog analysis MCP from dotnet-eng feed) mcp-server:dotnet-msbuild:binlog + +# dotnet-blazor plugin: Microsoft Learn documentation MCP server +mcp-server:dotnet-blazor:microsoft-docs + +# dotnet-blazor plugin: Playwright browser automation MCP server +mcp-server:dotnet-blazor:playwright diff --git a/plugins/dotnet-blazor/plugin.json b/plugins/dotnet-blazor/plugin.json index 1c0691c890..a9ef245013 100644 --- a/plugins/dotnet-blazor/plugin.json +++ b/plugins/dotnet-blazor/plugin.json @@ -2,5 +2,19 @@ "name": "dotnet-blazor", "version": "0.1.0", "description": "Skills for Blazor development: component authoring, interactivity, and web application patterns.", - "skills": ["./skills/"] + "skills": ["./skills/"], + "mcpServers": { + "microsoft-docs": { + "type": "http", + "url": "https://learn.microsoft.com/api/mcp", + "headers": {}, + "tools": ["*"] + }, + "playwright": { + "type": "local", + "command": "npx", + "args": ["@playwright/mcp@latest"], + "tools": ["*"] + } + } } From c12dffff09e7e517817630c902515f18770dd270 Mon Sep 17 00:00:00 2001 From: Javier Calvarro Nelson Date: Fri, 29 May 2026 17:27:34 +0200 Subject: [PATCH 2/7] Handle non-stdio MCP servers in skill-validator Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/Evaluate/AgentRunner.cs | 26 ++++++++++---- eng/skill-validator/src/Shared/Models.cs | 8 +++-- .../src/docs/InvestigatingResults.md | 2 ++ .../tests/Evaluate/EvalDiscoveryTests.cs | 36 +++++++++++++++++++ .../tests/Evaluate/RunnerTests.cs | 34 +++++++++++++++++- 5 files changed, 95 insertions(+), 11 deletions(-) diff --git a/eng/skill-validator/src/Evaluate/AgentRunner.cs b/eng/skill-validator/src/Evaluate/AgentRunner.cs index 4d17da2fce..c092c7f111 100644 --- a/eng/skill-validator/src/Evaluate/AgentRunner.cs +++ b/eng/skill-validator/src/Evaluate/AgentRunner.cs @@ -292,18 +292,25 @@ internal static async Task BuildSessionConfig( sdkMcp = new Dictionary(); foreach (var (name, def) in mcpServers) { - if (!IsAllowedMcpCommand(def.Command)) + // Only stdio servers are supported; reject unknown types early. + if (def.Type is not null and not "stdio") { Console.Error.WriteLine( - $"Skipping MCP server '{name}': command '{def.Command}' is not in the allowlist"); + $"Skipping MCP server '{name}': unsupported type '{def.Type}' (only 'stdio' is supported)"); continue; } - // Only stdio servers are supported; reject unknown types early. - if (def.Type is not null and not "stdio") + if (string.IsNullOrWhiteSpace(def.Command)) { Console.Error.WriteLine( - $"Skipping MCP server '{name}': unsupported type '{def.Type}' (only 'stdio' is supported)"); + $"Skipping MCP server '{name}': missing required command for stdio MCP server"); + continue; + } + + if (!IsAllowedMcpCommand(def.Command)) + { + Console.Error.WriteLine( + $"Skipping MCP server '{name}': command '{def.Command}' is not in the allowlist"); continue; } @@ -945,8 +952,11 @@ internal static void ScrubSensitiveEnvironment(ProcessStartInfo psi) "dotnet", "dnx", "node", "npx", "python", "python3", "uvx", }; - internal static bool IsAllowedMcpCommand(string command) + internal static bool IsAllowedMcpCommand(string? command) { + if (string.IsNullOrWhiteSpace(command)) + return false; + // Only allow bare command names (resolved via PATH), not paths. if (command.Contains(Path.DirectorySeparatorChar) || command.Contains(Path.AltDirectorySeparatorChar) || @@ -1005,8 +1015,10 @@ internal static bool IsAllowedMcpCommand(string command) ["uvx"] = new(StringComparer.Ordinal) { "--from" }, }; - internal static string[]? SanitizeMcpArgs(string command, string[] args) + internal static string[]? SanitizeMcpArgs(string command, string[]? args) { + args ??= []; + var cmdName = Path.GetFileNameWithoutExtension(command); if (!DangerousMcpArgs.TryGetValue(cmdName, out var blocked)) return args; diff --git a/eng/skill-validator/src/Shared/Models.cs b/eng/skill-validator/src/Shared/Models.cs index 1b032c5d13..04c134a0f8 100644 --- a/eng/skill-validator/src/Shared/Models.cs +++ b/eng/skill-validator/src/Shared/Models.cs @@ -3,12 +3,14 @@ namespace SkillValidator.Shared; // --- MCP server definition (from plugin.json) --- public sealed record MCPServerDef( - string Command, - string[] Args, + string? Command = null, + string[]? Args = null, string? Type = null, string[]? Tools = null, Dictionary? Env = null, - string? Cwd = null); + string? Cwd = null, + string? Url = null, + Dictionary? Headers = null); // --- Skill info --- diff --git a/eng/skill-validator/src/docs/InvestigatingResults.md b/eng/skill-validator/src/docs/InvestigatingResults.md index 7616caf9c5..070e9d709e 100644 --- a/eng/skill-validator/src/docs/InvestigatingResults.md +++ b/eng/skill-validator/src/docs/InvestigatingResults.md @@ -83,6 +83,8 @@ Each scenario includes two required runs (baseline + isolated). It may also incl > **Note:** Scenarios do not have a `passed` field. To determine pass/fail for an individual scenario, check whether `improvementScore >= 0`. This is the effective score: when no plugin run is present it equals `isolatedImprovementScore`; when a plugin run is present it is the min of isolated and plugin scores. The `passed` field exists only at the verdict level (per-skill). +> **Note:** During evaluation, skill-validator currently loads only stdio MCP servers. Plugin MCP entries with other types (for example `type: http`) are skipped with a stderr message instead of failing the run. + ### Breakdown fields The `isolatedBreakdown` and `pluginBreakdown` objects show how each metric contributed to the improvement score. Each field is a raw delta (not yet weighted). The final score is computed as a weighted sum: diff --git a/eng/skill-validator/tests/Evaluate/EvalDiscoveryTests.cs b/eng/skill-validator/tests/Evaluate/EvalDiscoveryTests.cs index c28e4e8c7e..f2de8acf55 100644 --- a/eng/skill-validator/tests/Evaluate/EvalDiscoveryTests.cs +++ b/eng/skill-validator/tests/Evaluate/EvalDiscoveryTests.cs @@ -71,6 +71,42 @@ public async Task FindsPluginMcpServersInGrandparentDirectory() } } + [Fact] + public async Task FindsHttpPluginMcpServersWithoutSynthesizingStdioFields() + { + var tmpDir = Path.Combine(Path.GetTempPath(), $"skill-test-{Guid.NewGuid():N}"); + var skillDir = Path.Combine(tmpDir, "skills", "my-skill"); + Directory.CreateDirectory(skillDir); + try + { + var pluginJson = """ + { + "mcpServers": { + "microsoft-docs": { + "type": "http", + "url": "https://learn.microsoft.com/api/mcp", + "headers": {}, + "tools": ["*"] + } + } + } + """; + await File.WriteAllTextAsync(Path.Combine(tmpDir, "plugin.json"), pluginJson, TestContext.Current.CancellationToken); + + var result = await EvaluateCommand.FindPluginMcpServers(skillDir); + Assert.NotNull(result); + Assert.True(result!.ContainsKey("microsoft-docs")); + Assert.Equal("http", result["microsoft-docs"].Type); + Assert.Equal("https://learn.microsoft.com/api/mcp", result["microsoft-docs"].Url); + Assert.Null(result["microsoft-docs"].Command); + Assert.Null(result["microsoft-docs"].Args); + } + finally + { + Directory.Delete(tmpDir, true); + } + } + [Fact] public async Task ReturnsNullWhenNoPluginJson() { diff --git a/eng/skill-validator/tests/Evaluate/RunnerTests.cs b/eng/skill-validator/tests/Evaluate/RunnerTests.cs index 610fdbee52..8759a16771 100644 --- a/eng/skill-validator/tests/Evaluate/RunnerTests.cs +++ b/eng/skill-validator/tests/Evaluate/RunnerTests.cs @@ -373,6 +373,31 @@ public async Task FiltersOutDisallowedMcpServersButKeepsAllowed() Assert.False(config.McpServers.ContainsKey("bad")); } + [Fact] + public async Task SkipsUnsupportedHttpMcpServerWithoutCrashing() + { + var mcpServers = new Dictionary + { + ["microsoft-docs"] = new(Type: "http", Url: "https://learn.microsoft.com/api/mcp", Tools: ["*"]), + ["good"] = new(Command: "node", Args: ["server.js"], Tools: ["*"]), + }; + var config = await AgentRunner.BuildSessionConfig(MockSkill, null, "gpt-4.1", "C:\\tmp\\work", mcpServers); + Assert.NotNull(config.McpServers); + Assert.True(config.McpServers.ContainsKey("good")); + Assert.False(config.McpServers.ContainsKey("microsoft-docs")); + } + + [Fact] + public async Task SkipsStdioMcpServerWithoutCommand() + { + var mcpServers = new Dictionary + { + ["broken"] = new(Args: ["server.js"], Tools: ["*"]), + }; + var config = await AgentRunner.BuildSessionConfig(MockSkill, null, "gpt-4.1", "C:\\tmp\\work", mcpServers); + Assert.Null(config.McpServers); + } + [Fact] public async Task RejectsMcpServerWithDangerousArgs() { @@ -520,7 +545,14 @@ public void ReturnsNullWhenKeyIsNotString() public class IsAllowedMcpCommandTests { + [Fact] + public void RejectsNullCommand() + { + Assert.False(AgentRunner.IsAllowedMcpCommand(null)); + } + [Theory] + [InlineData("", false)] [InlineData("dotnet", true)] [InlineData("node", true)] [InlineData("npx", true)] @@ -533,7 +565,7 @@ public class IsAllowedMcpCommandTests [InlineData("wget", false)] [InlineData("cmd", false)] [InlineData("powershell", false)] - public void ValidatesCommand(string command, bool expected) + public void ValidatesCommand(string? command, bool expected) { Assert.Equal(expected, AgentRunner.IsAllowedMcpCommand(command)); } From df9625d0da13bcf1b675ebb98c8b90f9b83796c7 Mon Sep 17 00:00:00 2001 From: Javier Calvarro Nelson Date: Fri, 29 May 2026 17:51:43 +0200 Subject: [PATCH 3/7] Use stdio for Playwright MCP server Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- plugins/dotnet-blazor/plugin.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/dotnet-blazor/plugin.json b/plugins/dotnet-blazor/plugin.json index a9ef245013..f41ece7b33 100644 --- a/plugins/dotnet-blazor/plugin.json +++ b/plugins/dotnet-blazor/plugin.json @@ -11,7 +11,7 @@ "tools": ["*"] }, "playwright": { - "type": "local", + "type": "stdio", "command": "npx", "args": ["@playwright/mcp@latest"], "tools": ["*"] From a0be15f915258691ce728cd69b263348cb6297be Mon Sep 17 00:00:00 2001 From: Javier Calvarro Nelson Date: Fri, 29 May 2026 18:02:09 +0200 Subject: [PATCH 4/7] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- plugins/dotnet-blazor/plugin.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/dotnet-blazor/plugin.json b/plugins/dotnet-blazor/plugin.json index f41ece7b33..4b9d281b4e 100644 --- a/plugins/dotnet-blazor/plugin.json +++ b/plugins/dotnet-blazor/plugin.json @@ -13,7 +13,7 @@ "playwright": { "type": "stdio", "command": "npx", - "args": ["@playwright/mcp@latest"], + "args": ["-y", "@playwright/mcp@latest"], "tools": ["*"] } } From 463bb057283700db226d74df564ce3845d16431f Mon Sep 17 00:00:00 2001 From: Javier Calvarro Nelson Date: Fri, 29 May 2026 18:12:45 +0200 Subject: [PATCH 5/7] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- plugins/dotnet-blazor/plugin.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/dotnet-blazor/plugin.json b/plugins/dotnet-blazor/plugin.json index 4b9d281b4e..5579a23fef 100644 --- a/plugins/dotnet-blazor/plugin.json +++ b/plugins/dotnet-blazor/plugin.json @@ -10,11 +10,13 @@ "headers": {}, "tools": ["*"] }, + "playwright": { "playwright": { "type": "stdio", "command": "npx", - "args": ["-y", "@playwright/mcp@latest"], + "args": ["@playwright/mcp@latest"], "tools": ["*"] } + } } } From 579fcb7269f9d16cd40d4fe9521d1fb1f747fbc0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 29 May 2026 16:17:22 +0000 Subject: [PATCH 6/7] fix: correct dotnet-blazor playwright MCP manifest format --- plugins/dotnet-blazor/plugin.json | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/plugins/dotnet-blazor/plugin.json b/plugins/dotnet-blazor/plugin.json index 5579a23fef..083b2d44cf 100644 --- a/plugins/dotnet-blazor/plugin.json +++ b/plugins/dotnet-blazor/plugin.json @@ -2,21 +2,27 @@ "name": "dotnet-blazor", "version": "0.1.0", "description": "Skills for Blazor development: component authoring, interactivity, and web application patterns.", - "skills": ["./skills/"], + "skills": [ + "./skills/" + ], "mcpServers": { "microsoft-docs": { "type": "http", "url": "https://learn.microsoft.com/api/mcp", "headers": {}, - "tools": ["*"] + "tools": [ + "*" + ] }, - "playwright": { "playwright": { "type": "stdio", "command": "npx", - "args": ["@playwright/mcp@latest"], - "tools": ["*"] - } + "args": [ + "@playwright/mcp@latest" + ], + "tools": [ + "*" + ] } } } From 8a00c7e65386eb3b4ea68dacd5f923a766682b70 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Jun 2026 10:01:40 +0200 Subject: [PATCH 7/7] [WIP] Address feedback on MCP servers for dotnet-blazor plugin integration (#723) * Initial plan * feat(dotnet-blazor): pin playwright MCP to 0.0.75 and add weekly update workflow * fix(update-playwright-mcp-version): use env vars in node scripts and descending sort --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> --- .../update-playwright-mcp-version.yml | 155 ++++++++++++++++++ plugins/dotnet-blazor/plugin.json | 2 +- 2 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/update-playwright-mcp-version.yml diff --git a/.github/workflows/update-playwright-mcp-version.yml b/.github/workflows/update-playwright-mcp-version.yml new file mode 100644 index 0000000000..e8bf07b2a2 --- /dev/null +++ b/.github/workflows/update-playwright-mcp-version.yml @@ -0,0 +1,155 @@ +name: Update Playwright MCP version for dotnet-blazor + +on: + schedule: + - cron: '0 9 * * 1' # Weekly on Monday at 9:00 UTC + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + update-version: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: true + + - name: Use Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 'lts/*' + + - name: Find latest stable Playwright MCP version published more than 24h ago + id: find-version + run: | + set -euo pipefail + + # Fetch all version publish times from the npm registry + NPM_TIMES=$(npm view @playwright/mcp time --json) + + # Find the latest stable version published more than 24h ago + # Pass the JSON via env var to avoid double-quote conflicts in bash interpolation + LATEST=$(NPM_TIMES="$NPM_TIMES" node -e ' + const times = JSON.parse(process.env.NPM_TIMES); + const now = Date.now(); + const threshold = now - 24 * 60 * 60 * 1000; + + const candidates = Object.entries(times) + .filter(([v]) => v !== "modified" && v !== "created") + .filter(([v]) => !/-(alpha|beta|rc|next|dev|canary)/.test(v)) + .filter(([, t]) => new Date(t).getTime() < threshold) + .sort(([, a], [, b]) => new Date(b) - new Date(a)); + + const latest = candidates[0]; + if (latest) { + process.stdout.write(latest[0]); + } + ') + + if [ -z "$LATEST" ]; then + echo "No eligible version found. Exiting." + echo "latest_version=" >> "$GITHUB_OUTPUT" + else + echo "Latest eligible version: $LATEST" + echo "latest_version=$LATEST" >> "$GITHUB_OUTPUT" + fi + + - name: Check current version in plugin.json + id: check-version + if: steps.find-version.outputs.latest_version != '' + run: | + set -euo pipefail + + CURRENT=$(node -e " + const fs = require('fs'); + const plugin = JSON.parse(fs.readFileSync('plugins/dotnet-blazor/plugin.json', 'utf8')); + const args = plugin.mcpServers.playwright.args; + const entry = args.find(a => a.startsWith('@playwright/mcp@')); + if (entry) { + process.stdout.write(entry.replace('@playwright/mcp@', '')); + } + ") + + echo "Current version: $CURRENT" + echo "Latest version: ${{ steps.find-version.outputs.latest_version }}" + echo "current_version=$CURRENT" >> "$GITHUB_OUTPUT" + + if [ "$CURRENT" = "${{ steps.find-version.outputs.latest_version }}" ]; then + echo "already_up_to_date=true" >> "$GITHUB_OUTPUT" + else + echo "already_up_to_date=false" >> "$GITHUB_OUTPUT" + fi + + - name: Update plugin.json with new version + if: steps.check-version.outputs.already_up_to_date == 'false' + run: | + set -euo pipefail + + NEW_VERSION="${{ steps.find-version.outputs.latest_version }}" + CURRENT_VERSION="${{ steps.check-version.outputs.current_version }}" + + NEW_PLAYWRIGHT_VERSION="$NEW_VERSION" node -e ' + const fs = require("fs"); + const pluginPath = "plugins/dotnet-blazor/plugin.json"; + const plugin = JSON.parse(fs.readFileSync(pluginPath, "utf8")); + const args = plugin.mcpServers.playwright.args; + const idx = args.findIndex(a => a.startsWith("@playwright/mcp@")); + if (idx !== -1) { + args[idx] = "@playwright/mcp@" + process.env.NEW_PLAYWRIGHT_VERSION; + } + fs.writeFileSync(pluginPath, JSON.stringify(plugin, null, 2) + "\n"); + ' + + echo "Updated plugin.json: @playwright/mcp@$CURRENT_VERSION -> @playwright/mcp@$NEW_VERSION" + + - name: Setup .NET SDK + if: steps.check-version.outputs.already_up_to_date == 'false' + uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5 + with: + global-json-file: global.json + + - name: Validate plugin with skill-validator + if: steps.check-version.outputs.already_up_to_date == 'false' + run: | + set -euo pipefail + + dotnet publish eng/skill-validator/src/SkillValidator.csproj + artifacts/publish/SkillValidator/release/skill-validator check \ + --plugin plugins/dotnet-blazor \ + --allowed-external-deps eng/allowed-external-deps.txt \ + --known-domains eng/known-domains.txt + env: + DOTNET_NOLOGO: true + + - name: Create PR + if: steps.check-version.outputs.already_up_to_date == 'false' + run: | + set -euo pipefail + + NEW_VERSION="${{ steps.find-version.outputs.latest_version }}" + TODAY=$(date -u '+%Y-%m-%d') + BRANCH="automated/dotnet-blazor-playwright-mcp-${NEW_VERSION}" + TITLE="[dotnet-blazor] ${TODAY} - Update playwright MCP version to ${NEW_VERSION}" + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -B "$BRANCH" + git add plugins/dotnet-blazor/plugin.json + git commit -m "chore(dotnet-blazor): update playwright MCP to $NEW_VERSION" + git push -f origin "$BRANCH" + + existing_pr=$(gh pr list --head "$BRANCH" --json number --jq '.[0].number' 2>/dev/null || true) + if [ -n "$existing_pr" ]; then + gh pr edit "$existing_pr" --title "$TITLE" + else + gh pr create \ + --title "$TITLE" \ + --body "Automated weekly update of the \`@playwright/mcp\` version in \`plugins/dotnet-blazor/plugin.json\` from \`${{ steps.check-version.outputs.current_version }}\` to \`${NEW_VERSION}\`." \ + --head "$BRANCH" + fi + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/plugins/dotnet-blazor/plugin.json b/plugins/dotnet-blazor/plugin.json index 083b2d44cf..17357ede7e 100644 --- a/plugins/dotnet-blazor/plugin.json +++ b/plugins/dotnet-blazor/plugin.json @@ -18,7 +18,7 @@ "type": "stdio", "command": "npx", "args": [ - "@playwright/mcp@latest" + "@playwright/mcp@0.0.75" ], "tools": [ "*"