Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
155 changes: 155 additions & 0 deletions .github/workflows/update-playwright-mcp-version.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
name: Update Playwright MCP version for dotnet-blazor

on:
schedule:
- cron: '0 9 * * 1' # Weekly on Monday at 9:00 UTC
workflow_dispatch:

permissions:
contents: write
pull-requests: write

jobs:
update-version:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: true

- name: Use Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: 'lts/*'

- name: Find latest stable Playwright MCP version published more than 24h ago
id: find-version
run: |
set -euo pipefail

# Fetch all version publish times from the npm registry
NPM_TIMES=$(npm view @playwright/mcp time --json)

# Find the latest stable version published more than 24h ago
# Pass the JSON via env var to avoid double-quote conflicts in bash interpolation
LATEST=$(NPM_TIMES="$NPM_TIMES" node -e '
const times = JSON.parse(process.env.NPM_TIMES);
const now = Date.now();
const threshold = now - 24 * 60 * 60 * 1000;

const candidates = Object.entries(times)
.filter(([v]) => v !== "modified" && v !== "created")
.filter(([v]) => !/-(alpha|beta|rc|next|dev|canary)/.test(v))
.filter(([, t]) => new Date(t).getTime() < threshold)
.sort(([, a], [, b]) => new Date(b) - new Date(a));

const latest = candidates[0];
if (latest) {
process.stdout.write(latest[0]);
}
')

if [ -z "$LATEST" ]; then
echo "No eligible version found. Exiting."
echo "latest_version=" >> "$GITHUB_OUTPUT"
else
echo "Latest eligible version: $LATEST"
echo "latest_version=$LATEST" >> "$GITHUB_OUTPUT"
fi

- name: Check current version in plugin.json
id: check-version
if: steps.find-version.outputs.latest_version != ''
run: |
set -euo pipefail

CURRENT=$(node -e "
const fs = require('fs');
const plugin = JSON.parse(fs.readFileSync('plugins/dotnet-blazor/plugin.json', 'utf8'));
const args = plugin.mcpServers.playwright.args;
const entry = args.find(a => a.startsWith('@playwright/mcp@'));
if (entry) {
process.stdout.write(entry.replace('@playwright/mcp@', ''));
}
")

echo "Current version: $CURRENT"
echo "Latest version: ${{ steps.find-version.outputs.latest_version }}"
echo "current_version=$CURRENT" >> "$GITHUB_OUTPUT"

if [ "$CURRENT" = "${{ steps.find-version.outputs.latest_version }}" ]; then
echo "already_up_to_date=true" >> "$GITHUB_OUTPUT"
else
echo "already_up_to_date=false" >> "$GITHUB_OUTPUT"
fi

- name: Update plugin.json with new version
if: steps.check-version.outputs.already_up_to_date == 'false'
run: |
set -euo pipefail

NEW_VERSION="${{ steps.find-version.outputs.latest_version }}"
CURRENT_VERSION="${{ steps.check-version.outputs.current_version }}"

NEW_PLAYWRIGHT_VERSION="$NEW_VERSION" node -e '
const fs = require("fs");
const pluginPath = "plugins/dotnet-blazor/plugin.json";
const plugin = JSON.parse(fs.readFileSync(pluginPath, "utf8"));
const args = plugin.mcpServers.playwright.args;
const idx = args.findIndex(a => a.startsWith("@playwright/mcp@"));
if (idx !== -1) {
args[idx] = "@playwright/mcp@" + process.env.NEW_PLAYWRIGHT_VERSION;
}
fs.writeFileSync(pluginPath, JSON.stringify(plugin, null, 2) + "\n");
'

echo "Updated plugin.json: @playwright/mcp@$CURRENT_VERSION -> @playwright/mcp@$NEW_VERSION"

- name: Setup .NET SDK
if: steps.check-version.outputs.already_up_to_date == 'false'
uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5
with:
global-json-file: global.json

- name: Validate plugin with skill-validator
if: steps.check-version.outputs.already_up_to_date == 'false'
run: |
set -euo pipefail

dotnet publish eng/skill-validator/src/SkillValidator.csproj
artifacts/publish/SkillValidator/release/skill-validator check \
--plugin plugins/dotnet-blazor \
--allowed-external-deps eng/allowed-external-deps.txt \
--known-domains eng/known-domains.txt
env:
DOTNET_NOLOGO: true

- name: Create PR
if: steps.check-version.outputs.already_up_to_date == 'false'
run: |
set -euo pipefail

NEW_VERSION="${{ steps.find-version.outputs.latest_version }}"
TODAY=$(date -u '+%Y-%m-%d')
BRANCH="automated/dotnet-blazor-playwright-mcp-${NEW_VERSION}"
TITLE="[dotnet-blazor] ${TODAY} - Update playwright MCP version to ${NEW_VERSION}"

git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git checkout -B "$BRANCH"
git add plugins/dotnet-blazor/plugin.json
git commit -m "chore(dotnet-blazor): update playwright MCP to $NEW_VERSION"
git push -f origin "$BRANCH"

existing_pr=$(gh pr list --head "$BRANCH" --json number --jq '.[0].number' 2>/dev/null || true)
if [ -n "$existing_pr" ]; then
gh pr edit "$existing_pr" --title "$TITLE"
else
gh pr create \
--title "$TITLE" \
--body "Automated weekly update of the \`@playwright/mcp\` version in \`plugins/dotnet-blazor/plugin.json\` from \`${{ steps.check-version.outputs.current_version }}\` to \`${NEW_VERSION}\`." \
--head "$BRANCH"
fi
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
6 changes: 6 additions & 0 deletions eng/allowed-external-deps.txt
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,9 @@ tool-ref:msbuild:#tool:agent/runSubagent

# dotnet-msbuild plugin: Microsoft.AITools.BinlogMcp server (binlog analysis MCP from dotnet-eng feed)
mcp-server:dotnet-msbuild:binlog

# dotnet-blazor plugin: Microsoft Learn documentation MCP server
mcp-server:dotnet-blazor:microsoft-docs

# dotnet-blazor plugin: Playwright browser automation MCP server
mcp-server:dotnet-blazor:playwright
26 changes: 19 additions & 7 deletions eng/skill-validator/src/Evaluate/AgentRunner.cs
Original file line number Diff line number Diff line change
Expand Up @@ -292,18 +292,25 @@ internal static async Task<SessionConfig> BuildSessionConfig(
sdkMcp = new Dictionary<string, McpServerConfig>();
foreach (var (name, def) in mcpServers)
{
if (!IsAllowedMcpCommand(def.Command))
// Only stdio servers are supported; reject unknown types early.
if (def.Type is not null and not "stdio")
{
Console.Error.WriteLine(
$"Skipping MCP server '{name}': command '{def.Command}' is not in the allowlist");
$"Skipping MCP server '{name}': unsupported type '{def.Type}' (only 'stdio' is supported)");
continue;
}

// Only stdio servers are supported; reject unknown types early.
if (def.Type is not null and not "stdio")
if (string.IsNullOrWhiteSpace(def.Command))
{
Console.Error.WriteLine(
$"Skipping MCP server '{name}': unsupported type '{def.Type}' (only 'stdio' is supported)");
$"Skipping MCP server '{name}': missing required command for stdio MCP server");
continue;
}

if (!IsAllowedMcpCommand(def.Command))
{
Console.Error.WriteLine(
$"Skipping MCP server '{name}': command '{def.Command}' is not in the allowlist");
continue;
}

Expand Down Expand Up @@ -945,8 +952,11 @@ internal static void ScrubSensitiveEnvironment(ProcessStartInfo psi)
"dotnet", "dnx", "node", "npx", "python", "python3", "uvx",
};

internal static bool IsAllowedMcpCommand(string command)
internal static bool IsAllowedMcpCommand(string? command)
{
if (string.IsNullOrWhiteSpace(command))
return false;

// Only allow bare command names (resolved via PATH), not paths.
if (command.Contains(Path.DirectorySeparatorChar) ||
command.Contains(Path.AltDirectorySeparatorChar) ||
Expand Down Expand Up @@ -1005,8 +1015,10 @@ internal static bool IsAllowedMcpCommand(string command)
["uvx"] = new(StringComparer.Ordinal) { "--from" },
};

internal static string[]? SanitizeMcpArgs(string command, string[] args)
internal static string[]? SanitizeMcpArgs(string command, string[]? args)
{
args ??= [];

var cmdName = Path.GetFileNameWithoutExtension(command);
if (!DangerousMcpArgs.TryGetValue(cmdName, out var blocked))
return args;
Expand Down
8 changes: 5 additions & 3 deletions eng/skill-validator/src/Shared/Models.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,14 @@ namespace SkillValidator.Shared;
// --- MCP server definition (from plugin.json) ---

public sealed record MCPServerDef(
string Command,
string[] Args,
string? Command = null,
string[]? Args = null,
string? Type = null,
string[]? Tools = null,
Dictionary<string, string>? Env = null,
string? Cwd = null);
string? Cwd = null,
string? Url = null,
Dictionary<string, string>? Headers = null);

// --- Skill info ---

Expand Down
2 changes: 2 additions & 0 deletions eng/skill-validator/src/docs/InvestigatingResults.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ Each scenario includes two required runs (baseline + isolated). It may also incl

> **Note:** Scenarios do not have a `passed` field. To determine pass/fail for an individual scenario, check whether `improvementScore >= 0`. This is the effective score: when no plugin run is present it equals `isolatedImprovementScore`; when a plugin run is present it is the min of isolated and plugin scores. The `passed` field exists only at the verdict level (per-skill).

> **Note:** During evaluation, skill-validator currently loads only stdio MCP servers. Plugin MCP entries with other types (for example `type: http`) are skipped with a stderr message instead of failing the run.

### Breakdown fields

The `isolatedBreakdown` and `pluginBreakdown` objects show how each metric contributed to the improvement score. Each field is a raw delta (not yet weighted). The final score is computed as a weighted sum:
Expand Down
36 changes: 36 additions & 0 deletions eng/skill-validator/tests/Evaluate/EvalDiscoveryTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,42 @@ public async Task FindsPluginMcpServersInGrandparentDirectory()
}
}

[Fact]
public async Task FindsHttpPluginMcpServersWithoutSynthesizingStdioFields()
{
var tmpDir = Path.Combine(Path.GetTempPath(), $"skill-test-{Guid.NewGuid():N}");
var skillDir = Path.Combine(tmpDir, "skills", "my-skill");
Directory.CreateDirectory(skillDir);
try
{
var pluginJson = """
{
"mcpServers": {
"microsoft-docs": {
"type": "http",
"url": "https://learn.microsoft.com/api/mcp",
"headers": {},
"tools": ["*"]
}
}
}
""";
await File.WriteAllTextAsync(Path.Combine(tmpDir, "plugin.json"), pluginJson, TestContext.Current.CancellationToken);

var result = await EvaluateCommand.FindPluginMcpServers(skillDir);
Assert.NotNull(result);
Assert.True(result!.ContainsKey("microsoft-docs"));
Assert.Equal("http", result["microsoft-docs"].Type);
Assert.Equal("https://learn.microsoft.com/api/mcp", result["microsoft-docs"].Url);
Assert.Null(result["microsoft-docs"].Command);
Assert.Null(result["microsoft-docs"].Args);
}
finally
{
Directory.Delete(tmpDir, true);
}
}

[Fact]
public async Task ReturnsNullWhenNoPluginJson()
{
Expand Down
34 changes: 33 additions & 1 deletion eng/skill-validator/tests/Evaluate/RunnerTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,31 @@ public async Task FiltersOutDisallowedMcpServersButKeepsAllowed()
Assert.False(config.McpServers.ContainsKey("bad"));
}

[Fact]
public async Task SkipsUnsupportedHttpMcpServerWithoutCrashing()
{
var mcpServers = new Dictionary<string, MCPServerDef>
{
["microsoft-docs"] = new(Type: "http", Url: "https://learn.microsoft.com/api/mcp", Tools: ["*"]),
["good"] = new(Command: "node", Args: ["server.js"], Tools: ["*"]),
};
var config = await AgentRunner.BuildSessionConfig(MockSkill, null, "gpt-4.1", "C:\\tmp\\work", mcpServers);
Assert.NotNull(config.McpServers);
Assert.True(config.McpServers.ContainsKey("good"));
Assert.False(config.McpServers.ContainsKey("microsoft-docs"));
}

[Fact]
public async Task SkipsStdioMcpServerWithoutCommand()
{
var mcpServers = new Dictionary<string, MCPServerDef>
{
["broken"] = new(Args: ["server.js"], Tools: ["*"]),
};
var config = await AgentRunner.BuildSessionConfig(MockSkill, null, "gpt-4.1", "C:\\tmp\\work", mcpServers);
Assert.Null(config.McpServers);
}

[Fact]
public async Task RejectsMcpServerWithDangerousArgs()
{
Expand Down Expand Up @@ -520,7 +545,14 @@ public void ReturnsNullWhenKeyIsNotString()

public class IsAllowedMcpCommandTests
{
[Fact]
public void RejectsNullCommand()
{
Assert.False(AgentRunner.IsAllowedMcpCommand(null));
}

[Theory]
[InlineData("", false)]
[InlineData("dotnet", true)]
[InlineData("node", true)]
[InlineData("npx", true)]
Expand All @@ -533,7 +565,7 @@ public class IsAllowedMcpCommandTests
[InlineData("wget", false)]
[InlineData("cmd", false)]
[InlineData("powershell", false)]
public void ValidatesCommand(string command, bool expected)
public void ValidatesCommand(string? command, bool expected)
{
Assert.Equal(expected, AgentRunner.IsAllowedMcpCommand(command));
}
Expand Down
24 changes: 23 additions & 1 deletion plugins/dotnet-blazor/plugin.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,27 @@
"name": "dotnet-blazor",
"version": "0.1.0",
"description": "Skills for Blazor development: component authoring, interactivity, and web application patterns.",
"skills": ["./skills/"]
"skills": [
"./skills/"
],
"mcpServers": {
"microsoft-docs": {
"type": "http",
"url": "https://learn.microsoft.com/api/mcp",
"headers": {},
"tools": [
"*"
]
},
Comment thread
javiercn marked this conversation as resolved.
Comment thread
javiercn marked this conversation as resolved.
"playwright": {
Comment thread
AbhitejJohn marked this conversation as resolved.
"type": "stdio",
"command": "npx",
"args": [
"@playwright/mcp@0.0.75"
],
"tools": [
"*"
]
}
}
}
Loading