mirror of
https://bitbucket.org/siakitem/my-pi.git
synced 2026-08-28 08:35:57 +00:00
74 lines
3.8 KiB
JSON
74 lines
3.8 KiB
JSON
{
|
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
"title": "pi-chrome benchmark manifest",
|
|
"description": "Manifest for browser-control benchmark pages. Recipes are canonical intent, not guaranteed raw tool JSON: runners may need to adapt descriptive selectors (frame/shadow) and expand path placeholders such as ${REPO_ROOT} before invoking tools.",
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"required": ["id", "file", "category", "goal", "expected", "recipe"],
|
|
"additionalProperties": true,
|
|
"properties": {
|
|
"id": { "type": "string" },
|
|
"file": { "type": "string" },
|
|
"category": { "$ref": "#/$defs/category" },
|
|
"difficulty": { "enum": ["L1", "L2", "L3"] },
|
|
"gate": { "description": "Ship-gate bucket. core blocks releases, conditional requires declared capability/environment, quality is adversarial/flaky signal and should not block general ship.", "enum": ["core", "conditional", "quality"] },
|
|
"tags": { "type": "array", "items": { "type": "string" } },
|
|
"goal": { "type": "string" },
|
|
"gradeSource": { "enum": ["page", "harness", "both"], "default": "page" },
|
|
"expected": {
|
|
"type": "object",
|
|
"description": "Expected verdict by execution mode. CONDITIONAL means prerequisites, browser policy, hardware, or missing tool primitives decide outcome.",
|
|
"required": ["synthetic", "trusted", "manual"],
|
|
"properties": {
|
|
"synthetic": { "$ref": "#/$defs/outcome" },
|
|
"trusted": { "$ref": "#/$defs/outcome" },
|
|
"manual": { "$ref": "#/$defs/outcome" }
|
|
}
|
|
},
|
|
"manualBaseline": { "description": "Whether human baseline has been explicitly verified in this repo/environment.", "enum": ["unverified", "verified"] },
|
|
"manualBaselineVerifiedAt": { "type": "string", "format": "date-time" },
|
|
"recipe": {
|
|
"type": "array",
|
|
"description": "Ordered tool intent. Params may include descriptive selectors (e.g. frame/shadow notation) or path placeholders for runners to adapt.",
|
|
"items": {
|
|
"type": "object",
|
|
"required": ["tool", "params"],
|
|
"properties": {
|
|
"tool": { "type": "string" },
|
|
"params": { "type": "object", "description": "Tool params. For upload recipes, paths are strings after runner expansion; ${REPO_ROOT} means repository root." }
|
|
}
|
|
}
|
|
},
|
|
"requires": {
|
|
"type": "object",
|
|
"additionalProperties": true,
|
|
"properties": {
|
|
"cdp": { "type": "boolean" },
|
|
"clipboard": { "type": "boolean" },
|
|
"touch": { "type": "boolean" },
|
|
"secureOrigin": { "type": "boolean" },
|
|
"fileSystem": { "type": "boolean" },
|
|
"dialogs": { "type": "boolean" },
|
|
"downloads": { "type": "boolean" },
|
|
"popups": { "type": "boolean" }
|
|
}
|
|
},
|
|
"prerequisites": { "type": "array", "items": { "type": "string" } },
|
|
"notes": { "type": "array", "items": { "type": "string" } },
|
|
"thresholds": { "type": "object", "additionalProperties": true },
|
|
"timeoutMs": { "type": "number" },
|
|
"verdictDelayMs": { "type": "number" },
|
|
"flakeRisk": { "enum": ["low", "medium", "high"] }
|
|
}
|
|
},
|
|
"$defs": {
|
|
"outcome": { "enum": ["PASS", "FAIL", "SKIP", "WARN", "CONDITIONAL"] },
|
|
"category": {
|
|
"enum": [
|
|
"activation-gates", "agent-safety", "aria-snapshot", "canvas-svg", "clipboard", "contextmenu", "csp", "dialogs", "dom-complexity", "downloads", "drag-drop", "editing", "files", "fingerprint", "focus-keyboard", "forms", "frames", "frameworks", "hover", "keyboard", "lazy-loading", "native-controls", "observability", "pointer-humanization", "popups", "scroll", "scroll-visibility", "spa-routing", "task-completion", "timing", "touch", "trusted-input"
|
|
]
|
|
}
|
|
}
|
|
}
|