admin/my-pi
1
0
Fork 0
mirror of https://bitbucket.org/siakitem/my-pi.git synced 2026-08-28 08:35:57 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
29ea39f9b1b3f9c2c0c44a38d916702f6c353aa0
my-pi/pi-chrome/test-suite/task-manifest.json
T

417 lines
11 KiB
JSON
Raw Blame History

[
{
"id": "choredesk-l1-ticket-priority",
"site": "choredesk",
"startUrl": "scenarios/choredesk/index.html?task=choredesk-l1-ticket-priority&run=$RUN_ID&seed=$SEED",
"difficulty": "L1",
"category": "enterprise-crud",
"intent": "Set ticket T-104 priority to High and status to In Progress, then save it.",
"expectedActions": {
"min": 3,
"max": 6
},
"requires": [
"snapshot",
"click",
"select/change",
"form editing"
],
"grader": {
"type": "inPageProgrammatic",
"expression": "window.__taskVerdict || (window.__taskValidation?.done ? \"PASS\" : \"FAIL\")",
"passValue": "PASS"
},
"browserGymLike": {
"observation": [
"dom",
"accessibility_tree",
"screenshot_optional"
],
"actionSpace": [
"click",
"type",
"select",
"evaluate_optional"
],
"reset": "navigate startUrl with fresh $RUN_ID or click Reset run",
"stepReturn": "obs,reward,terminated,truncated,info"
},
"viewport": {
"width": 1280,
"height": 720
},
"slowMoMs": 1000,
"timeoutMs": 5000,
"locale": null,
"timezoneId": null,
"humanBaseline": null,
"actionSubsets": [
"bid",
"nav",
"tab",
"infeas",
"chat"
],
"rewardShape": "binary",
"taskId": "choredesk.L1.ticket-priority",
"seed": 101,
"maxSteps": 8,
"allowedActions": [
"click",
"fill",
"select_option",
"goto",
"report_infeasible",
"send_msg_to_user"
],
"setupUrl": "scenarios/choredesk/index.html?task=choredesk-l1-ticket-priority&run=$RUN_ID&seed=$SEED",
"validateScript": "scenarios/choredesk/index.html#window.__choredesk.validate()",
"cheatScript": "scenarios/choredesk/cheats.js#choredesk-l1-ticket-priority",
"tags": [
"form-fill",
"ticket",
"select"
],
"rubric": [
{
"id": "priorityHigh",
"weight": 0.33,
"description": "T-104 priority is High"
},
{
"id": "statusInProgress",
"weight": 0.33,
"description": "T-104 status is In Progress"
},
{
"id": "saved",
"weight": 0.34,
"description": "T-104 was saved"
}
],
"goal": "Set ticket T-104 priority to High and status to In Progress, then save it.",
"goalObject": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "Set ticket T-104 priority to High and status to In Progress, then save it."
}
]
}
],
"gradeSource": "page"
},
{
"id": "choredesk-l2-refund-message",
"site": "choredesk",
"startUrl": "scenarios/choredesk/index.html?task=choredesk-l2-refund-message&run=$RUN_ID&seed=$SEED",
"difficulty": "L2",
"category": "cross-page-memory",
"intent": "Resolve a customer billing task by looking up customer email and order total on separate pages, then composing a message with exact required fields.",
"expectedActions": {
"min": 8,
"max": 14
},
"requires": [
"multi-page navigation",
"table lookup",
"working memory",
"form composition"
],
"grader": {
"type": "inPageProgrammatic",
"expression": "window.__taskVerdict || (window.__taskValidation?.done ? \"PASS\" : \"FAIL\")",
"passValue": "PASS"
},
"browserGymLike": {
"observation": [
"dom",
"accessibility_tree",
"screenshot_optional"
],
"actionSpace": [
"click",
"type",
"select",
"evaluate_optional"
],
"reset": "navigate startUrl with fresh $RUN_ID or click Reset run",
"stepReturn": "obs,reward,terminated,truncated,info"
},
"viewport": {
"width": 1280,
"height": 720
},
"slowMoMs": 1000,
"timeoutMs": 5000,
"locale": null,
"timezoneId": null,
"humanBaseline": null,
"actionSubsets": [
"bid",
"nav",
"tab",
"infeas",
"chat"
],
"rewardShape": "binary",
"taskId": "choredesk.L2.refund-message",
"seed": 202,
"maxSteps": 16,
"allowedActions": [
"click",
"fill",
"select_option",
"goto",
"report_infeasible",
"send_msg_to_user"
],
"setupUrl": "scenarios/choredesk/index.html?task=choredesk-l2-refund-message&run=$RUN_ID&seed=$SEED",
"validateScript": "scenarios/choredesk/index.html#window.__choredesk.validate()",
"cheatScript": "scenarios/choredesk/cheats.js#choredesk-l2-refund-message",
"tags": [
"lookup",
"cross-page-memory",
"message-compose"
],
"rubric": [
{
"id": "messageSubject",
"weight": 0.34,
"description": "Billing message subject matches"
},
{
"id": "email",
"weight": 0.33,
"description": "Body includes Noah email"
},
{
"id": "total",
"weight": 0.33,
"description": "Body includes exact order total"
}
],
"goal": "Resolve a customer billing task by looking up customer email and order total on separate pages, then composing a message with exact required fields.",
"goalObject": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "Resolve a customer billing task by looking up customer email and order total on separate pages, then composing a message with exact required fields."
}
]
}
],
"gradeSource": "page"
},
{
"id": "choredesk-l3-restock-ticket",
"site": "choredesk",
"startUrl": "scenarios/choredesk/index.html?task=choredesk-l3-restock-ticket&run=$RUN_ID&seed=$SEED",
"difficulty": "L3",
"category": "cross-page-state-update",
"intent": "Use ticket text to identify an item, map item to SKU in Catalog, update Inventory reorder quantity, and add exact internal note back on the ticket.",
"expectedActions": {
"min": 12,
"max": 20
},
"requires": [
"multi-page navigation",
"entity resolution",
"state mutation",
"exact text note",
"audit-save behavior"
],
"grader": {
"type": "inPageProgrammatic",
"expression": "window.__taskVerdict || (window.__taskValidation?.done ? \"PASS\" : \"FAIL\")",
"passValue": "PASS"
},
"browserGymLike": {
"observation": [
"dom",
"accessibility_tree",
"screenshot_optional"
],
"actionSpace": [
"click",
"type",
"select",
"evaluate_optional"
],
"reset": "navigate startUrl with fresh $RUN_ID or click Reset run",
"stepReturn": "obs,reward,terminated,truncated,info"
},
"viewport": {
"width": 1280,
"height": 720
},
"slowMoMs": 1000,
"timeoutMs": 5000,
"locale": null,
"timezoneId": null,
"humanBaseline": null,
"actionSubsets": [
"bid",
"nav",
"tab",
"infeas",
"chat"
],
"rewardShape": "binary",
"taskId": "choredesk.L3.restock-ticket",
"seed": 303,
"maxSteps": 24,
"allowedActions": [
"click",
"fill",
"select_option",
"goto",
"report_infeasible",
"send_msg_to_user"
],
"setupUrl": "scenarios/choredesk/index.html?task=choredesk-l3-restock-ticket&run=$RUN_ID&seed=$SEED",
"validateScript": "scenarios/choredesk/index.html#window.__choredesk.validate()",
"cheatScript": "scenarios/choredesk/cheats.js#choredesk-l3-restock-ticket",
"tags": [
"entity-resolution",
"cross-page-state-update",
"exact-note"
],
"rubric": [
{
"id": "inventory",
"weight": 0.4,
"description": "AST-LAMP reorder quantity is 12 and saved"
},
{
"id": "ticketNote",
"weight": 0.4,
"description": "T-205 note contains exact phrase"
},
{
"id": "audit",
"weight": 0.2,
"description": "Both save audit entries exist"
}
],
"goal": "Use ticket text to identify an item, map item to SKU in Catalog, update Inventory reorder quantity, and add exact internal note back on the ticket.",
"goalObject": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "Use ticket text to identify an item, map item to SKU in Catalog, update Inventory reorder quantity, and add exact internal note back on the ticket."
}
]
}
],
"gradeSource": "page"
},
{
"id": "mini-shop-red-second-cheapest",
"site": "mini-shop",
"startUrl": "fixtures/sites/mini-shop/index.html?task=mini-shop-red-second-cheapest&run=$RUN_ID&seed=$SEED",
"difficulty": "L2",
"category": "task-completion",
"intent": "Add the second-cheapest red product to cart, checkout, and leave the generated order id visible.",
"expectedActions": {
"min": 5,
"max": 9
},
"requires": [
"filter/sort reasoning",
"cart state",
"checkout flow"
],
"grader": {
"type": "inPageProgrammatic",
"expression": "window.__taskVerdict || (window.__taskValidation?.done ? \"PASS\" : \"FAIL\")",
"passValue": "PASS"
},
"browserGymLike": {
"observation": [
"dom",
"accessibility_tree",
"screenshot_optional"
],
"actionSpace": [
"click",
"read_table",
"evaluate_optional"
],
"reset": "navigate startUrl with fresh $RUN_ID",
"stepReturn": "obs,reward,terminated,truncated,info"
},
"viewport": {
"width": 1280,
"height": 720
},
"slowMoMs": 1000,
"timeoutMs": 5000,
"locale": null,
"timezoneId": null,
"humanBaseline": null,
"actionSubsets": [
"bid",
"nav",
"tab",
"infeas",
"chat"
],
"rewardShape": "binary",
"taskId": "mini-shop.L2.red-second-cheapest",
"seed": 404,
"maxSteps": 12,
"allowedActions": [
"click",
"goto",
"report_infeasible",
"send_msg_to_user"
],
"setupUrl": "fixtures/sites/mini-shop/index.html?task=mini-shop-red-second-cheapest&run=$RUN_ID&seed=$SEED",
"validateScript": "fixtures/sites/mini-shop/grader.js#MiniShopGrader.validate",
"cheatScript": "fixtures/sites/mini-shop/cheats.js#mini-shop-red-second-cheapest",
"tags": [
"shopping",
"sort",
"cart",
"checkout"
],
"rubric": [
{
"id": "addedTarget",
"weight": 0.4,
"description": "Cart includes second-cheapest red product"
},
{
"id": "noWrongItems",
"weight": 0.3,
"description": "Cart has no wrong products"
},
{
"id": "checkedOut",
"weight": 0.3,
"description": "Checkout produced order id"
}
],
"goal": "Add the second-cheapest red product to cart, checkout, and leave the generated order id visible.",
"goalObject": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "Add the second-cheapest red product to cart, checkout, and leave the generated order id visible."
}
]
}
],
"gradeSource": "both"
}
]
Reference in New Issue View Git Blame Copy Permalink