chore: merge dev into v2 (#35591)
Co-authored-by: Frank <frank@anoma.ly> Co-authored-by: Aarav Sareen <96787824+arvsrn@users.noreply.github.com> Co-authored-by: Brendan Allan <git@brendonovich.dev> Co-authored-by: opencode-agent[bot] <opencode-agent[bot]@users.noreply.github.com> Co-authored-by: Jack <jack@anoma.ly> Co-authored-by: Brendan Allan <14191578+Brendonovich@users.noreply.github.com> Co-authored-by: Shoubhit Dash <shoubhit2005@gmail.com> Co-authored-by: opencode-agent[bot] <219766164+opencode-agent[bot]@users.noreply.github.com> Co-authored-by: James Long <longster@gmail.com> Co-authored-by: Dustin Deus <deusdustin@gmail.com> Co-authored-by: starptech <starptech@starptechs-MBP.fritz.box> Co-authored-by: Luke Parker <10430890+Hona@users.noreply.github.com> Co-authored-by: 𝓛𝓲𝓽𝓽𝓵𝓮 𝓕𝓻𝓪𝓷𝓴 <little-frank@opencord.local> Co-authored-by: Dax <mail@thdxr.com> Co-authored-by: usrnk1 <7547651+usrnk1@users.noreply.github.com> Co-authored-by: Jay <53023+jayair@users.noreply.github.com> Co-authored-by: runvip <164729189+runvip@users.noreply.github.com> Co-authored-by: opencode <opencode@sst.dev> Co-authored-by: Julian Coy <julian@ex-machina.co> Co-authored-by: Vladimir Glafirov <vglafirov@gitlab.com> Co-authored-by: Adam <2363879+adamdotdevin@users.noreply.github.com> Co-authored-by: Kit Langton <kit.langton@gmail.com> Co-authored-by: Simon Klee <hello@simonklee.dk> Co-authored-by: Jay <air@live.ca> Co-authored-by: David Hill <1879069+iamdavidhill@users.noreply.github.com>
This commit is contained in:
Aiden Cline
@@ -0,0 +1,61 @@
|
||||
# Timeline Layout Continuity
|
||||
|
||||
Run from `packages/app`:
|
||||
|
||||
```sh
|
||||
bun run test:stability
|
||||
```
|
||||
|
||||
The suite runs a production build in one Chromium worker. Selected scenarios use deterministic 4x CPU stress after application readiness. This is a stress profile, not emulation of a specific device.
|
||||
|
||||
## What It Proves
|
||||
|
||||
The continuity probe samples DOM-derived layout and visibility state across browser render opportunities. Tests declare explicit contracts such as:
|
||||
|
||||
- Preserve a visible semantic anchor while the user is away from the bottom.
|
||||
- Preserve end anchoring while active content grows or new content appears.
|
||||
- Keep adjacent visible rows ordered without material overlap.
|
||||
- Keep user-selected disclosure state through updates and virtualization.
|
||||
- Avoid a sampled blank interval while one visible surface replaces another.
|
||||
- Preserve logical row and control identity where local state or focus depends on it.
|
||||
- Keep keyboard, wheel, and nested-scroll ownership consistent during remeasurement.
|
||||
|
||||
The suite exercises real browser reducer, projection, component, virtualizer, layout, focus, and interaction code. The backend and event producer are controlled fixtures.
|
||||
|
||||
## What It Does Not Prove
|
||||
|
||||
The pass/fail oracle does not inspect every compositor-presented pixel. A sample taken after `requestAnimationFrame` is a DOM/layout observation, not proof that every sampled state was displayed or that every displayed frame was sampled.
|
||||
|
||||
The suite does not provide complete coverage for:
|
||||
|
||||
- Compositor-only or raster-only glitches.
|
||||
- Color, contrast, canvas, WebGL, masks, irregular clips, or arbitrary occlusion.
|
||||
- Physical display refresh rates, native OS scaling, or a named low-end device.
|
||||
- TCP packetization, proxy buffering, or the complete real server/provider pipeline.
|
||||
|
||||
Playwright video, trace, screenshots, and observation JSON are diagnostic evidence. They are not pixel baselines and do not participate in normal pass/fail decisions.
|
||||
|
||||
For optional before/violation/after screenshots, set `OPENCODE_STABILITY_CAPTURE=1`. Capture is opt-in because compositor readback can perturb timing.
|
||||
|
||||
## Test Layers
|
||||
|
||||
- **Projection:** admitted rows, grouping, labels, and final visible states.
|
||||
- **Local state:** disclosure state, identity, duplicate delivery, and virtualization restoration.
|
||||
- **Interaction:** wheel, keyboard, nested scrolling, actionability, and focus behavior.
|
||||
- **Layout continuity:** anchoring, adjacency, responsive reflow, and visible surface handoffs.
|
||||
- **Reducer hardening:** validly shaped but intentionally reordered, duplicated, removed, or replaced events.
|
||||
- **Oracle contract:** pure analyzer and browser sampler calibration tests.
|
||||
|
||||
Production-lifecycle fixtures should model states emitted by the current producer. Impossible or reordered sequences belong in reducer-hardening tests and must not be described as normal provider behavior.
|
||||
|
||||
## Diagnostics
|
||||
|
||||
Failures retain:
|
||||
|
||||
- `video.webm`
|
||||
- `trace.zip`
|
||||
- failure screenshot
|
||||
- sampled DOM/layout trace JSON
|
||||
- event markers and summarized violations
|
||||
|
||||
The analyzer records both unclipped layout bounds and ancestor-clipped visible intersections. Scrollbar and raw `scrollTop` changes alone do not fail continuity checks; user-visible semantic anchor movement does.
|
||||
@@ -0,0 +1,250 @@
|
||||
import { expect, test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import {
|
||||
assistantMessage,
|
||||
partUpdated,
|
||||
setupTimeline,
|
||||
shell,
|
||||
textPart,
|
||||
toolPart,
|
||||
userMessage,
|
||||
waitForVisualSettle,
|
||||
type TimelineMessage,
|
||||
} from "./fixture"
|
||||
|
||||
test.describe("timeline adverse visual stability", () => {
|
||||
test("does not pull a scrolled-away user while an active shell grows", async ({ page }, testInfo) => {
|
||||
const activeShellID = "prt_adverse_01_shell"
|
||||
const messages = [
|
||||
...history(24),
|
||||
userMessage(),
|
||||
assistantMessage([shell(activeShellID, "running")], { completed: false }),
|
||||
]
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages,
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
eventRetry: 30,
|
||||
})
|
||||
const scroller = page.locator(".scroll-view__viewport", {
|
||||
has: page.locator('[data-timeline-row="AssistantPart"]'),
|
||||
})
|
||||
await scroller.evaluate((element) => {
|
||||
element.dispatchEvent(new WheelEvent("wheel", { bubbles: true, cancelable: true, deltaY: -450 }))
|
||||
element.scrollTop = Math.max(0, element.scrollHeight - element.clientHeight - 450)
|
||||
})
|
||||
await page.waitForTimeout(150)
|
||||
await expect
|
||||
.poll(() => scroller.evaluate((element) => element.scrollHeight - element.clientHeight - element.scrollTop))
|
||||
.toBeGreaterThan(100)
|
||||
const anchor = await scroller.evaluate((element) => {
|
||||
const view = element.getBoundingClientRect()
|
||||
return [...element.querySelectorAll<HTMLElement>("[data-timeline-key]")].find((row) => {
|
||||
const rect = row.getBoundingClientRect()
|
||||
return rect.top >= view.top + 40 && rect.bottom <= view.bottom - 40
|
||||
})?.dataset.timelineKey
|
||||
})
|
||||
expect(anchor).toBeTruthy()
|
||||
await waitForVisualSettle(page, [`[data-timeline-key="${anchor}"]`])
|
||||
|
||||
const regions = defineVisualRegions({
|
||||
anchor: { selector: `[data-timeline-key="${anchor}"]` },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(shell(activeShellID, "running", lines(1))), 180)
|
||||
await timeline.send(partUpdated(shell(activeShellID, "running", lines(10))), 90)
|
||||
await timeline.send(partUpdated(shell(activeShellID, "running", lines(50))), 350)
|
||||
await timeline.send(partUpdated(shell(activeShellID, "completed", lines(50))), 500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"scrolled-away-shell",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["anchor"] },
|
||||
{ type: "unique", regions: ["anchor"] },
|
||||
{ type: "stable", regions: ["anchor"] },
|
||||
{ type: "fixed", regions: ["anchor"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
]),
|
||||
)
|
||||
})
|
||||
|
||||
test("preserves an explicit shell state across virtualization", async ({ page }) => {
|
||||
const targetID = "prt_virtual_shell"
|
||||
const messages = [
|
||||
userMessage(undefined, { id: "msg_0000_virtual_user", created: 1700000000000 }),
|
||||
assistantMessage([shell(targetID, "completed", lines(20))], {
|
||||
id: "msg_0001_virtual_assistant",
|
||||
parentID: "msg_0000_virtual_user",
|
||||
created: 1700000001000,
|
||||
}),
|
||||
...history(35, 10),
|
||||
]
|
||||
await setupTimeline(page, { messages, settings: { shellToolPartsExpanded: false } })
|
||||
const scroller = page.locator(".scroll-view__viewport", { has: page.locator("[data-timeline-row]") })
|
||||
await scroller.evaluate((element) => {
|
||||
element.dispatchEvent(new WheelEvent("wheel", { bubbles: true, cancelable: true, deltaY: -1_000 }))
|
||||
element.scrollTop = 0
|
||||
})
|
||||
await page.waitForTimeout(300)
|
||||
const trigger = page.locator(`[data-timeline-part-id="${targetID}"] [data-slot="collapsible-trigger"]`)
|
||||
await expect(trigger).toBeVisible()
|
||||
await trigger.click()
|
||||
await expect(trigger).toHaveAttribute("aria-expanded", "true")
|
||||
|
||||
await scroller.evaluate((element) => (element.scrollTop = element.scrollHeight))
|
||||
await expect(page.locator(`[data-timeline-part-id="${targetID}"]`)).toHaveCount(0)
|
||||
await scroller.evaluate((element) => (element.scrollTop = 0))
|
||||
await expect(trigger).toBeVisible()
|
||||
await expect(trigger).toHaveAttribute("aria-expanded", "true")
|
||||
})
|
||||
|
||||
test("keeps narrow viewport rows ordered during long shell growth", async ({ page }, testInfo) => {
|
||||
const shellID = "prt_narrow_01_shell"
|
||||
const followingID = "prt_narrow_02_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage(
|
||||
[shell(shellID, "running"), textPart(followingID, "A narrow following row that wraps across lines.")],
|
||||
{
|
||||
completed: false,
|
||||
},
|
||||
),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
viewport: { width: 430, height: 800 },
|
||||
cpuRate: 4,
|
||||
})
|
||||
await waitForVisualSettle(page, [
|
||||
`[data-timeline-part-id="${shellID}"]`,
|
||||
`[data-timeline-part-id="${followingID}"]`,
|
||||
])
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: {
|
||||
selector: `[data-timeline-part-id="${followingID}"]`,
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(shell(shellID, "running", wideLines(10))), 100)
|
||||
await timeline.send(partUpdated(shell(shellID, "running", wideLines(50))), 300)
|
||||
await timeline.send(partUpdated(shell(shellID, "completed", wideLines(50))), 500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"narrow-shell",
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["shell", "following"] },
|
||||
{ type: "unique", regions: ["shell", "following"] },
|
||||
{ type: "stable", regions: ["shell", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["shell", "following"] },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
test("keeps visible rows ordered while resizing desktop to narrow and back", async ({ page }, testInfo) => {
|
||||
const shellID = "prt_resize_01_shell"
|
||||
const contextIDs = ["prt_resize_02_read", "prt_resize_03_glob"]
|
||||
const followingID = "prt_resize_04_following"
|
||||
await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([
|
||||
shell(shellID, "completed", wideLines(15)),
|
||||
toolPart(contextIDs[0]!, "read", "completed", { filePath: "src/a.ts" }),
|
||||
toolPart(contextIDs[1]!, "glob", "completed", { path: ".", pattern: "**/*.ts" }),
|
||||
textPart(followingID, "Following responsive timeline content that wraps on narrow screens."),
|
||||
]),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
seedHistory: true,
|
||||
})
|
||||
const group = `[data-timeline-part-ids="${contextIDs.join(",")}"]`
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
context: { selector: group, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: {
|
||||
selector: `[data-timeline-part-id="${followingID}"]`,
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await page.setViewportSize({ width: 430, height: 800 })
|
||||
await page.waitForTimeout(500)
|
||||
await page.setViewportSize({ width: 900, height: 800 })
|
||||
await page.waitForTimeout(500)
|
||||
await page.setViewportSize({ width: 1400, height: 900 })
|
||||
await page.waitForTimeout(500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"responsive-resize",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["shell", "context", "following"] },
|
||||
{ type: "unique", regions: ["shell", "context", "following"] },
|
||||
{ type: "stable", regions: ["shell", "context", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 4, maxReversals: 4 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "flow", regions: ["shell", "context", "following"] },
|
||||
]),
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
function history(count: number, offset = 0): TimelineMessage[] {
|
||||
return Array.from({ length: count }, (_, index) => {
|
||||
const value = index + offset
|
||||
const prefix = `msg_0${String(value).padStart(3, "0")}_history`
|
||||
const userID = `${prefix}_a_user`
|
||||
return [
|
||||
userMessage(undefined, { id: userID, created: 1699990000000 + value * 10_000 }),
|
||||
assistantMessage(
|
||||
[
|
||||
textPart(
|
||||
`prt_history_${String(value).padStart(3, "0")}`,
|
||||
`Historical response ${value}. ${"Stable history content. ".repeat(8)}`,
|
||||
),
|
||||
],
|
||||
{
|
||||
id: `${prefix}_b_assistant`,
|
||||
parentID: userID,
|
||||
created: 1699990001000 + value * 10_000,
|
||||
},
|
||||
),
|
||||
]
|
||||
}).flat()
|
||||
}
|
||||
|
||||
function lines(count: number) {
|
||||
return Array.from({ length: count }, (_, index) => `line ${index + 1}`).join("\n")
|
||||
}
|
||||
|
||||
function wideLines(count: number) {
|
||||
return Array.from({ length: count }, (_, index) => `line ${index + 1} ${"wide-output-".repeat(20)}`).join("\n")
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
import { expect, test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import {
|
||||
assistantID,
|
||||
assistantMessage,
|
||||
event,
|
||||
partUpdated,
|
||||
setupTimeline,
|
||||
textPart,
|
||||
toolPart,
|
||||
userMessage,
|
||||
waitForVisualSettle,
|
||||
} from "./fixture"
|
||||
|
||||
const inputs = {
|
||||
read: { filePath: "src/a.ts", offset: 0, limit: 120 },
|
||||
glob: { path: ".", pattern: "**/*.ts" },
|
||||
grep: { path: ".", pattern: "stable", include: "*.ts" },
|
||||
list: { path: "src" },
|
||||
}
|
||||
|
||||
test("appends context operations while the group is expanded", async ({ page }, testInfo) => {
|
||||
const firstID = "prt_append_01_read"
|
||||
const followingID = "prt_append_99_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([toolPart(firstID, "read", "running", inputs.read), textPart(followingID, "Following append")], {
|
||||
completed: false,
|
||||
}),
|
||||
],
|
||||
cpuRate: 4,
|
||||
})
|
||||
const initialGroup = `[data-timeline-part-ids="${firstID}"]`
|
||||
await page.locator(`${initialGroup} [data-slot="collapsible-trigger"]`).click()
|
||||
await waitForVisualSettle(page, [initialGroup, `[data-timeline-part-id="${followingID}"]`])
|
||||
const regions = defineVisualRegions({
|
||||
context: {
|
||||
selector: '[data-timeline-part-ids^="prt_append_01_read"]',
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(toolPart("prt_append_02_glob", "glob", "running", inputs.glob)), 180)
|
||||
await timeline.send(partUpdated(toolPart("prt_append_03_grep", "grep", "completed", inputs.grep)), 240)
|
||||
await timeline.send(partUpdated(toolPart("prt_append_04_list", "list", "completed", inputs.list)), 500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"context-append",
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["context", "following"] },
|
||||
{ type: "unique", regions: ["context", "following"] },
|
||||
{ type: "stable", regions: ["context", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: ["following"], maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["context", "following"] },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
await expect(
|
||||
page.locator(
|
||||
'[data-timeline-part-ids="prt_append_01_read,prt_append_02_glob,prt_append_03_grep,prt_append_04_list"]',
|
||||
),
|
||||
).toBeVisible()
|
||||
await expect(
|
||||
page.locator('[data-timeline-part-ids^="prt_append_01_read"] [data-slot="collapsible-trigger"]'),
|
||||
).toHaveAttribute("aria-expanded", "true")
|
||||
})
|
||||
|
||||
test("splits and merges context groups when a middle text part changes", async ({ page }, testInfo) => {
|
||||
const textID = "prt_split_02_text"
|
||||
const followingID = "prt_split_99_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([
|
||||
toolPart("prt_split_01_read", "read", "completed", inputs.read),
|
||||
textPart(textID, "Boundary"),
|
||||
toolPart("prt_split_03_glob", "glob", "completed", inputs.glob),
|
||||
textPart(followingID, "Following split groups"),
|
||||
]),
|
||||
],
|
||||
cpuRate: 4,
|
||||
})
|
||||
const regions = defineVisualRegions({
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(
|
||||
event("message.part.removed", { sessionID: "ses_timeline_stability", messageID: assistantID, partID: textID }),
|
||||
500,
|
||||
)
|
||||
await expect(page.locator('[data-timeline-part-ids="prt_split_01_read,prt_split_03_glob"]')).toBeVisible()
|
||||
await timeline.send(partUpdated(textPart(textID, "Boundary restored")), 500)
|
||||
await expect(page.locator('[data-timeline-part-ids="prt_split_01_read"]')).toBeVisible()
|
||||
await expect(page.locator('[data-timeline-part-ids="prt_split_03_glob"]')).toBeVisible()
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"context-split-merge",
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["following"] },
|
||||
{ type: "unique", regions: ["following"] },
|
||||
{ type: "stable", regions: ["following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 1 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
test("removing the first context member replaces the group once without overlapping following content", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
const ids = ["prt_key_01_read", "prt_key_02_glob", "prt_key_03_grep"]
|
||||
const followingID = "prt_key_99_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([
|
||||
toolPart(ids[0]!, "read", "completed", inputs.read),
|
||||
toolPart(ids[1]!, "glob", "completed", inputs.glob),
|
||||
toolPart(ids[2]!, "grep", "completed", inputs.grep),
|
||||
textPart(followingID, "Following replaced group"),
|
||||
]),
|
||||
],
|
||||
cpuRate: 4,
|
||||
})
|
||||
const original = page.locator(`[data-timeline-part-ids="${ids.join(",")}"]`)
|
||||
const originalRowKey = await original.evaluate((element) =>
|
||||
element.closest("[data-timeline-key]")?.getAttribute("data-timeline-key"),
|
||||
)
|
||||
await original.locator('[data-slot="collapsible-trigger"]').click()
|
||||
await expect(original.locator('[data-slot="collapsible-trigger"]')).toHaveAttribute("aria-expanded", "true")
|
||||
const regions = defineVisualRegions({
|
||||
context: {
|
||||
selector: '[data-timeline-part-ids*="prt_key_02_glob"]',
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(
|
||||
event("message.part.removed", { sessionID: "ses_timeline_stability", messageID: assistantID, partID: ids[0] }),
|
||||
500,
|
||||
)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"context-first-remove",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["context", "following"] },
|
||||
{ type: "unique", regions: ["context", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "flow", regions: ["context", "following"] },
|
||||
]),
|
||||
)
|
||||
await expect(page.locator(`[data-timeline-part-ids="${ids.slice(1).join(",")}"]`)).toBeVisible()
|
||||
expect(
|
||||
await page
|
||||
.locator(`[data-timeline-part-ids="${ids.slice(1).join(",")}"]`)
|
||||
.evaluate((element) => element.closest("[data-timeline-key]")?.getAttribute("data-timeline-key")),
|
||||
).toBe(originalRowKey)
|
||||
await expect(
|
||||
page.locator(`[data-timeline-part-ids="${ids.slice(1).join(",")}"] [data-slot="collapsible-trigger"]`),
|
||||
).toHaveAttribute("aria-expanded", "true")
|
||||
})
|
||||
@@ -0,0 +1,113 @@
|
||||
import { test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import {
|
||||
assistantMessage,
|
||||
partUpdated,
|
||||
setupTimeline,
|
||||
shell,
|
||||
textPart,
|
||||
userMessage,
|
||||
waitForVisualSettle,
|
||||
} from "./fixture"
|
||||
|
||||
// Fractional scaling exercises different browser rounding than the baseline.
|
||||
for (const deviceScaleFactor of [1, 1.25]) {
|
||||
test(`keeps shell growth ordered at device scale ${deviceScaleFactor}`, async ({ page }, testInfo) => {
|
||||
const shellID = `prt_dpr_${String(deviceScaleFactor).replace(".", "_")}_01_shell`
|
||||
const followingID = `prt_dpr_${String(deviceScaleFactor).replace(".", "_")}_02_following`
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([shell(shellID, "running"), textPart(followingID, "Following scaled shell")], {
|
||||
completed: false,
|
||||
}),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
deviceScaleFactor,
|
||||
seedHistory: true,
|
||||
})
|
||||
await waitForVisualSettle(page, [
|
||||
`[data-timeline-part-id="${shellID}"]`,
|
||||
`[data-timeline-part-id="${followingID}"]`,
|
||||
])
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: {
|
||||
selector: `[data-timeline-part-id="${followingID}"]`,
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(shell(shellID, "running", lines(20))), 180)
|
||||
await timeline.send(partUpdated(shell(shellID, "completed", lines(20))), 500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(testInfo, `dpr-${deviceScaleFactor}`, trace, shellPlan(regions))
|
||||
})
|
||||
}
|
||||
|
||||
for (const reducedMotion of [true]) {
|
||||
test(`keeps shell and status transitions ordered with reduced motion ${reducedMotion}`, async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
const shellID = `prt_motion_${reducedMotion}_01_shell`
|
||||
const followingID = `prt_motion_${reducedMotion}_02_following`
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([shell(shellID, "running"), textPart(followingID, "Following motion profile")], {
|
||||
completed: false,
|
||||
}),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
reducedMotion,
|
||||
cpuRate: 4,
|
||||
seedHistory: true,
|
||||
})
|
||||
await waitForVisualSettle(page, [
|
||||
`[data-timeline-part-id="${shellID}"]`,
|
||||
`[data-timeline-part-id="${followingID}"]`,
|
||||
])
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: {
|
||||
selector: `[data-timeline-part-id="${followingID}"]`,
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(shell(shellID, "completed", lines(10))), 500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(testInfo, `reduced-motion-${reducedMotion}`, trace, shellPlan(regions))
|
||||
})
|
||||
}
|
||||
|
||||
function lines(count: number) {
|
||||
return Array.from({ length: count }, (_, index) => `line ${index + 1}`).join("\n")
|
||||
}
|
||||
|
||||
function shellPlan<Regions extends ReturnType<typeof defineVisualRegions>>(
|
||||
regions: Regions & Record<"shell" | "following", { selector: string }>,
|
||||
) {
|
||||
return visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["shell", "following"] },
|
||||
{ type: "unique", regions: ["shell", "following"] },
|
||||
{ type: "stable", regions: ["shell", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["shell", "following"] },
|
||||
],
|
||||
{ perMarker: true },
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
import { test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import {
|
||||
assistantMessage,
|
||||
partUpdated,
|
||||
setupTimeline,
|
||||
textPart,
|
||||
toolPart,
|
||||
userMessage,
|
||||
waitForVisualSettle,
|
||||
} from "./fixture"
|
||||
|
||||
const profiles = [
|
||||
{ name: "edit", tool: "edit", input: { filePath: "src/edit.ts" } },
|
||||
{
|
||||
name: "multi patch",
|
||||
tool: "apply_patch",
|
||||
input: { files: ["src/a.ts", "src/b.ts", "src/old.ts", "src/moved.ts"] },
|
||||
},
|
||||
] as const
|
||||
|
||||
for (const profile of profiles) {
|
||||
test(`stabilizes ${profile.name} pending to completed`, async ({ page }, testInfo) => {
|
||||
const partID = `prt_file_matrix_${profiles.indexOf(profile)}`
|
||||
const followingID = `prt_file_matrix_following_${profiles.indexOf(profile)}`
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage(
|
||||
[
|
||||
toolPart(partID, profile.tool, "pending", profile.input),
|
||||
textPart(followingID, `Following ${profile.name}`),
|
||||
],
|
||||
{ completed: false },
|
||||
),
|
||||
],
|
||||
settings: { editToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
})
|
||||
await waitForVisualSettle(page, [`[data-timeline-part-id="${partID}"]`, `[data-timeline-part-id="${followingID}"]`])
|
||||
const regions = defineVisualRegions({
|
||||
tool: { selector: `[data-timeline-part-id="${partID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: {
|
||||
selector: `[data-timeline-part-id="${followingID}"]`,
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(toolPart(partID, profile.tool, "running", profile.input)), 180)
|
||||
await timeline.send(partUpdated(completedPart(partID, profile)), 900)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
`file-${profile.name}`,
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["tool", "following"] },
|
||||
{ type: "unique", regions: ["tool", "following"] },
|
||||
{ type: "stable", regions: ["tool", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0, maxReversals: 1 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["tool", "following"] },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
function completedPart(partID: string, profile: (typeof profiles)[number]) {
|
||||
if (profile.tool === "edit") {
|
||||
return toolPart(partID, profile.tool, "completed", profile.input, {
|
||||
metadata: {
|
||||
filediff: {
|
||||
file: "src/edit.ts",
|
||||
additions: 50,
|
||||
deletions: 50,
|
||||
before: source(50, false),
|
||||
after: source(50, true),
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
const files = [
|
||||
patchFile("src/a.ts", "update"),
|
||||
patchFile("src/b.ts", "add"),
|
||||
patchFile("src/old.ts", "delete"),
|
||||
{ ...patchFile("src/moved.ts", "move"), move: "src/new-place.ts" },
|
||||
]
|
||||
return toolPart(partID, profile.tool, "completed", profile.input, { metadata: { files } })
|
||||
}
|
||||
|
||||
function patchFile(filePath: string, type: "add" | "update" | "delete" | "move") {
|
||||
return {
|
||||
filePath,
|
||||
relativePath: filePath,
|
||||
type,
|
||||
additions: type === "delete" ? 0 : 20,
|
||||
deletions: type === "add" ? 0 : 20,
|
||||
before: type === "add" ? undefined : source(20, false),
|
||||
after: type === "delete" ? undefined : source(20, true),
|
||||
}
|
||||
}
|
||||
|
||||
function source(count: number, changed: boolean) {
|
||||
return Array.from(
|
||||
{ length: count },
|
||||
(_, index) => `export const value${index} = ${changed ? index + 1 : index}\n`,
|
||||
).join("")
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
import { expect, test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import {
|
||||
assistantMessage,
|
||||
partUpdated,
|
||||
setupTimeline,
|
||||
textPart,
|
||||
toolPart,
|
||||
userMessage,
|
||||
waitForVisualSettle,
|
||||
} from "./fixture"
|
||||
|
||||
test("adds patch files incrementally without resetting outer expansion", async ({ page }, testInfo) => {
|
||||
const patchID = "prt_incremental_01_patch"
|
||||
const followingID = "prt_incremental_02_following"
|
||||
const first = patchFile("src/a.ts", "update")
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage(
|
||||
[
|
||||
toolPart(patchID, "apply_patch", "running", { files: [first.filePath] }, { metadata: { files: [first] } }),
|
||||
textPart(followingID, "Following incremental patch"),
|
||||
],
|
||||
{ completed: false },
|
||||
),
|
||||
],
|
||||
settings: { editToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
seedHistory: true,
|
||||
})
|
||||
const trigger = page.locator(`[data-timeline-part-id="${patchID}"] [data-slot="collapsible-trigger"]`).first()
|
||||
await expect(trigger).toHaveAttribute("aria-expanded", "true")
|
||||
await waitForVisualSettle(page, [`[data-timeline-part-id="${patchID}"]`, `[data-timeline-part-id="${followingID}"]`])
|
||||
const regions = defineVisualRegions({
|
||||
patch: { selector: `[data-timeline-part-id="${patchID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
const second = patchFile("src/b.ts", "add")
|
||||
const third = patchFile("src/old.ts", "delete")
|
||||
await timeline.send(
|
||||
partUpdated(
|
||||
toolPart(
|
||||
patchID,
|
||||
"apply_patch",
|
||||
"running",
|
||||
{ files: [first.filePath, second.filePath] },
|
||||
{ metadata: { files: [first, second] } },
|
||||
),
|
||||
),
|
||||
240,
|
||||
)
|
||||
await timeline.send(
|
||||
partUpdated(
|
||||
toolPart(
|
||||
patchID,
|
||||
"apply_patch",
|
||||
"completed",
|
||||
{ files: [first.filePath, second.filePath, third.filePath] },
|
||||
{ metadata: { files: [first, second, third] } },
|
||||
),
|
||||
),
|
||||
800,
|
||||
)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"incremental-patch",
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["patch", "following"] },
|
||||
{ type: "unique", regions: ["patch", "following"] },
|
||||
{ type: "stable", regions: ["patch", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: ["following"], maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["patch", "following"] },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
await expect(trigger).toHaveAttribute("aria-expanded", "true")
|
||||
await expect(page.locator('[data-scope="apply-patch"] [data-type="delete"]')).toBeVisible()
|
||||
})
|
||||
|
||||
function patchFile(filePath: string, type: "add" | "update" | "delete") {
|
||||
return {
|
||||
filePath,
|
||||
relativePath: filePath,
|
||||
type,
|
||||
additions: type === "delete" ? 0 : 4,
|
||||
deletions: type === "add" ? 0 : 3,
|
||||
before: type === "add" ? undefined : source(false),
|
||||
after: type === "delete" ? undefined : source(true),
|
||||
}
|
||||
}
|
||||
|
||||
function source(changed: boolean) {
|
||||
return Array.from({ length: 12 }, (_, index) => `export const value${index} = ${changed ? index + 1 : index}\n`).join(
|
||||
"",
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import {
|
||||
assistantMessage,
|
||||
event,
|
||||
toolPart,
|
||||
userMessage,
|
||||
validateTimelineEvent,
|
||||
validateTimelineMessages,
|
||||
type PartSeed,
|
||||
} from "./fixture"
|
||||
|
||||
describe("timeline fixture validation", () => {
|
||||
test("accepts a valid timeline", () => {
|
||||
expect(validateTimelineMessages([userMessage(), assistantMessage()])).toHaveLength(2)
|
||||
})
|
||||
|
||||
test("rejects malformed SDK values at runtime", () => {
|
||||
expect(() =>
|
||||
assistantMessage([], {
|
||||
error: { name: "APIError", data: { message: "failed" } } as never,
|
||||
}),
|
||||
).toThrow()
|
||||
expect(() =>
|
||||
validateTimelineEvent({
|
||||
directory: "C:/OpenCode/TimelineStability",
|
||||
payload: {
|
||||
id: "evt_invalid_status",
|
||||
type: "session.status",
|
||||
properties: { sessionID: "ses_timeline_stability", status: { type: "retry", attempt: 1 } },
|
||||
},
|
||||
}),
|
||||
).toThrow()
|
||||
})
|
||||
|
||||
test("rejects duplicate IDs and orphan assistants", () => {
|
||||
expect(() => validateTimelineMessages([userMessage(), userMessage()])).toThrow(/duplicate message ID/)
|
||||
expect(() =>
|
||||
validateTimelineMessages([userMessage(), assistantMessage([], { parentID: "msg_missing_parent" })]),
|
||||
).toThrow(/parent user/)
|
||||
})
|
||||
|
||||
test("assigns deterministic event IDs", () => {
|
||||
const first = event("session.status", { sessionID: "ses_timeline_stability", status: { type: "busy" } })
|
||||
const second = event("session.status", { sessionID: "ses_timeline_stability", status: { type: "idle" } })
|
||||
expect(first.payload.id).toMatch(/^evt_timeline_\d{4}$/)
|
||||
expect(Number(second.payload.id.slice(-4))).toBe(Number(first.payload.id.slice(-4)) + 1)
|
||||
})
|
||||
})
|
||||
|
||||
if (false) {
|
||||
const userSeed = { id: "prt_type_user", type: "text", text: "typed" } satisfies PartSeed<"user">
|
||||
userMessage([userSeed])
|
||||
|
||||
// @ts-expect-error Tool completion fields are not valid while pending.
|
||||
toolPart("prt_invalid_pending", "bash", "pending", {}, { output: "impossible" })
|
||||
// @ts-expect-error Tool completion fields are not valid while running.
|
||||
toolPart("prt_invalid_running", "bash", "running", {}, { output: "impossible" })
|
||||
// @ts-expect-error Tool error fields are not valid after completion.
|
||||
toolPart("prt_invalid_completed", "bash", "completed", {}, { error: "impossible" })
|
||||
|
||||
assistantMessage([
|
||||
// @ts-expect-error Agent references belong to user messages, not assistant messages.
|
||||
{ id: "prt_invalid_owner", type: "agent", name: "explore", source: { value: "@explore", start: 0, end: 8 } },
|
||||
])
|
||||
|
||||
// @ts-expect-error Retry status events require message and next.
|
||||
event("session.status", { sessionID: "ses_timeline_stability", status: { type: "retry", attempt: 1 } })
|
||||
}
|
||||
@@ -0,0 +1,561 @@
|
||||
import { base64Encode } from "@opencode-ai/core/util/encode"
|
||||
import { Event } from "@opencode-ai/schema/event"
|
||||
import { SessionStatusEvent } from "@opencode-ai/schema/session-status-event"
|
||||
import { SessionV1 } from "@opencode-ai/schema/session-v1"
|
||||
import type {
|
||||
AssistantMessage,
|
||||
GlobalEvent,
|
||||
Message,
|
||||
Part,
|
||||
Session,
|
||||
SessionStatus,
|
||||
ToolPart,
|
||||
ToolState,
|
||||
UserMessage,
|
||||
} from "@opencode-ai/sdk/v2/client"
|
||||
import { expect, type Page } from "@playwright/test"
|
||||
import { Schema } from "effect"
|
||||
import { mockOpenCodeServer } from "../../utils/mock-server"
|
||||
import { installSseTransport } from "../../utils/sse-transport"
|
||||
import { expectSessionTitle } from "../../utils/waits"
|
||||
|
||||
export const directory = "C:/OpenCode/TimelineStability"
|
||||
export const projectID = "proj_timeline_stability"
|
||||
export const sessionID = "ses_timeline_stability"
|
||||
export const userID = "msg_1000_timeline_user"
|
||||
export const assistantID = "msg_1001_timeline_assistant"
|
||||
export const title = "Timeline visual stability"
|
||||
export const model = { providerID: "opencode", modelID: "claude-opus-4-6", variant: "max" }
|
||||
|
||||
type TimelinePayload = Extract<
|
||||
GlobalEvent["payload"],
|
||||
{
|
||||
type:
|
||||
| "message.updated"
|
||||
| "message.removed"
|
||||
| "message.part.updated"
|
||||
| "message.part.removed"
|
||||
| "message.part.delta"
|
||||
| "session.status"
|
||||
}
|
||||
>
|
||||
|
||||
type DeepReadonly<Value> = Value extends readonly unknown[]
|
||||
? { readonly [Key in keyof Value]: DeepReadonly<Value[Key]> }
|
||||
: Value extends object
|
||||
? { readonly [Key in keyof Value]: DeepReadonly<Value[Key]> }
|
||||
: Value
|
||||
|
||||
export type TimelineEvent = DeepReadonly<Omit<GlobalEvent, "payload"> & { payload: TimelinePayload }>
|
||||
export type EventPayload = TimelineEvent
|
||||
export type ToolStatus = ToolState["status"]
|
||||
export type TimelineMessage = { info: UserMessage; parts: Part[] } | { info: AssistantMessage; parts: Part[] }
|
||||
|
||||
type UserPart = Extract<Part, { type: "text" | "file" | "agent" | "subtask" }>
|
||||
type AssistantPart = Exclude<Part, { type: "agent" | "subtask" }>
|
||||
type OwnedPart<Owner extends Message["role"]> = Owner extends "user" ? UserPart : AssistantPart
|
||||
export type PartSeed<Owner extends Message["role"]> =
|
||||
OwnedPart<Owner> extends infer Candidate
|
||||
? Candidate extends Part
|
||||
? Omit<Candidate, "sessionID" | "messageID">
|
||||
: never
|
||||
: never
|
||||
|
||||
type ToolOptions<State extends ToolStatus> = State extends "pending"
|
||||
? { output?: never; title?: never; metadata?: never; error?: never }
|
||||
: State extends "running"
|
||||
? { title?: string; metadata?: Record<string, unknown>; output?: never; error?: never }
|
||||
: State extends "error"
|
||||
? { error?: string; metadata?: Record<string, unknown>; output?: never; title?: never }
|
||||
: { output?: string; title?: string; metadata?: Record<string, unknown>; error?: never }
|
||||
|
||||
const decodeOptions = { errors: "all", onExcessProperty: "error" } as const
|
||||
const decodeMessage = Schema.decodeUnknownSync(SessionV1.WithParts)
|
||||
const decodePart = Schema.decodeUnknownSync(SessionV1.Part)
|
||||
const decodeStatus = Schema.decodeUnknownSync(SessionStatusEvent.Info)
|
||||
const timelineEventSchema = Schema.Union([
|
||||
eventSchema("message.updated", SessionV1.Event.MessageUpdated.data),
|
||||
eventSchema("message.removed", SessionV1.Event.MessageRemoved.data),
|
||||
eventSchema("message.part.updated", SessionV1.Event.PartUpdated.data),
|
||||
eventSchema("message.part.removed", SessionV1.Event.PartRemoved.data),
|
||||
eventSchema("message.part.delta", SessionV1.Event.PartDelta.data),
|
||||
eventSchema("session.status", SessionStatusEvent.Status.data),
|
||||
])
|
||||
const decodeEvent = Schema.decodeUnknownSync(timelineEventSchema)
|
||||
let eventSequence = 0
|
||||
|
||||
export async function setupTimeline(
|
||||
page: Page,
|
||||
input: {
|
||||
messages?: TimelineMessage[]
|
||||
settings?: Record<string, boolean>
|
||||
sessions?: Session[]
|
||||
cpuRate?: number
|
||||
viewport?: { width: number; height: number }
|
||||
eventRetry?: number
|
||||
reducedMotion?: boolean
|
||||
locale?: string
|
||||
deviceScaleFactor?: number
|
||||
seedHistory?: boolean
|
||||
} = {},
|
||||
) {
|
||||
const sessions = input.sessions ?? [session()]
|
||||
const messages = validateTimelineMessages([
|
||||
...(input.seedHistory ? historyMessages(18) : []),
|
||||
...(input.messages ?? [userMessage(), assistantMessage()]),
|
||||
])
|
||||
const active = messages.findLast((message) => message.info.role === "assistant")
|
||||
const initialStatus = decodeStatus(
|
||||
active?.info.role === "assistant" && active.info.time.completed === undefined ? { type: "busy" } : { type: "idle" },
|
||||
decodeOptions,
|
||||
)
|
||||
const transport = await installSseTransport<EventPayload>(page, {
|
||||
server: `http://${process.env.PLAYWRIGHT_SERVER_HOST ?? "127.0.0.1"}:${process.env.PLAYWRIGHT_SERVER_PORT ?? "4096"}`,
|
||||
retry: input.eventRetry ?? 20,
|
||||
})
|
||||
await mockOpenCodeServer(page, {
|
||||
directory,
|
||||
project: project(),
|
||||
provider: provider(),
|
||||
sessions,
|
||||
sessionStatus: { [sessionID]: initialStatus },
|
||||
pageMessages: () => ({
|
||||
items: messages,
|
||||
}),
|
||||
})
|
||||
await page.addInitScript((settings) => {
|
||||
localStorage.setItem(
|
||||
"settings.v3",
|
||||
JSON.stringify({
|
||||
general: {
|
||||
editToolPartsExpanded: false,
|
||||
shellToolPartsExpanded: false,
|
||||
showReasoningSummaries: false,
|
||||
showSessionProgressBar: true,
|
||||
...settings,
|
||||
},
|
||||
}),
|
||||
)
|
||||
}, input.settings ?? {})
|
||||
if (input.locale) {
|
||||
await page.addInitScript((locale) => {
|
||||
localStorage.setItem("opencode.global.dat:language", JSON.stringify({ locale }))
|
||||
}, input.locale)
|
||||
}
|
||||
if (input.reducedMotion) await page.emulateMedia({ reducedMotion: "reduce" })
|
||||
await page.setViewportSize(input.viewport ?? { width: 1400, height: 900 })
|
||||
if (input.deviceScaleFactor) {
|
||||
const devtools = await page.context().newCDPSession(page)
|
||||
const viewport = input.viewport ?? { width: 1400, height: 900 }
|
||||
await devtools.send("Emulation.setDeviceMetricsOverride", {
|
||||
width: viewport.width,
|
||||
height: viewport.height,
|
||||
deviceScaleFactor: input.deviceScaleFactor,
|
||||
mobile: false,
|
||||
})
|
||||
}
|
||||
await page.goto(`/${base64Encode(directory)}/session/${sessionID}`)
|
||||
await transport.waitForConnection()
|
||||
await expectSessionTitle(page, title)
|
||||
if (input.cpuRate && input.cpuRate > 1) {
|
||||
const devtools = await page.context().newCDPSession(page)
|
||||
await devtools.send("Emulation.setCPUThrottlingRate", { rate: input.cpuRate })
|
||||
}
|
||||
|
||||
return {
|
||||
transport,
|
||||
async send(event: TimelineEvent, delay = 0) {
|
||||
const valid = validateTimelineEvent(event)
|
||||
await transport.send(valid, { marker: describeEvent(valid) })
|
||||
if (delay) await page.waitForTimeout(delay)
|
||||
},
|
||||
async sendAll(sequence: { event: TimelineEvent; delay: number }[]) {
|
||||
for (const item of sequence) {
|
||||
const valid = validateTimelineEvent(item.event)
|
||||
await transport.send(valid, { marker: describeEvent(valid) })
|
||||
await page.waitForTimeout(item.delay)
|
||||
}
|
||||
},
|
||||
async settle(frames = 3) {
|
||||
await page.evaluate(
|
||||
(frames) =>
|
||||
new Promise<void>((resolve) => {
|
||||
let remaining = frames
|
||||
const tick = () => {
|
||||
remaining--
|
||||
if (remaining <= 0) return resolve()
|
||||
requestAnimationFrame(tick)
|
||||
}
|
||||
requestAnimationFrame(tick)
|
||||
}),
|
||||
frames,
|
||||
)
|
||||
},
|
||||
async waitForPart(partID: string) {
|
||||
await expect(page.locator(`[data-timeline-part-id="${partID}"]`).first()).toBeVisible()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function describeEvent(event: EventPayload) {
|
||||
if (event.payload.type === "message.part.updated") {
|
||||
const part = event.payload.properties.part
|
||||
return [
|
||||
event.payload.type,
|
||||
part.id,
|
||||
part.type === "tool" ? part.tool : part.type,
|
||||
part.type === "tool" ? part.state.status : undefined,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(":")
|
||||
}
|
||||
if (event.payload.type === "session.status") {
|
||||
const status = event.payload.properties.status
|
||||
return [event.payload.type, status.type, status.type === "retry" ? status.attempt : undefined]
|
||||
.filter((value) => value !== undefined)
|
||||
.join(":")
|
||||
}
|
||||
return event.payload.type
|
||||
}
|
||||
|
||||
export function event<const Type extends TimelinePayload["type"]>(
|
||||
type: Type,
|
||||
properties: Extract<TimelinePayload, { type: Type }>["properties"],
|
||||
): TimelineEvent
|
||||
export function event(type: TimelinePayload["type"], properties: TimelinePayload["properties"]): TimelineEvent {
|
||||
return validateTimelineEvent({
|
||||
directory,
|
||||
payload: { id: `evt_timeline_${String(++eventSequence).padStart(4, "0")}`, type, properties },
|
||||
})
|
||||
}
|
||||
|
||||
export function validateTimelineEvent(input: unknown): TimelineEvent {
|
||||
return decodeEvent(input, decodeOptions)
|
||||
}
|
||||
|
||||
export function validateTimelineMessages(input: readonly TimelineMessage[]): TimelineMessage[] {
|
||||
input.forEach((message) => decodeMessage(message, decodeOptions))
|
||||
const messages = [...input]
|
||||
const messageIDs = new Set<string>()
|
||||
const partIDs = new Set<string>()
|
||||
const users = new Set(messages.filter((message) => message.info.role === "user").map((message) => message.info.id))
|
||||
|
||||
messages.forEach((message) => {
|
||||
if (messageIDs.has(message.info.id))
|
||||
throw new Error(`Timeline fixture has duplicate message ID: ${message.info.id}`)
|
||||
messageIDs.add(message.info.id)
|
||||
if (message.info.role === "assistant" && !users.has(message.info.parentID))
|
||||
throw new Error(`Timeline assistant ${message.info.id} must reference a parent user in the fixture`)
|
||||
message.parts.forEach((part) => {
|
||||
if (part.sessionID !== message.info.sessionID || part.messageID !== message.info.id)
|
||||
throw new Error(`Timeline part ${part.id} ownership does not match message ${message.info.id}`)
|
||||
if (message.info.role === "user" && !["text", "file", "agent", "subtask"].includes(part.type))
|
||||
throw new Error(`Timeline user message ${message.info.id} cannot own ${part.type} part ${part.id}`)
|
||||
if (message.info.role === "assistant" && ["agent", "subtask"].includes(part.type))
|
||||
throw new Error(`Timeline assistant message ${message.info.id} cannot own ${part.type} part ${part.id}`)
|
||||
if (partIDs.has(part.id)) throw new Error(`Timeline fixture has duplicate part ID: ${part.id}`)
|
||||
partIDs.add(part.id)
|
||||
})
|
||||
})
|
||||
return messages
|
||||
}
|
||||
|
||||
export async function waitForVisualSettle(page: Page, selectors: string[], stableFrames = 3) {
|
||||
await page.waitForFunction(
|
||||
({ selectors, stableFrames }) => {
|
||||
const elements = selectors.map((selector) => document.querySelector<HTMLElement>(selector))
|
||||
if (elements.some((element) => !element)) return false
|
||||
return new Promise<boolean>((resolve) => {
|
||||
let stable = 0
|
||||
let previous = ""
|
||||
const sample = () => {
|
||||
const signature = JSON.stringify(
|
||||
elements.map((element) => {
|
||||
const rect = element!.getBoundingClientRect()
|
||||
return [Math.round(rect.top * 10), Math.round(rect.bottom * 10), Math.round(rect.height * 10)]
|
||||
}),
|
||||
)
|
||||
stable = signature === previous ? stable + 1 : 0
|
||||
previous = signature
|
||||
const ordered = elements
|
||||
.slice(1)
|
||||
.every(
|
||||
(element, index) =>
|
||||
elements[index]!.getBoundingClientRect().bottom <= element!.getBoundingClientRect().top + 0.5,
|
||||
)
|
||||
if (stable >= stableFrames && ordered) return resolve(true)
|
||||
requestAnimationFrame(sample)
|
||||
}
|
||||
requestAnimationFrame(sample)
|
||||
})
|
||||
},
|
||||
{ selectors, stableFrames },
|
||||
)
|
||||
}
|
||||
|
||||
export function historyMessages(count: number): TimelineMessage[] {
|
||||
return Array.from({ length: count }, (_, index) => {
|
||||
const value = String(index).padStart(4, "0")
|
||||
const historyUserID = `msg_0${value}_history_a_user`
|
||||
return [
|
||||
userMessage(undefined, { id: historyUserID, created: 1690000000000 + index * 10_000 }),
|
||||
assistantMessage(
|
||||
[
|
||||
{
|
||||
id: `prt_0${value}_history_text`,
|
||||
type: "text",
|
||||
text: `Historical response ${index}. ${"Existing session content keeps the virtual timeline realistic. ".repeat(5)}`,
|
||||
},
|
||||
],
|
||||
{
|
||||
id: `msg_0${value}_history_b_assistant`,
|
||||
parentID: historyUserID,
|
||||
created: 1690000001000 + index * 10_000,
|
||||
},
|
||||
),
|
||||
]
|
||||
}).flat()
|
||||
}
|
||||
|
||||
export function partUpdated(part: Part | PartSeed<"assistant">) {
|
||||
const owned = "messageID" in part ? part : { ...part, sessionID, messageID: assistantID }
|
||||
decodePart(owned, decodeOptions)
|
||||
return event("message.part.updated", {
|
||||
sessionID,
|
||||
part: owned,
|
||||
time: 1700000002000,
|
||||
})
|
||||
}
|
||||
|
||||
export function partDelta(partID: string, delta: string, messageID = assistantID) {
|
||||
return event("message.part.delta", { sessionID, messageID, partID, field: "text", delta })
|
||||
}
|
||||
|
||||
export function messageUpdated(info: Message) {
|
||||
return event("message.updated", { sessionID, info })
|
||||
}
|
||||
|
||||
export function status(type: SessionStatus["type"], attempt = 1) {
|
||||
return event("session.status", {
|
||||
sessionID,
|
||||
status: type === "retry" ? { type, attempt, message: "Rate limited", next: 1700000010000 } : { type },
|
||||
})
|
||||
}
|
||||
|
||||
export function userMessage(
|
||||
parts?: PartSeed<"user">[],
|
||||
input: { id?: string; summary?: UserMessage["summary"]; created?: number } = {},
|
||||
): Extract<TimelineMessage, { info: { role: "user" } }> {
|
||||
const id = input.id ?? userID
|
||||
const seeds = parts ?? [userText("Build the timeline stability matrix.", { id: `prt_${id}_text` })]
|
||||
const message = {
|
||||
info: {
|
||||
id,
|
||||
sessionID,
|
||||
role: "user",
|
||||
time: { created: input.created ?? 1700000000000 },
|
||||
summary: input.summary ?? { diffs: [] },
|
||||
agent: "build",
|
||||
model,
|
||||
},
|
||||
parts: seeds.map((part) => ({
|
||||
...part,
|
||||
sessionID,
|
||||
messageID: id,
|
||||
})),
|
||||
} satisfies Extract<TimelineMessage, { info: { role: "user" } }>
|
||||
decodeMessage(message, decodeOptions)
|
||||
return message
|
||||
}
|
||||
|
||||
export function assistantMessage(
|
||||
parts: PartSeed<"assistant">[] = [],
|
||||
input: {
|
||||
id?: string
|
||||
parentID?: string
|
||||
completed?: boolean
|
||||
error?: AssistantMessage["error"]
|
||||
created?: number
|
||||
} = {},
|
||||
): Extract<TimelineMessage, { info: { role: "assistant" } }> {
|
||||
const id = input.id ?? assistantID
|
||||
const message = {
|
||||
info: {
|
||||
id,
|
||||
sessionID,
|
||||
role: "assistant",
|
||||
time: {
|
||||
created: input.created ?? 1700000001000,
|
||||
...(input.completed === false ? {} : { completed: (input.created ?? 1700000001000) + 1_000 }),
|
||||
},
|
||||
parentID: input.parentID ?? userID,
|
||||
modelID: model.modelID,
|
||||
providerID: model.providerID,
|
||||
mode: "build",
|
||||
agent: "build",
|
||||
path: { cwd: directory, root: directory },
|
||||
cost: 0.01,
|
||||
tokens: { input: 100, output: 200, reasoning: 0, cache: { read: 0, write: 0 } },
|
||||
variant: "max",
|
||||
...(input.error ? { error: input.error } : {}),
|
||||
},
|
||||
parts: parts.map((part) => ({ ...part, sessionID, messageID: id })),
|
||||
} satisfies Extract<TimelineMessage, { info: { role: "assistant" } }>
|
||||
decodeMessage(message, decodeOptions)
|
||||
return message
|
||||
}
|
||||
|
||||
export function userText(
|
||||
text: string,
|
||||
input: Partial<Omit<Extract<PartSeed<"user">, { type: "text" }>, "type" | "text">> = {},
|
||||
): Extract<PartSeed<"user">, { type: "text" }> {
|
||||
return { id: "prt_user_text", type: "text", text, ...input }
|
||||
}
|
||||
|
||||
export function textPart(id: string, text: string): Extract<PartSeed<"assistant">, { type: "text" }> {
|
||||
return { id, type: "text", text }
|
||||
}
|
||||
|
||||
export function reasoningPart(id: string, text: string): Extract<PartSeed<"assistant">, { type: "reasoning" }> {
|
||||
return { id, type: "reasoning", text, time: { start: 1700000001000 } }
|
||||
}
|
||||
|
||||
export function toolPart(
|
||||
id: string,
|
||||
tool: string,
|
||||
state: "pending",
|
||||
input: Record<string, unknown>,
|
||||
options?: ToolOptions<"pending">,
|
||||
): Omit<ToolPart, "sessionID" | "messageID">
|
||||
export function toolPart(
|
||||
id: string,
|
||||
tool: string,
|
||||
state: "running",
|
||||
input: Record<string, unknown>,
|
||||
options?: ToolOptions<"running">,
|
||||
): Omit<ToolPart, "sessionID" | "messageID">
|
||||
export function toolPart(
|
||||
id: string,
|
||||
tool: string,
|
||||
state: "completed",
|
||||
input: Record<string, unknown>,
|
||||
options?: ToolOptions<"completed">,
|
||||
): Omit<ToolPart, "sessionID" | "messageID">
|
||||
export function toolPart(
|
||||
id: string,
|
||||
tool: string,
|
||||
state: "error",
|
||||
input: Record<string, unknown>,
|
||||
options?: ToolOptions<"error">,
|
||||
): Omit<ToolPart, "sessionID" | "messageID">
|
||||
export function toolPart(
|
||||
id: string,
|
||||
tool: string,
|
||||
state: ToolStatus,
|
||||
input: Record<string, unknown>,
|
||||
options: ToolOptions<ToolStatus> = {},
|
||||
): Omit<ToolPart, "sessionID" | "messageID"> {
|
||||
const base = { id, type: "tool" as const, callID: `call_${id}`, tool }
|
||||
if (state === "pending") return { ...base, state: { status: state, input, raw: "" } }
|
||||
if (state === "running")
|
||||
return {
|
||||
...base,
|
||||
state: {
|
||||
status: state,
|
||||
input,
|
||||
title: options.title,
|
||||
metadata: options.metadata ?? {},
|
||||
time: { start: 1700000001000 },
|
||||
},
|
||||
}
|
||||
if (state === "error")
|
||||
return {
|
||||
...base,
|
||||
state: {
|
||||
status: state,
|
||||
input,
|
||||
error: options.error ?? "Tool failed",
|
||||
metadata: options.metadata ?? {},
|
||||
time: { start: 1700000001000, end: 1700000002000 },
|
||||
},
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
state: {
|
||||
status: state,
|
||||
input,
|
||||
output: options.output ?? "Completed",
|
||||
title: options.title ?? tool,
|
||||
metadata: options.metadata ?? {},
|
||||
time: { start: 1700000001000, end: 1700000002000 },
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export function shell(
|
||||
id: string,
|
||||
state: ToolStatus,
|
||||
output = "",
|
||||
command = `echo ${id}`,
|
||||
): Omit<ToolPart, "sessionID" | "messageID"> {
|
||||
if (state === "pending") return toolPart(id, "bash", state, { command })
|
||||
if (state === "running")
|
||||
return toolPart(id, "bash", state, { command }, { title: command, metadata: { command, output } })
|
||||
if (state === "error")
|
||||
return toolPart(id, "bash", state, { command }, { error: output || undefined, metadata: { command, output } })
|
||||
return toolPart(id, "bash", state, { command }, { title: command, output, metadata: { command, output } })
|
||||
}
|
||||
|
||||
export function completedAssistantInfo(info: AssistantMessage): AssistantMessage {
|
||||
return { ...info, time: { ...info.time, completed: 1700000003000 } }
|
||||
}
|
||||
|
||||
export function project() {
|
||||
return {
|
||||
id: projectID,
|
||||
worktree: directory,
|
||||
vcs: "git",
|
||||
name: "timeline-stability",
|
||||
time: { created: 1700000000000, updated: 1700000000000 },
|
||||
sandboxes: [],
|
||||
}
|
||||
}
|
||||
|
||||
export function session(input: Partial<Session> = {}): Session {
|
||||
return {
|
||||
id: sessionID,
|
||||
slug: "timeline-stability",
|
||||
projectID,
|
||||
directory,
|
||||
title,
|
||||
version: "dev",
|
||||
time: { created: 1700000000000, updated: 1700000000000 },
|
||||
...input,
|
||||
}
|
||||
}
|
||||
|
||||
function eventSchema<
|
||||
const Type extends TimelinePayload["type"],
|
||||
const Properties extends Schema.Codec<unknown, unknown>,
|
||||
>(type: Type, properties: Properties) {
|
||||
return Schema.Struct({
|
||||
directory: Schema.String,
|
||||
project: Schema.optional(Schema.String),
|
||||
workspace: Schema.optional(Schema.String),
|
||||
payload: Schema.Struct({ id: Event.ID, type: Schema.Literal(type), properties }),
|
||||
})
|
||||
}
|
||||
|
||||
function provider() {
|
||||
return {
|
||||
all: [
|
||||
{
|
||||
id: "opencode",
|
||||
name: "OpenCode",
|
||||
models: { "claude-opus-4-6": { id: "claude-opus-4-6", name: "Claude Opus 4.6", limit: { context: 200_000 } } },
|
||||
},
|
||||
],
|
||||
connected: ["opencode"],
|
||||
default: { providerID: "opencode", modelID: "claude-opus-4-6" },
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
import { expect, test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import { assistantMessage, setupTimeline, shell, textPart, toolPart, userMessage, waitForVisualSettle } from "./fixture"
|
||||
|
||||
test("expands and collapses a long completed shell without overlap", async ({ page }, testInfo) => {
|
||||
const shellID = "prt_interaction_01_shell"
|
||||
const followingID = "prt_interaction_02_following"
|
||||
await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([shell(shellID, "completed", lines(50)), textPart(followingID, "Following shell expansion")]),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: false },
|
||||
cpuRate: 4,
|
||||
seedHistory: true,
|
||||
})
|
||||
const trigger = page.locator(`[data-timeline-part-id="${shellID}"] [data-slot="collapsible-trigger"]`)
|
||||
await waitForVisualSettle(page, [`[data-timeline-part-id="${shellID}"]`, `[data-timeline-part-id="${followingID}"]`])
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
const plan = visualPlan(regions, [
|
||||
{ type: "required", regions: ["shell", "following"] },
|
||||
{ type: "unique", regions: ["shell", "following"] },
|
||||
{ type: "stable", regions: ["shell", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["shell", "following"] },
|
||||
])
|
||||
await startVisualProbe(page, regions)
|
||||
await trigger.click()
|
||||
await expect(trigger).toHaveAttribute("aria-expanded", "true")
|
||||
await page.waitForTimeout(500)
|
||||
const expanded = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(testInfo, "shell-expand", expanded, plan)
|
||||
|
||||
await startVisualProbe(page, regions)
|
||||
await trigger.click()
|
||||
await expect(trigger).toHaveAttribute("aria-expanded", "false")
|
||||
await page.waitForTimeout(500)
|
||||
const collapsed = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(testInfo, "shell-collapse", collapsed, plan)
|
||||
})
|
||||
|
||||
test("expands and collapses a completed context group without overlap", async ({ page }, testInfo) => {
|
||||
const ids = [
|
||||
"prt_interaction_01_read",
|
||||
"prt_interaction_02_glob",
|
||||
"prt_interaction_03_grep",
|
||||
"prt_interaction_04_list",
|
||||
]
|
||||
const group = `[data-timeline-part-ids="${ids.join(",")}"]`
|
||||
const followingID = "prt_interaction_context_following"
|
||||
await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([
|
||||
toolPart(ids[0]!, "read", "completed", { filePath: "src/a.ts" }),
|
||||
toolPart(ids[1]!, "glob", "completed", { path: ".", pattern: "**/*.ts" }),
|
||||
toolPart(ids[2]!, "grep", "completed", { path: ".", pattern: "stable" }),
|
||||
toolPart(ids[3]!, "list", "completed", { path: "src" }),
|
||||
textPart(followingID, "Following context expansion"),
|
||||
]),
|
||||
],
|
||||
cpuRate: 4,
|
||||
seedHistory: true,
|
||||
})
|
||||
const trigger = page.locator(`${group} [data-slot="collapsible-trigger"]`)
|
||||
await waitForVisualSettle(page, [group, `[data-timeline-part-id="${followingID}"]`])
|
||||
for (const [name, expanded] of [
|
||||
["context-expand", true],
|
||||
["context-collapse", false],
|
||||
["context-reexpand", true],
|
||||
] as const) {
|
||||
const regions = defineVisualRegions({
|
||||
context: { selector: group, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: {
|
||||
selector: `[data-timeline-part-id="${followingID}"]`,
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await trigger.click()
|
||||
await expect(trigger).toHaveAttribute("aria-expanded", String(expanded))
|
||||
await page.waitForTimeout(500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
name,
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["context", "following"] },
|
||||
{ type: "unique", regions: ["context", "following"] },
|
||||
{ type: "stable", regions: ["context", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["context", "following"] },
|
||||
]),
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
test("expands and collapses an edit diff without moving twice", async ({ page }, testInfo) => {
|
||||
const editID = "prt_interaction_edit"
|
||||
const followingID = "prt_interaction_edit_following"
|
||||
await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([
|
||||
toolPart(
|
||||
editID,
|
||||
"edit",
|
||||
"completed",
|
||||
{ filePath: "src/edit.ts" },
|
||||
{
|
||||
metadata: {
|
||||
filediff: {
|
||||
file: "src/edit.ts",
|
||||
additions: 40,
|
||||
deletions: 40,
|
||||
before: source(40, false),
|
||||
after: source(40, true),
|
||||
},
|
||||
},
|
||||
},
|
||||
),
|
||||
textPart(followingID, "Following edit expansion"),
|
||||
]),
|
||||
],
|
||||
settings: { editToolPartsExpanded: false },
|
||||
cpuRate: 4,
|
||||
seedHistory: true,
|
||||
})
|
||||
const trigger = page.locator(`[data-timeline-part-id="${editID}"] [data-slot="collapsible-trigger"]`).first()
|
||||
await waitForVisualSettle(page, [`[data-timeline-part-id="${editID}"]`, `[data-timeline-part-id="${followingID}"]`])
|
||||
const regions = defineVisualRegions({
|
||||
edit: { selector: `[data-timeline-part-id="${editID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await trigger.click()
|
||||
await expect(trigger).toHaveAttribute("aria-expanded", "true")
|
||||
await page.waitForTimeout(900)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"edit-expand",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["edit", "following"] },
|
||||
{ type: "unique", regions: ["edit", "following"] },
|
||||
{ type: "stable", regions: ["edit", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0, maxReversals: 1 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["edit", "following"] },
|
||||
]),
|
||||
)
|
||||
})
|
||||
|
||||
test("shows all and expands historical diff summary without overlap", async ({ page }, testInfo) => {
|
||||
const firstUser = userMessage(undefined, {
|
||||
summary: {
|
||||
diffs: Array.from({ length: 12 }, (_, index) => ({
|
||||
file: `src/diff-${index}.ts`,
|
||||
additions: 1,
|
||||
deletions: 1,
|
||||
patch: `@@ -1 +1 @@\n-export const value = ${index}\n+export const value = ${index + 1}`,
|
||||
})),
|
||||
},
|
||||
})
|
||||
const nextUserID = "msg_2000_diff_interaction_user"
|
||||
await setupTimeline(page, {
|
||||
messages: [
|
||||
firstUser,
|
||||
assistantMessage(),
|
||||
userMessage(undefined, { id: nextUserID, created: 1700000010000 }),
|
||||
assistantMessage([], {
|
||||
id: "msg_2001_diff_interaction_assistant",
|
||||
parentID: nextUserID,
|
||||
created: 1700000011000,
|
||||
}),
|
||||
],
|
||||
cpuRate: 4,
|
||||
})
|
||||
const scroller = page.locator(".scroll-view__viewport", { has: page.locator("[data-timeline-row]") })
|
||||
await scroller.evaluate((element) => (element.scrollTop = 0))
|
||||
const diff = page.locator('[data-timeline-row="DiffSummary"]')
|
||||
const following = page.locator(`[data-message-id="${nextUserID}"]`).first()
|
||||
await expect(diff).toBeVisible()
|
||||
const regions = defineVisualRegions({
|
||||
diff: { selector: '[data-timeline-row="DiffSummary"]' },
|
||||
following: { selector: `[data-message-id="${nextUserID}"]` },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await page.getByText(/show all/i).click()
|
||||
await page.waitForTimeout(500)
|
||||
await diff.locator('[data-slot="session-turn-diff-trigger"]').first().click()
|
||||
await page.waitForTimeout(900)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"diff-summary-expand",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["diff", "following"] },
|
||||
{ type: "unique", regions: ["diff", "following"] },
|
||||
{ type: "stable", regions: ["diff", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 1, maxReversals: 2 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "flow", regions: ["diff", "following"] },
|
||||
]),
|
||||
)
|
||||
})
|
||||
|
||||
function lines(count: number) {
|
||||
return Array.from({ length: count }, (_, index) => `line ${index + 1}`).join("\n")
|
||||
}
|
||||
|
||||
function source(count: number, changed: boolean) {
|
||||
return Array.from(
|
||||
{ length: count },
|
||||
(_, index) => `export const value${index} = ${changed ? index + 1 : index}\n`,
|
||||
).join("")
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
import { expect, test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
mapVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import {
|
||||
assistantMessage,
|
||||
completedAssistantInfo,
|
||||
messageUpdated,
|
||||
partDelta,
|
||||
partUpdated,
|
||||
reasoningPart,
|
||||
setupTimeline,
|
||||
shell,
|
||||
status,
|
||||
textPart,
|
||||
userMessage,
|
||||
waitForVisualSettle,
|
||||
} from "./fixture"
|
||||
|
||||
test.describe("timeline visual lifecycle stability", () => {
|
||||
test("streams empty, short, and long parallel shells to staggered completion", async ({ page }, testInfo) => {
|
||||
test.setTimeout(180_000)
|
||||
const ids = ["prt_parallel_01_empty", "prt_parallel_02_short", "prt_parallel_03_long"] as const
|
||||
const initial = ids.map((id) => shell(id, "running"))
|
||||
const followingID = "prt_parallel_04_following"
|
||||
const assistant = assistantMessage([...initial, textPart(followingID, "Following all parallel shells.")], {
|
||||
completed: false,
|
||||
})
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [userMessage(), assistant],
|
||||
settings: { shellToolPartsExpanded: true, showReasoningSummaries: true },
|
||||
cpuRate: 4,
|
||||
eventRetry: 24,
|
||||
seedHistory: true,
|
||||
})
|
||||
await timeline.send(status("busy"), 150)
|
||||
for (const id of ids) await timeline.waitForPart(id)
|
||||
const scroller = page.locator(".scroll-view__viewport", {
|
||||
has: page.locator('[data-timeline-row="AssistantPart"]'),
|
||||
})
|
||||
await scroller.evaluate((element) => (element.scrollTop = element.scrollHeight))
|
||||
const regions = defineVisualRegions({
|
||||
prt_shell_empty: shellRegion(ids[0]),
|
||||
prt_shell_short: shellRegion(ids[1]),
|
||||
prt_shell_long: shellRegion(ids[2]),
|
||||
following: shellRegion(followingID),
|
||||
})
|
||||
await waitForVisualSettle(page, [`[data-timeline-part-id="${followingID}"]`])
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.sendAll([
|
||||
{ event: partUpdated(shell(ids[0]!, "completed", "")), delay: 180 },
|
||||
{ event: partUpdated(shell(ids[2]!, "running", lines(10))), delay: 70 },
|
||||
{ event: partUpdated(shell(ids[1]!, "running", lines(2))), delay: 110 },
|
||||
{ event: partUpdated(shell(ids[2]!, "running", lines(25))), delay: 80 },
|
||||
{ event: partUpdated(shell(ids[1]!, "completed", lines(2))), delay: 260 },
|
||||
{ event: partUpdated(shell(ids[2]!, "running", lines(50))), delay: 100 },
|
||||
{ event: partUpdated(shell(ids[2]!, "completed", lines(50))), delay: 450 },
|
||||
{ event: messageUpdated(completedAssistantInfo(assistant.info)), delay: 100 },
|
||||
{ event: status("idle"), delay: 700 },
|
||||
])
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"parallel-shells",
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["prt_shell_empty", "prt_shell_short", "prt_shell_long", "following"] },
|
||||
{ type: "unique", regions: ["prt_shell_empty", "prt_shell_short", "prt_shell_long"] },
|
||||
{ type: "stable", regions: ["prt_shell_empty", "prt_shell_short", "prt_shell_long", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0, maxReversals: 4 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["prt_shell_empty", "prt_shell_short", "prt_shell_long", "following"] },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
await expect(page.locator(`[data-timeline-part-id="${ids[2]}"] [data-slot="bash-pre"]`)).toContainText("line 50")
|
||||
|
||||
const short = page.locator(`[data-timeline-part-id="${ids[1]}"]`)
|
||||
await short.locator('[data-slot="collapsible-trigger"]').click()
|
||||
await expect(short.locator('[data-slot="collapsible-trigger"]')).toHaveAttribute("aria-expanded", "false")
|
||||
await timeline.send(partUpdated(textPart("prt_late_sibling", "A later sibling rerender.")), 250)
|
||||
await expect(short.locator('[data-slot="collapsible-trigger"]')).toHaveAttribute("aria-expanded", "false")
|
||||
})
|
||||
|
||||
test("replaces thinking with streamed reasoning and text without a blank visible turn", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
const reasoningID = "prt_reasoning_visible"
|
||||
const textID = "prt_streamed_text"
|
||||
const assistant = assistantMessage([], { completed: false })
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [userMessage(), assistant],
|
||||
settings: { showReasoningSummaries: true },
|
||||
cpuRate: 4,
|
||||
})
|
||||
await timeline.send(status("busy"), 120)
|
||||
await expect(page.locator('[data-timeline-row="Thinking"]')).toBeVisible()
|
||||
|
||||
const regions = defineVisualRegions({
|
||||
thinking: { selector: '[data-timeline-row="Thinking"]' },
|
||||
reasoning: {
|
||||
selector: `[data-timeline-part-id="${reasoningID}"]`,
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
text: { selector: `[data-timeline-part-id="${textID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(reasoningPart(reasoningID, "")), 100)
|
||||
await expect(page.locator(`[data-timeline-part-id="${reasoningID}"]`)).toHaveCount(0)
|
||||
await timeline.send(partUpdated(reasoningPart(reasoningID, "## Planning\n\nChecking the visible timeline.")), 160)
|
||||
await timeline.waitForPart(reasoningID)
|
||||
await expect(page.locator('[data-timeline-row="Thinking"]')).toHaveCount(0)
|
||||
await timeline.send(partUpdated(textPart(textID, "Starting")), 100)
|
||||
await timeline.send(partDelta(textID, " **stable"), 90)
|
||||
await timeline.send(partDelta(textID, " output** with `code` and [a link"), 130)
|
||||
await timeline.send(partDelta(textID, "](https://example.com)."), 220)
|
||||
await timeline.send(messageUpdated(completedAssistantInfo(assistant.info)), 120)
|
||||
await timeline.send(status("idle"), 500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"reasoning-text-handoff",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["reasoning", "text"] },
|
||||
{ type: "continuous-any", regions: ["thinking", "reasoning", "text"] },
|
||||
{ type: "unique", regions: ["reasoning", "text"] },
|
||||
{ type: "stable", regions: ["reasoning", "text"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxReversals: 4 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "flow", regions: ["reasoning", "text"] },
|
||||
]),
|
||||
)
|
||||
await expect(page.locator(`[data-timeline-part-id="${textID}"]`)).toContainText("stable output")
|
||||
})
|
||||
})
|
||||
|
||||
function lines(count: number) {
|
||||
return Array.from({ length: count }, (_, index) => `line ${index + 1}`).join("\n")
|
||||
}
|
||||
|
||||
function shellRegion(id: string) {
|
||||
return { selector: `[data-timeline-part-id="${id}"]`, closest: '[data-timeline-row="AssistantPart"]' }
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import { expect, test } from "@playwright/test"
|
||||
import {
|
||||
analyzeVisualObservations,
|
||||
defineVisualRegions,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import { assistantMessage, setupTimeline, textPart, userMessage } from "./fixture"
|
||||
|
||||
test("detects blanking caused by ancestor opacity", async ({ page }) => {
|
||||
const partID = "prt_oracle_ancestor_opacity"
|
||||
await setupTimeline(page, { messages: [userMessage(), assistantMessage([textPart(partID, "Visible content")])] })
|
||||
const row = page.locator(`[data-timeline-part-id="${partID}"]`).first()
|
||||
const regions = defineVisualRegions({
|
||||
content: { selector: `[data-timeline-part-id="${partID}"]` },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await row.evaluate((element) => {
|
||||
element.parentElement!.style.opacity = "0"
|
||||
})
|
||||
await page.waitForTimeout(50)
|
||||
await row.evaluate((element) => {
|
||||
element.parentElement!.style.opacity = "1"
|
||||
})
|
||||
await page.waitForTimeout(50)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
const issues = analyzeVisualObservations(
|
||||
trace.samples,
|
||||
visualPlan(regions, [
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all" },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
]),
|
||||
)
|
||||
|
||||
expect(issues.some((issue) => issue.includes("blanked between visible frames"))).toBe(true)
|
||||
})
|
||||
|
||||
test("detects root opacity when probing descendant opacity", async ({ page }) => {
|
||||
const partID = "prt_oracle_descendant_opacity"
|
||||
await setupTimeline(page, { messages: [userMessage(), assistantMessage([textPart(partID, "Visible content")])] })
|
||||
const row = page.locator(`[data-timeline-part-id="${partID}"]`).first()
|
||||
await row.evaluate((element) => {
|
||||
element.innerHTML = '<span data-probe-opacity="true">Visible content</span>'
|
||||
})
|
||||
const regions = defineVisualRegions({
|
||||
content: {
|
||||
selector: `[data-timeline-part-id="${partID}"]`,
|
||||
opacitySelectors: ['[data-probe-opacity="true"]'],
|
||||
},
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await row.evaluate((element) => {
|
||||
;(element as HTMLElement).style.opacity = "0"
|
||||
})
|
||||
await page.waitForTimeout(50)
|
||||
await row.evaluate((element) => {
|
||||
;(element as HTMLElement).style.opacity = "1"
|
||||
})
|
||||
await page.waitForTimeout(50)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
const issues = analyzeVisualObservations(
|
||||
trace.samples,
|
||||
visualPlan(regions, [
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all" },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
]),
|
||||
)
|
||||
|
||||
expect(issues.some((issue) => issue.includes("blanked between visible frames"))).toBe(true)
|
||||
})
|
||||
@@ -0,0 +1,17 @@
|
||||
import config from "../playwright.config"
|
||||
|
||||
export default {
|
||||
...config,
|
||||
testDir: ".",
|
||||
testMatch: "**/*.spec.ts",
|
||||
outputDir: "../../test-results/timeline-stability",
|
||||
reporter: [["html", { outputFolder: "../../playwright-report/timeline-stability", open: "never" }], ["line"]],
|
||||
retries: 0,
|
||||
workers: 1,
|
||||
use: {
|
||||
...config.use,
|
||||
trace: "retain-on-failure",
|
||||
screenshot: "only-on-failure",
|
||||
video: "retain-on-failure",
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
import { expect, test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import {
|
||||
assistantMessage,
|
||||
partUpdated,
|
||||
setupTimeline,
|
||||
shell,
|
||||
textPart,
|
||||
userMessage,
|
||||
type TimelineMessage,
|
||||
} from "./fixture"
|
||||
|
||||
test("does not reverse visible rows when the user wheels during shell remeasurement", async ({ page }, testInfo) => {
|
||||
const shellID = "prt_wheel_01_shell"
|
||||
const followingID = "prt_wheel_02_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
...history(12),
|
||||
userMessage(),
|
||||
assistantMessage([shell(shellID, "running"), textPart(followingID, "Following wheel interaction")], {
|
||||
completed: false,
|
||||
}),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
reducedMotion: true,
|
||||
seedHistory: true,
|
||||
})
|
||||
const scroller = page.locator(".scroll-view__viewport", { has: page.locator("[data-timeline-row]") })
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(shell(shellID, "running", lines(30))), 80)
|
||||
await scroller.evaluate((element) =>
|
||||
element.dispatchEvent(new WheelEvent("wheel", { bubbles: true, cancelable: true, deltaY: -180 })),
|
||||
)
|
||||
await scroller.evaluate((element) => (element.scrollTop -= 180))
|
||||
await timeline.send(partUpdated(shell(shellID, "running", lines(50))), 250)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(testInfo, "wheel-during-resize", trace, rowPairPlan(regions, 1))
|
||||
})
|
||||
|
||||
test("keeps moving upward while drag-selecting above the timeline", async ({ page }) => {
|
||||
await setupTimeline(page, {
|
||||
messages: history(80),
|
||||
viewport: { width: 1400, height: 700 },
|
||||
reducedMotion: true,
|
||||
})
|
||||
const scroller = page.locator(".scroll-view__viewport", { has: page.locator("[data-timeline-row]") })
|
||||
const text = page.getByText("History 79.", { exact: false })
|
||||
await expect(text).toBeVisible()
|
||||
await scroller.evaluate((element) => {
|
||||
element.dataset.selectionLength = "0"
|
||||
document.addEventListener("selectionchange", () => {
|
||||
element.dataset.selectionLength = String(
|
||||
Math.max(Number(element.dataset.selectionLength), window.getSelection()?.toString().length ?? 0),
|
||||
)
|
||||
})
|
||||
})
|
||||
const textBox = await text.boundingBox()
|
||||
const scrollBox = await scroller.boundingBox()
|
||||
expect(textBox).not.toBeNull()
|
||||
expect(scrollBox).not.toBeNull()
|
||||
if (!textBox || !scrollBox) return
|
||||
|
||||
await page.mouse.move(textBox.x + textBox.width - 10, textBox.y + textBox.height / 2)
|
||||
await page.mouse.down()
|
||||
await page.mouse.move(textBox.x + 20, scrollBox.y - 120, { steps: 30 })
|
||||
|
||||
await expect.poll(() => scroller.evaluate((element) => Number(element.dataset.selectionLength))).toBeGreaterThan(0)
|
||||
await expect
|
||||
.poll(() => scroller.evaluate((element) => element.scrollHeight - element.clientHeight - element.scrollTop))
|
||||
.toBeGreaterThan(500)
|
||||
await page.mouse.up()
|
||||
})
|
||||
|
||||
test("does not pull a keyboard-scrolled user during shell remeasurement", async ({ page }, testInfo) => {
|
||||
const shellID = "prt_keyboard_01_shell"
|
||||
const followingID = "prt_keyboard_02_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
...history(12),
|
||||
userMessage(),
|
||||
assistantMessage([shell(shellID, "running"), textPart(followingID, "Following keyboard interaction")], {
|
||||
completed: false,
|
||||
}),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
})
|
||||
const scroller = page.locator(".scroll-view__viewport", { has: page.locator("[data-timeline-row]") })
|
||||
await scroller.focus()
|
||||
for (let index = 0; index < 3; index++) {
|
||||
await scroller.press("PageUp")
|
||||
await page.waitForTimeout(250)
|
||||
}
|
||||
await expect
|
||||
.poll(() => scroller.evaluate((element) => element.scrollHeight - element.clientHeight - element.scrollTop), {
|
||||
timeout: 20_000,
|
||||
})
|
||||
.toBeGreaterThan(80)
|
||||
await page.waitForFunction(() => {
|
||||
const root = [...document.querySelectorAll<HTMLElement>(".scroll-view__viewport")].find((element) =>
|
||||
element.querySelector("[data-timeline-row]"),
|
||||
)
|
||||
if (!root) return false
|
||||
return new Promise<boolean>((resolve) => {
|
||||
const top = root.scrollTop
|
||||
requestAnimationFrame(() => requestAnimationFrame(() => resolve(Math.abs(root.scrollTop - top) <= 0.5)))
|
||||
})
|
||||
})
|
||||
const anchor = await scroller.evaluate((element) => {
|
||||
const view = element.getBoundingClientRect()
|
||||
return [...element.querySelectorAll<HTMLElement>("[data-timeline-key]")].find((row) => {
|
||||
const rect = row.getBoundingClientRect()
|
||||
return rect.top >= view.top + 40 && rect.bottom <= view.bottom - 40
|
||||
})?.dataset.timelineKey
|
||||
})
|
||||
expect(anchor).toBeTruthy()
|
||||
const regions = defineVisualRegions({
|
||||
anchor: { selector: `[data-timeline-key="${anchor}"]` },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(shell(shellID, "running", lines(50))), 400)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(testInfo, "keyboard-during-resize", trace, anchorPlan(regions))
|
||||
})
|
||||
|
||||
test("tracks keyboard scrolling from a focused timeline descendant", async ({ page }, testInfo) => {
|
||||
const shellID = "prt_descendant_keyboard_01_shell"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [...history(12), userMessage(), assistantMessage([shell(shellID, "completed", lines(5))])],
|
||||
settings: { shellToolPartsExpanded: false },
|
||||
cpuRate: 4,
|
||||
reducedMotion: true,
|
||||
})
|
||||
const scroller = page.locator(".scroll-view__viewport", { has: page.locator("[data-timeline-row]") })
|
||||
const row = page.locator(`[data-timeline-part-id="${shellID}"]`).first()
|
||||
const trigger = page.locator(`[data-timeline-part-id="${shellID}"] [data-slot="collapsible-trigger"]`)
|
||||
await row.evaluate((element) => element.setAttribute("tabindex", "0"))
|
||||
await row.focus()
|
||||
for (let index = 0; index < 3; index++) {
|
||||
await row.press("PageUp")
|
||||
await page.waitForTimeout(250)
|
||||
}
|
||||
await expect
|
||||
.poll(() => scroller.evaluate((element) => element.scrollHeight - element.clientHeight - element.scrollTop))
|
||||
.toBeGreaterThan(5)
|
||||
const anchor = await scroller.evaluate((element) => {
|
||||
const view = element.getBoundingClientRect()
|
||||
return [...element.querySelectorAll<HTMLElement>("[data-timeline-key]")].find((row) => {
|
||||
const rect = row.getBoundingClientRect()
|
||||
return rect.top >= view.top + 40 && rect.bottom <= view.bottom - 40
|
||||
})?.dataset.timelineKey
|
||||
})
|
||||
expect(anchor).toBeTruthy()
|
||||
const regions = defineVisualRegions({
|
||||
anchor: { selector: `[data-timeline-key="${anchor}"]` },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await trigger.click()
|
||||
await page.waitForTimeout(300)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(testInfo, "descendant-keyboard-resize", trace, anchorPlan(regions))
|
||||
})
|
||||
|
||||
test("does not claim keyboard scrolling owned by a nested scrollable", async ({ page }) => {
|
||||
const shellID = "prt_nested_keyboard_shell"
|
||||
await setupTimeline(page, {
|
||||
messages: [userMessage(), assistantMessage([shell(shellID, "completed", lines(50))])],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
reducedMotion: true,
|
||||
seedHistory: true,
|
||||
})
|
||||
const scroller = page.locator(".scroll-view__viewport", { has: page.locator("[data-timeline-row]") })
|
||||
const nested = page.locator(`[data-timeline-part-id="${shellID}"] [data-scrollable]`)
|
||||
await nested.evaluate((element) => (element.scrollTop = element.scrollHeight))
|
||||
await nested.focus()
|
||||
await page.waitForFunction(() => {
|
||||
const root = [...document.querySelectorAll<HTMLElement>(".scroll-view__viewport")].find((element) =>
|
||||
element.querySelector("[data-timeline-row]"),
|
||||
)
|
||||
if (!root) return false
|
||||
return new Promise<boolean>((resolve) => {
|
||||
const top = root.scrollTop
|
||||
requestAnimationFrame(() => requestAnimationFrame(() => resolve(Math.abs(root.scrollTop - top) <= 0.5)))
|
||||
})
|
||||
})
|
||||
const before = await scroller.evaluate((element) => element.scrollTop)
|
||||
const nestedBefore = await nested.evaluate((element) => element.scrollTop)
|
||||
await nested.press("PageUp")
|
||||
await page.waitForTimeout(300)
|
||||
expect(await scroller.evaluate((element) => element.scrollTop)).toBe(before)
|
||||
expect(await nested.evaluate((element) => element.scrollTop)).toBeLessThan(nestedBefore)
|
||||
|
||||
await nested.evaluate((element) => (element.scrollTop = 0))
|
||||
await scroller.evaluate((element) => (element.scrollTop = Math.min(300, element.scrollHeight - element.clientHeight)))
|
||||
const boundaryBefore = await scroller.evaluate((element) => element.scrollTop)
|
||||
expect(boundaryBefore).toBeGreaterThan(0)
|
||||
await nested.press("PageUp")
|
||||
await expect.poll(() => scroller.evaluate((element) => element.scrollTop)).toBeLessThan(boundaryBefore)
|
||||
|
||||
const nonOverflowing = page.locator(`[data-timeline-part-id="${shellID}"]`).first()
|
||||
await nonOverflowing.evaluate((element) => {
|
||||
element.setAttribute("data-scrollable", "")
|
||||
element.setAttribute("tabindex", "0")
|
||||
})
|
||||
await nonOverflowing.focus()
|
||||
const nonOverflowBefore = await scroller.evaluate((element) => element.scrollTop)
|
||||
await nonOverflowing.press("PageUp")
|
||||
await expect.poll(() => scroller.evaluate((element) => element.scrollTop)).toBeLessThan(nonOverflowBefore)
|
||||
})
|
||||
|
||||
test("jump to latest lands on stable final rows after offscreen growth", async ({ page }, testInfo) => {
|
||||
const shellID = "prt_jump_01_shell"
|
||||
const followingID = "prt_jump_02_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
...history(20),
|
||||
userMessage(),
|
||||
assistantMessage([shell(shellID, "running"), textPart(followingID, "Latest visible row")], { completed: false }),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
})
|
||||
const scroller = page.locator(".scroll-view__viewport", { has: page.locator("[data-timeline-row]") })
|
||||
await scroller.evaluate(
|
||||
(element) => (element.scrollTop = Math.max(0, element.scrollHeight - element.clientHeight - 600)),
|
||||
)
|
||||
await timeline.send(partUpdated(shell(shellID, "running", lines(50))), 300)
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await page.getByRole("button", { name: /Jump to latest/i }).click()
|
||||
await expect(page.locator(`[data-timeline-part-id="${followingID}"]`)).toBeVisible()
|
||||
await page.waitForTimeout(600)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"jump-latest",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["shell", "following"] },
|
||||
{ type: "unique", regions: ["shell", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 1 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "acquire-bottom-anchor" },
|
||||
{ type: "flow", regions: ["shell", "following"] },
|
||||
]),
|
||||
)
|
||||
})
|
||||
|
||||
test("handles a single row taller than the viewport", async ({ page }, testInfo) => {
|
||||
const shellID = "prt_tall_01_shell"
|
||||
const followingID = "prt_tall_02_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([shell(shellID, "running"), textPart(followingID, "After tall row")], { completed: false }),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
viewport: { width: 900, height: 360 },
|
||||
cpuRate: 4,
|
||||
seedHistory: true,
|
||||
})
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(shell(shellID, "completed", lines(100))), 700)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"taller-than-viewport",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["shell", "following"] },
|
||||
{ type: "unique", regions: ["shell", "following"] },
|
||||
{ type: "stable", regions: ["shell", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["shell", "following"] },
|
||||
]),
|
||||
)
|
||||
})
|
||||
|
||||
function history(count: number): TimelineMessage[] {
|
||||
return Array.from({ length: count }, (_, index) => {
|
||||
const prefix = `msg_${String(index).padStart(4, "0")}_scroll`
|
||||
const userID = `${prefix}_a_user`
|
||||
return [
|
||||
userMessage(undefined, { id: userID, created: 1690000000000 + index * 10_000 }),
|
||||
assistantMessage(
|
||||
[textPart(`prt_${String(index).padStart(4, "0")}_scroll`, `History ${index}. ${"content ".repeat(30)}`)],
|
||||
{
|
||||
id: `${prefix}_b_assistant`,
|
||||
parentID: userID,
|
||||
created: 1690000001000 + index * 10_000,
|
||||
},
|
||||
),
|
||||
]
|
||||
}).flat()
|
||||
}
|
||||
|
||||
function lines(count: number) {
|
||||
return Array.from({ length: count }, (_, index) => `line ${index + 1}`).join("\n")
|
||||
}
|
||||
|
||||
function rowPairPlan(
|
||||
regions: Record<"shell" | "following", { selector: string; closest?: string }>,
|
||||
maxPositionReversals: number,
|
||||
) {
|
||||
return visualPlan(regions, [
|
||||
{ type: "required", regions: ["shell", "following"] },
|
||||
{ type: "unique", regions: ["shell", "following"] },
|
||||
{ type: "stable", regions: ["shell", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "flow", regions: ["shell", "following"] },
|
||||
])
|
||||
}
|
||||
|
||||
function anchorPlan(regions: Record<"anchor", { selector: string; closest?: string }>) {
|
||||
return visualPlan(regions, [
|
||||
{ type: "required", regions: ["anchor"] },
|
||||
{ type: "unique", regions: ["anchor"] },
|
||||
{ type: "stable", regions: ["anchor"] },
|
||||
{ type: "fixed", regions: ["anchor"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
])
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
import { test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import {
|
||||
assistantMessage,
|
||||
partUpdated,
|
||||
setupTimeline,
|
||||
shell,
|
||||
textPart,
|
||||
userMessage,
|
||||
waitForVisualSettle,
|
||||
} from "./fixture"
|
||||
|
||||
const profiles = [
|
||||
{
|
||||
name: "empty running to completed",
|
||||
updates: [{ state: "completed" as const, output: "", delay: 350 }],
|
||||
},
|
||||
{
|
||||
name: "50 lines arriving incrementally",
|
||||
updates: [
|
||||
{ state: "running" as const, output: lines(1), delay: 100 },
|
||||
{ state: "running" as const, output: lines(10), delay: 160 },
|
||||
{ state: "running" as const, output: lines(25), delay: 90 },
|
||||
{ state: "running" as const, output: lines(50), delay: 220 },
|
||||
{ state: "completed" as const, output: lines(50), delay: 500 },
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "wide ANSI and CRLF output",
|
||||
updates: [
|
||||
{
|
||||
state: "running" as const,
|
||||
output: Array.from({ length: 20 }, (_, index) => `\u001b[32mline ${index}\u001b[0m ${"wide-".repeat(30)}`).join(
|
||||
"\r\n",
|
||||
),
|
||||
delay: 240,
|
||||
},
|
||||
{
|
||||
state: "completed" as const,
|
||||
output: Array.from({ length: 20 }, (_, index) => `line ${index} ${"wide-".repeat(30)}`).join("\n"),
|
||||
delay: 500,
|
||||
},
|
||||
],
|
||||
},
|
||||
] as const
|
||||
|
||||
for (const profile of profiles) {
|
||||
test(`keeps rows stable for shell ${profile.name}`, async ({ page }, testInfo) => {
|
||||
const shellID = `prt_matrix_${profiles.indexOf(profile)}_01_shell`
|
||||
const followingID = `prt_matrix_${profiles.indexOf(profile)}_02_following`
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([shell(shellID, "running"), textPart(followingID, "Following shell row")], {
|
||||
completed: false,
|
||||
}),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
seedHistory: true,
|
||||
})
|
||||
const scroller = page.locator(".scroll-view__viewport", { has: page.locator("[data-timeline-row]") })
|
||||
await scroller.evaluate((element) => (element.scrollTop = element.scrollHeight))
|
||||
await waitForVisualSettle(page, [
|
||||
`[data-timeline-part-id="${shellID}"]`,
|
||||
`[data-timeline-part-id="${followingID}"]`,
|
||||
])
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: {
|
||||
selector: `[data-timeline-part-id="${followingID}"]`,
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
for (const update of profile.updates) {
|
||||
await timeline.send(partUpdated(shell(shellID, update.state, update.output)), update.delay)
|
||||
}
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
`shell-${profiles.indexOf(profile)}`,
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["shell", "following"] },
|
||||
{ type: "unique", regions: ["shell", "following"] },
|
||||
{ type: "stable", regions: ["shell", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0, maxReversals: 1 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["shell", "following"] },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
test("keeps following row stable when a collapsed shell receives 50 lines", async ({ page }, testInfo) => {
|
||||
const shellID = "prt_matrix_collapsed_01_shell"
|
||||
const followingID = "prt_matrix_collapsed_02_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([shell(shellID, "running"), textPart(followingID, "Following collapsed shell")], {
|
||||
completed: false,
|
||||
}),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: false },
|
||||
cpuRate: 4,
|
||||
seedHistory: true,
|
||||
})
|
||||
await waitForVisualSettle(page, [`[data-timeline-part-id="${shellID}"]`, `[data-timeline-part-id="${followingID}"]`])
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(shell(shellID, "running", lines(50))), 240)
|
||||
await timeline.send(partUpdated(shell(shellID, "completed", lines(50))), 500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"collapsed-shell",
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["shell", "following"] },
|
||||
{ type: "unique", regions: ["shell", "following"] },
|
||||
{ type: "stable", regions: ["shell", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: ["following"], maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "flow", regions: ["shell", "following"] },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
test("keeps rows stable when a running shell becomes an error", async ({ page }, testInfo) => {
|
||||
const shellID = "prt_matrix_error_01_shell"
|
||||
const followingID = "prt_matrix_error_02_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([shell(shellID, "running", lines(10)), textPart(followingID, "Following failed shell")], {
|
||||
completed: false,
|
||||
}),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
seedHistory: true,
|
||||
})
|
||||
await waitForVisualSettle(page, [`[data-timeline-part-id="${shellID}"]`, `[data-timeline-part-id="${followingID}"]`])
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(
|
||||
partUpdated({
|
||||
...shell(shellID, "error"),
|
||||
state: {
|
||||
status: "error",
|
||||
input: { command: `echo ${shellID}` },
|
||||
error: "Command failed after output",
|
||||
metadata: {},
|
||||
time: { start: 1700000001000, end: 1700000002000 },
|
||||
},
|
||||
}),
|
||||
500,
|
||||
)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"shell-error",
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["shell", "following"] },
|
||||
{ type: "unique", regions: ["shell", "following"] },
|
||||
{ type: "stable", regions: ["shell", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: ["following"], maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["shell", "following"] },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
test("keeps rows stable when later text arrives before shell output", async ({ page }, testInfo) => {
|
||||
const shellID = "prt_late_text_01_shell"
|
||||
const followingID = "prt_late_text_02_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [userMessage(), assistantMessage([shell(shellID, "running")], { completed: false })],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
seedHistory: true,
|
||||
})
|
||||
await waitForVisualSettle(page, [`[data-timeline-part-id="${shellID}"]`])
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: {
|
||||
selector: `[data-timeline-part-id="${followingID}"]`,
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(textPart(followingID, "Later assistant content arrived before shell output.")), 240)
|
||||
await timeline.send(partUpdated(shell(shellID, "running", lines(20))), 300)
|
||||
await timeline.send(partUpdated(shell(shellID, "completed", lines(20))), 600)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"late-text-before-shell-output",
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["shell", "following"] },
|
||||
{ type: "unique", regions: ["shell", "following"] },
|
||||
{ type: "stable", regions: ["shell"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["shell", "following"] },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
function lines(count: number) {
|
||||
return Array.from({ length: count }, (_, index) => `line ${index + 1}`).join("\n")
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
import { expect, test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import {
|
||||
assistantMessage,
|
||||
partUpdated,
|
||||
session,
|
||||
sessionID,
|
||||
setupTimeline,
|
||||
textPart,
|
||||
toolPart,
|
||||
userMessage,
|
||||
} from "./fixture"
|
||||
|
||||
test("adds a task child-session link without replacing the task row", async ({ page }, testInfo) => {
|
||||
const taskID = "prt_task_link"
|
||||
const childID = "ses_task_child"
|
||||
const input = { description: "Inspect child", subagent_type: "explore" }
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [userMessage(), assistantMessage([toolPart(taskID, "task", "running", input)], { completed: false })],
|
||||
sessions: [session(), session({ id: childID, parentID: sessionID, title: "Inspect child" })],
|
||||
cpuRate: 4,
|
||||
})
|
||||
const regions = defineVisualRegions({
|
||||
task: { selector: `[data-timeline-part-id="${taskID}"] [data-slot="collapsible-trigger"]` },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(
|
||||
partUpdated(toolPart(taskID, "task", "completed", input, { metadata: { sessionId: childID } })),
|
||||
500,
|
||||
)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"task-link",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["task"] },
|
||||
{ type: "unique", regions: ["task"] },
|
||||
{ type: "stable", regions: ["task"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
]),
|
||||
)
|
||||
await expect(
|
||||
page.locator(`a[href$="/session/${childID}"]`, { has: page.locator('[data-component="task-tool-card"]') }),
|
||||
).toBeVisible()
|
||||
})
|
||||
|
||||
test("changes generic tool arguments without replacing the row", async ({ page }, testInfo) => {
|
||||
const toolID = "prt_generic_mutation"
|
||||
const followingID = "prt_generic_mutation_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage(
|
||||
[
|
||||
toolPart(toolID, "mcp_probe", "running", { target: "one", count: 1 }),
|
||||
textPart(followingID, "Following generic tool"),
|
||||
],
|
||||
{ completed: false },
|
||||
),
|
||||
],
|
||||
cpuRate: 4,
|
||||
})
|
||||
const regions = defineVisualRegions({
|
||||
tool: { selector: `[data-timeline-part-id="${toolID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(
|
||||
partUpdated(toolPart(toolID, "mcp_probe", "running", { target: "two", count: 2, mode: "deep" })),
|
||||
200,
|
||||
)
|
||||
await timeline.send(
|
||||
partUpdated(toolPart(toolID, "mcp_probe", "completed", { target: "two", count: 2, mode: "deep" })),
|
||||
400,
|
||||
)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"generic-mutation",
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["tool", "following"] },
|
||||
{ type: "unique", regions: ["tool", "following"] },
|
||||
{ type: "stable", regions: ["tool", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "flow", regions: ["tool", "following"] },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,198 @@
|
||||
import { expect, test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import {
|
||||
assistantMessage,
|
||||
directory,
|
||||
partUpdated,
|
||||
session,
|
||||
sessionID,
|
||||
setupTimeline,
|
||||
status,
|
||||
textPart,
|
||||
toolPart,
|
||||
userMessage,
|
||||
} from "./fixture"
|
||||
|
||||
test.describe("timeline tool state stability", () => {
|
||||
test("moves lightweight tools through pending, running, and completed without replacing rows", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
const ids = ["webfetch", "websearch", "task", "skill", "custom"] as const
|
||||
const inputs = {
|
||||
webfetch: { url: "https://example.com/docs" },
|
||||
websearch: { query: "timeline stability" },
|
||||
task: { description: "Inspect timeline", subagent_type: "explore" },
|
||||
skill: { name: "stability" },
|
||||
custom: { target: "timeline", depth: 2 },
|
||||
}
|
||||
const names = { webfetch: "webfetch", websearch: "websearch", task: "task", skill: "skill", custom: "mcp_probe" }
|
||||
const questionID = "prt_state_question"
|
||||
const todoID = "prt_state_todo"
|
||||
const initial = [
|
||||
...ids.map((id) => toolPart(`prt_state_${id}`, names[id], "pending", inputs[id])),
|
||||
toolPart(questionID, "question", "pending", questionInput()),
|
||||
toolPart(todoID, "todowrite", "pending", { todos: [{ content: "Hidden", status: "pending" }] }),
|
||||
textPart("prt_state_following", "Following lightweight tools"),
|
||||
]
|
||||
const childID = "ses_timeline_child"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [userMessage(), assistantMessage(initial, { completed: false })],
|
||||
sessions: [session(), session({ id: childID, parentID: sessionID, title: "Inspect timeline" })],
|
||||
cpuRate: 4,
|
||||
})
|
||||
await timeline.send(status("busy"), 120)
|
||||
for (const id of ids) await timeline.waitForPart(`prt_state_${id}`)
|
||||
await expect(page.locator(`[data-timeline-part-id="${questionID}"]`)).toHaveCount(0)
|
||||
await expect(page.locator(`[data-timeline-part-id="${todoID}"]`)).toHaveCount(0)
|
||||
|
||||
const regionIDs = [
|
||||
"prt_state_webfetch",
|
||||
"prt_state_websearch",
|
||||
"prt_state_task",
|
||||
"prt_state_skill",
|
||||
"prt_state_custom",
|
||||
] as const
|
||||
const regions = defineVisualRegions({
|
||||
prt_state_webfetch: toolRegion(regionIDs[0]),
|
||||
prt_state_websearch: toolRegion(regionIDs[1]),
|
||||
prt_state_task: toolRegion(regionIDs[2]),
|
||||
prt_state_skill: toolRegion(regionIDs[3]),
|
||||
prt_state_custom: toolRegion(regionIDs[4]),
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
for (const [index, id] of ids.entries()) {
|
||||
await timeline.send(
|
||||
partUpdated(toolPart(`prt_state_${id}`, names[id], "running", inputs[id])),
|
||||
[80, 240, 100, 360, 140][index],
|
||||
)
|
||||
}
|
||||
for (const [index, id] of ["skill", "webfetch", "custom", "task", "websearch"].entries()) {
|
||||
const key = id as (typeof ids)[number]
|
||||
const metadata = key === "task" ? { sessionId: childID } : key === "websearch" ? { provider: "exa" } : {}
|
||||
const output = key === "websearch" ? "Result https://example.com/result" : "Completed"
|
||||
await timeline.send(
|
||||
partUpdated(toolPart(`prt_state_${key}`, names[key], "completed", inputs[key], { metadata, output })),
|
||||
[110, 70, 280, 130, 420][index],
|
||||
)
|
||||
}
|
||||
await timeline.send(
|
||||
partUpdated(
|
||||
toolPart(questionID, "question", "completed", questionInput(), { metadata: { answers: [["Keep it stable"]] } }),
|
||||
),
|
||||
350,
|
||||
)
|
||||
await timeline.waitForPart(questionID)
|
||||
await timeline.send(status("idle"), 500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"lightweight-tools",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: regionIDs },
|
||||
{ type: "unique", regions: regionIDs },
|
||||
{ type: "stable", regions: regionIDs },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxReversals: 4 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
]),
|
||||
)
|
||||
await expect(page.locator(`[data-timeline-part-id="${questionID}"]`)).toContainText("Keep it stable")
|
||||
await expect(page.locator(`[data-timeline-part-id="${todoID}"]`)).toHaveCount(0)
|
||||
await expect(
|
||||
page.locator(`a[href$="/session/${childID}"]`, { has: page.locator('[data-component="task-tool-card"]') }),
|
||||
).toBeVisible()
|
||||
await expect(page.getByRole("button", { name: /Exa Web Search/ })).toBeVisible()
|
||||
})
|
||||
|
||||
test("keeps an expanded mixed context group stable through staggered completion and error", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
const ids = ["prt_ctx_01_read", "prt_ctx_02_glob", "prt_ctx_03_grep", "prt_ctx_04_list"]
|
||||
const tools = ["read", "glob", "grep", "list"]
|
||||
const inputs = [
|
||||
{ filePath: "src/a.ts", offset: 0, limit: 120 },
|
||||
{ path: directory, pattern: "**/*.ts" },
|
||||
{ path: directory, pattern: "stability", include: "*.ts" },
|
||||
{ path: "src" },
|
||||
]
|
||||
const context = ids.map((id, index) => toolPart(id, tools[index]!, "pending", inputs[index]!))
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([...context, textPart("prt_ctx_following", "Following context")], { completed: false }),
|
||||
],
|
||||
cpuRate: 4,
|
||||
})
|
||||
await timeline.send(status("busy"), 100)
|
||||
const groupSelector = `[data-timeline-part-ids="${ids.join(",")}"]`
|
||||
const group = page.locator(groupSelector)
|
||||
await expect(group).toBeVisible()
|
||||
await group.locator('[data-slot="collapsible-trigger"]').click()
|
||||
await expect(group.locator('[data-slot="collapsible-trigger"]')).toHaveAttribute("aria-expanded", "true")
|
||||
|
||||
const regions = defineVisualRegions({
|
||||
status: {
|
||||
selector: `${groupSelector} [data-component="tool-status-title"]`,
|
||||
opacitySelectors: ['[data-slot="tool-status-active"]', '[data-slot="tool-status-done"]'],
|
||||
},
|
||||
context: { selector: groupSelector, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: {
|
||||
selector: '[data-timeline-part-id="prt_ctx_following"]',
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
for (const [index, delay] of [90, 260, 70, 380].entries()) {
|
||||
await timeline.send(partUpdated(toolPart(ids[index]!, tools[index]!, "running", inputs[index]!)), delay)
|
||||
}
|
||||
await timeline.send(partUpdated(toolPart(ids[1]!, tools[1]!, "completed", inputs[1]!)), 130)
|
||||
await timeline.send(partUpdated(toolPart(ids[3]!, tools[3]!, "completed", inputs[3]!)), 210)
|
||||
await timeline.send(
|
||||
partUpdated(toolPart(ids[0]!, tools[0]!, "error", inputs[0]!, { error: "Read interrupted" })),
|
||||
110,
|
||||
)
|
||||
await timeline.send(partUpdated(toolPart(ids[2]!, tools[2]!, "completed", inputs[2]!)), 250)
|
||||
await expect(group.locator('[data-component="tool-status-title"]')).toHaveAttribute("aria-label", "Explored")
|
||||
await timeline.send(status("idle"), 700)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"mixed-context",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["context", "following"] },
|
||||
{ type: "unique", regions: ["context"] },
|
||||
{ type: "stable", regions: ["context"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxReversals: 4 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "flow", regions: ["context", "following"] },
|
||||
]),
|
||||
)
|
||||
await expect(group.locator('[data-component="tool-status-title"]')).toHaveAttribute("aria-label", "Explored")
|
||||
await expect(group.locator('[data-slot="collapsible-trigger"]')).toHaveAttribute("aria-expanded", "true")
|
||||
await group.locator('[data-slot="collapsible-trigger"]').click()
|
||||
await expect(group.locator('[data-slot="collapsible-trigger"]')).toHaveAttribute("aria-expanded", "false")
|
||||
await timeline.send(partUpdated(textPart("prt_ctx_late_sibling", "Later sibling content")), 200)
|
||||
await expect(group.locator('[data-slot="collapsible-trigger"]')).toHaveAttribute("aria-expanded", "false")
|
||||
await group.locator('[data-slot="collapsible-trigger"]').click()
|
||||
await expect(group.locator('[data-slot="collapsible-trigger"]')).toHaveAttribute("aria-expanded", "true")
|
||||
})
|
||||
})
|
||||
|
||||
function questionInput() {
|
||||
return { questions: [{ header: "Stability", question: "Keep it stable?", options: [] }] }
|
||||
}
|
||||
|
||||
function toolRegion(id: string) {
|
||||
return { selector: `[data-timeline-part-id="${id}"]`, closest: '[data-timeline-row="AssistantPart"]' }
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
import { expect, test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import {
|
||||
assistantID,
|
||||
assistantMessage,
|
||||
completedAssistantInfo,
|
||||
event,
|
||||
messageUpdated,
|
||||
partDelta,
|
||||
partUpdated,
|
||||
setupTimeline,
|
||||
shell,
|
||||
status,
|
||||
textPart,
|
||||
toolPart,
|
||||
userMessage,
|
||||
} from "./fixture"
|
||||
|
||||
test("keeps unchanged siblings stable while a middle part is inserted and removed", async ({ page }, testInfo) => {
|
||||
const firstID = "prt_mutation_01_first"
|
||||
const middleID = "prt_mutation_02_middle"
|
||||
const lastID = "prt_mutation_03_last"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([textPart(firstID, "First stable row"), textPart(lastID, "Last stable row")], {
|
||||
completed: false,
|
||||
}),
|
||||
],
|
||||
cpuRate: 4,
|
||||
})
|
||||
const regions = defineVisualRegions({
|
||||
first: { selector: `[data-timeline-part-id="${firstID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
last: { selector: `[data-timeline-part-id="${lastID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(textPart(middleID, "Inserted middle row. ".repeat(12))), 350)
|
||||
await expect(page.locator(`[data-timeline-part-id="${middleID}"]`)).toBeVisible()
|
||||
await timeline.send(
|
||||
event("message.part.removed", { sessionID: "ses_timeline_stability", messageID: assistantID, partID: middleID }),
|
||||
500,
|
||||
)
|
||||
await expect(page.locator(`[data-timeline-part-id="${middleID}"]`)).toHaveCount(0)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(testInfo, "middle-insert-remove", trace, stablePairPlan(regions, 1))
|
||||
})
|
||||
|
||||
test("streams text through growth, canonical replacement, and completion", async ({ page }, testInfo) => {
|
||||
const textID = "prt_text_reconcile"
|
||||
const followingID = "prt_text_reconcile_following"
|
||||
const assistant = assistantMessage([textPart(textID, "Starting"), textPart(followingID, "Following text row")], {
|
||||
completed: false,
|
||||
})
|
||||
const timeline = await setupTimeline(page, { messages: [userMessage(), assistant], cpuRate: 4 })
|
||||
const regions = defineVisualRegions({
|
||||
text: { selector: `[data-timeline-part-id="${textID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partDelta(textID, " streamed content"), 100)
|
||||
await timeline.send(partDelta(textID, "\n\n- item one\n- item two\n- item three"), 180)
|
||||
await timeline.send(partUpdated(textPart(textID, "Canonical replacement with a shorter final paragraph.")), 200)
|
||||
await timeline.send(messageUpdated(completedAssistantInfo(assistant.info)), 500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"text-reconcile",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["text", "following"] },
|
||||
{ type: "unique", regions: ["text", "following"] },
|
||||
{ type: "stable", regions: ["text", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 1, maxReversals: 2 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["text", "following"] },
|
||||
]),
|
||||
)
|
||||
})
|
||||
|
||||
test("inserts a completed question between stable rows", async ({ page }, testInfo) => {
|
||||
const firstID = "prt_question_01_first"
|
||||
const questionID = "prt_question_02_hidden"
|
||||
const lastID = "prt_question_03_last"
|
||||
const input = { questions: [{ header: "Choice", question: "Keep stable?", options: [] }] }
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage(
|
||||
[
|
||||
textPart(firstID, "Before question"),
|
||||
toolPart(questionID, "question", "running", input),
|
||||
textPart(lastID, "After question"),
|
||||
],
|
||||
{ completed: false },
|
||||
),
|
||||
],
|
||||
cpuRate: 4,
|
||||
})
|
||||
await expect(page.locator(`[data-timeline-part-id="${questionID}"]`)).toHaveCount(0)
|
||||
const regions = defineVisualRegions({
|
||||
first: { selector: `[data-timeline-part-id="${firstID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
last: { selector: `[data-timeline-part-id="${lastID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(
|
||||
partUpdated(toolPart(questionID, "question", "completed", input, { metadata: { answers: [["Yes"]] } })),
|
||||
600,
|
||||
)
|
||||
await expect(page.locator(`[data-timeline-part-id="${questionID}"]`)).toBeVisible()
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(testInfo, "question-insert", trace, stablePairPlan(regions, 0))
|
||||
})
|
||||
|
||||
test("replaces thinking with an assistant error without a blank turn", async ({ page }, testInfo) => {
|
||||
const assistant = assistantMessage([], { completed: false })
|
||||
const timeline = await setupTimeline(page, { messages: [userMessage(), assistant], cpuRate: 4 })
|
||||
await timeline.send(status("busy"), 150)
|
||||
await expect(page.locator('[data-timeline-row="Thinking"]')).toBeVisible()
|
||||
const regions = defineVisualRegions({
|
||||
thinking: { selector: '[data-timeline-row="Thinking"]' },
|
||||
error: { selector: '[data-timeline-row="Error"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(
|
||||
messageUpdated({
|
||||
...assistant.info,
|
||||
error: { name: "APIError", data: { message: "Provider failed visibly", isRetryable: false } },
|
||||
}),
|
||||
500,
|
||||
)
|
||||
await expect(page.locator('[data-timeline-row="Thinking"]')).toHaveCount(0)
|
||||
await expect(page.locator('[data-timeline-row="Error"]')).toContainText("Provider failed visibly")
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"thinking-error",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["thinking", "error"] },
|
||||
{ type: "continuous-any", regions: ["thinking", "error"] },
|
||||
{ type: "unique", regions: ["thinking", "error"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all" },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
]),
|
||||
)
|
||||
})
|
||||
|
||||
test("updates retry attempts and long provider messages without remounting the retry row", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [userMessage(), assistantMessage([], { completed: false })],
|
||||
cpuRate: 4,
|
||||
})
|
||||
await timeline.send(status("retry", 1), 120)
|
||||
await expect(page.locator('[data-timeline-row="Retry"]')).toBeVisible()
|
||||
const regions = defineVisualRegions({
|
||||
retry: { selector: '[data-timeline-row="Retry"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(
|
||||
event("session.status", {
|
||||
sessionID: "ses_timeline_stability",
|
||||
status: {
|
||||
type: "retry",
|
||||
attempt: 2,
|
||||
message: "A very long provider retry message ".repeat(8),
|
||||
next: Date.now() + 10_000,
|
||||
},
|
||||
}),
|
||||
300,
|
||||
)
|
||||
await timeline.send(status("retry", 3), 300)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"retry-evolution",
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["retry"] },
|
||||
{ type: "unique", regions: ["retry"] },
|
||||
{ type: "stable", regions: ["retry"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
test("reducer-hardening: removes a historical turn one message at a time without moving a visible lower anchor twice", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
const removeUserID = "msg_0500_remove_user"
|
||||
const removeAssistantID = "msg_0501_remove_assistant"
|
||||
const anchorUserID = "msg_2000_anchor_user"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(undefined, { id: removeUserID, created: 1690000000000 }),
|
||||
assistantMessage([textPart("prt_remove_text", "Removed historical content. ".repeat(15))], {
|
||||
id: removeAssistantID,
|
||||
parentID: removeUserID,
|
||||
created: 1690000001000,
|
||||
}),
|
||||
userMessage(undefined, { id: anchorUserID, created: 1700000000000 }),
|
||||
assistantMessage([textPart("prt_anchor_text", "Visible anchor response")], {
|
||||
id: "msg_2001_anchor_assistant",
|
||||
parentID: anchorUserID,
|
||||
created: 1700000001000,
|
||||
}),
|
||||
],
|
||||
cpuRate: 4,
|
||||
})
|
||||
const regions = defineVisualRegions({
|
||||
anchor: { selector: `[data-timeline-row="UserMessage"][data-message-id="${anchorUserID}"]` },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(
|
||||
event("message.removed", { sessionID: "ses_timeline_stability", messageID: removeAssistantID }),
|
||||
200,
|
||||
)
|
||||
await timeline.send(event("message.removed", { sessionID: "ses_timeline_stability", messageID: removeUserID }), 500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"historical-turn-remove",
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["anchor"] },
|
||||
{ type: "unique", regions: ["anchor"] },
|
||||
{ type: "stable", regions: ["anchor"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
function stablePairPlan(
|
||||
regions: Record<"first" | "last", { selector: string; closest?: string }>,
|
||||
maxPositionReversals: number,
|
||||
) {
|
||||
return visualPlan(regions, [
|
||||
{ type: "required", regions: ["first", "last"] },
|
||||
{ type: "unique", regions: ["first", "last"] },
|
||||
{ type: "stable", regions: ["first", "last"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
])
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
import type { Page } from "@playwright/test"
|
||||
import { benchmark, expect } from "../benchmark"
|
||||
import { setupTimelineBenchmark } from "./session-timeline-benchmark.fixture"
|
||||
|
||||
const changedLinesPerFile = 100
|
||||
const linesPerSide = changedLinesPerFile / 2
|
||||
const fileCounts = [1, 10, 100, 1_000, 10_000]
|
||||
const filesPerDirectory = 100
|
||||
const readyFrames = 3
|
||||
const completionTimeoutMs = Number(process.env.REVIEW_PANE_COMPLETION_TIMEOUT_MS ?? 900_000)
|
||||
|
||||
type ReviewPaneScalingSample = {
|
||||
observedAtMs: number
|
||||
logicalRows: number
|
||||
treeRows: number
|
||||
fileRows: number
|
||||
diffLines: number
|
||||
header: string
|
||||
ready: boolean
|
||||
}
|
||||
|
||||
type ReviewPaneScalingProbe = {
|
||||
startedAt?: number
|
||||
firstTreeRowMs?: number
|
||||
logicalTreeReadyMs?: number
|
||||
firstDiffRenderMs?: number
|
||||
stableReadyMs?: number
|
||||
samples: ReviewPaneScalingSample[]
|
||||
frameTimesMs: number[]
|
||||
longTasks: { startTime: number; duration: number }[]
|
||||
stop: () => void
|
||||
}
|
||||
|
||||
benchmark.describe("performance: review pane scaling", () => {
|
||||
for (const fileCount of fileCounts) {
|
||||
const changedLines = fileCount * changedLinesPerFile
|
||||
|
||||
benchmark(
|
||||
`${changedLines} changed lines across ${fileCount} ${fileCount === 1 ? "file" : "files"}`,
|
||||
async ({ page, report }) => {
|
||||
benchmark.setTimeout(1_200_000)
|
||||
await page.emulateMedia({ reducedMotion: "reduce" })
|
||||
|
||||
const patchByteLimit = Number(process.env.REVIEW_PANE_PATCH_BYTE_LIMIT ?? Number.POSITIVE_INFINITY)
|
||||
if (Number.isNaN(patchByteLimit) || patchByteLimit < 0)
|
||||
throw new Error(`Invalid REVIEW_PANE_PATCH_BYTE_LIMIT: ${process.env.REVIEW_PANE_PATCH_BYTE_LIMIT}`)
|
||||
const responseBody = JSON.stringify(createScalingDiffs(fileCount, patchByteLimit))
|
||||
await setupTimelineBenchmark(page, {
|
||||
historyTurns: 0,
|
||||
eventBatch: 1,
|
||||
newLayoutDesigns: true,
|
||||
})
|
||||
await page.route("**/vcs/diff**", (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: "application/json",
|
||||
headers: { "access-control-allow-origin": "*" },
|
||||
body: responseBody,
|
||||
}),
|
||||
)
|
||||
|
||||
const expectedRows = fileCount + 2 + Math.ceil(fileCount / filesPerDirectory)
|
||||
const metrics = await measureReviewPaneLoad(page, {
|
||||
expectedFile: reviewFile(0),
|
||||
expectedRows,
|
||||
})
|
||||
const search = await measureBroadReviewSearch(page, fileCount)
|
||||
|
||||
expect(metrics.logicalRows).toBe(expectedRows)
|
||||
expect(metrics.fileRows).toBeGreaterThan(0)
|
||||
expect(metrics.treeRows).toBeGreaterThan(0)
|
||||
expect(metrics.diffLines).toBeGreaterThan(0)
|
||||
expect(search.logicalRows).toBe(fileCount)
|
||||
expect(search.renderedRows).toBeGreaterThan(0)
|
||||
report(
|
||||
{ ...metrics, search },
|
||||
{
|
||||
fileCount,
|
||||
changedLinesPerFile,
|
||||
changedLines,
|
||||
additions: changedLines / 2,
|
||||
deletions: changedLines / 2,
|
||||
patchLines: changedLines,
|
||||
patchByteLimit: Number.isFinite(patchByteLimit) ? patchByteLimit : null,
|
||||
payloadBytes: new TextEncoder().encode(responseBody).byteLength,
|
||||
expectedRows,
|
||||
},
|
||||
)
|
||||
},
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
async function measureBroadReviewSearch(page: Page, expectedRows: number) {
|
||||
const filter = page.getByRole("searchbox", { name: "Filter files" })
|
||||
await filter.evaluate((element) => {
|
||||
element.addEventListener(
|
||||
"input",
|
||||
() => {
|
||||
;(window as Window & { __reviewSearchStartedAt?: number }).__reviewSearchStartedAt = performance.now()
|
||||
},
|
||||
{ once: true, capture: true },
|
||||
)
|
||||
})
|
||||
await filter.fill("file-")
|
||||
|
||||
return page.evaluate((expectedRows) => {
|
||||
const startedAt = (window as Window & { __reviewSearchStartedAt?: number }).__reviewSearchStartedAt!
|
||||
return new Promise<{ stableMs: number; logicalRows: number; renderedRows: number }>((resolve) => {
|
||||
let previous = -1
|
||||
let streak = 0
|
||||
const sample = () => {
|
||||
const tree = document.querySelector<HTMLElement>('#review-panel [data-component="file-tree-v2"]')
|
||||
const rows = [...document.querySelectorAll<HTMLElement>('#review-panel [data-slot="file-tree-v2-row"]')]
|
||||
const logicalRows = Number(tree?.dataset.totalRows ?? rows.length)
|
||||
const ready =
|
||||
logicalRows === expectedRows && rows.length > 0 && rows.every((row) => row.textContent?.includes("file-"))
|
||||
streak = ready && rows.length === previous ? streak + 1 : ready ? 1 : 0
|
||||
previous = rows.length
|
||||
if (streak >= 3) {
|
||||
resolve({ stableMs: performance.now() - startedAt, logicalRows, renderedRows: rows.length })
|
||||
return
|
||||
}
|
||||
requestAnimationFrame(sample)
|
||||
}
|
||||
requestAnimationFrame(sample)
|
||||
})
|
||||
}, expectedRows)
|
||||
}
|
||||
|
||||
function createScalingDiffs(fileCount: number, patchByteLimit: number) {
|
||||
const changes = Array.from({ length: linesPerSide }, (_, index) => {
|
||||
const line = String(index).padStart(3, "0")
|
||||
return `-export const value_${line} = "before"\n+export const value_${line} = "after"`
|
||||
}).join("\n")
|
||||
let patchBytes = 0
|
||||
let capped = false
|
||||
|
||||
return Array.from({ length: fileCount }, (_, index) => {
|
||||
const file = reviewFile(index)
|
||||
const fullPatch = [
|
||||
`diff --git a/${file} b/${file}`,
|
||||
`--- a/${file}`,
|
||||
`+++ b/${file}`,
|
||||
`@@ -1,${linesPerSide} +1,${linesPerSide} @@`,
|
||||
changes,
|
||||
].join("\n")
|
||||
if (index === 0 && fullPatch.length > patchByteLimit)
|
||||
throw new Error(`REVIEW_PANE_PATCH_BYTE_LIMIT must include the active patch (${fullPatch.length} bytes)`)
|
||||
const patch = !capped && patchBytes + fullPatch.length <= patchByteLimit ? fullPatch : emptyReviewPatch(file)
|
||||
if (patch === fullPatch) patchBytes += fullPatch.length
|
||||
else capped = true
|
||||
return {
|
||||
file,
|
||||
patch,
|
||||
additions: linesPerSide,
|
||||
deletions: linesPerSide,
|
||||
status: "modified" as const,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function emptyReviewPatch(file: string) {
|
||||
return [`diff --git a/${file} b/${file}`, `--- a/${file}`, `+++ b/${file}`].join("\n")
|
||||
}
|
||||
|
||||
function reviewFile(index: number) {
|
||||
return `src/review/d${String(Math.floor(index / filesPerDirectory)).padStart(5, "0")}/file-${String(index).padStart(5, "0")}.ts`
|
||||
}
|
||||
|
||||
async function measureReviewPaneLoad(page: Page, input: { expectedFile: string; expectedRows: number }) {
|
||||
const toggle = page.getByRole("button", { name: "Toggle review" })
|
||||
await expect(toggle).toBeVisible()
|
||||
await toggle.evaluate((element) => element.setAttribute("data-review-pane-scaling-toggle", ""))
|
||||
await installReviewPaneScalingProbe(page, input)
|
||||
await toggle.click()
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
(window as Window & { __reviewPaneScalingProbe?: ReviewPaneScalingProbe }).__reviewPaneScalingProbe
|
||||
?.stableReadyMs !== undefined,
|
||||
undefined,
|
||||
{ timeout: completionTimeoutMs },
|
||||
)
|
||||
|
||||
return page.evaluate(() => {
|
||||
const probe = (window as Window & { __reviewPaneScalingProbe?: ReviewPaneScalingProbe }).__reviewPaneScalingProbe!
|
||||
probe.stop()
|
||||
const startedAt = probe.startedAt!
|
||||
const final = probe.samples.at(-1)!
|
||||
const resources = performance
|
||||
.getEntriesByType("resource")
|
||||
.filter((entry) => entry.name.includes("/vcs/diff")) as PerformanceResourceTiming[]
|
||||
const resource = resources.at(-1)
|
||||
const longTasks = probe.longTasks.filter(
|
||||
(entry) => entry.startTime >= startedAt && entry.startTime <= startedAt + probe.stableReadyMs!,
|
||||
)
|
||||
const frameGaps = probe.frameTimesMs.map((time, index) => time - (probe.frameTimesMs[index - 1] ?? 0))
|
||||
|
||||
return {
|
||||
firstTreeRowMs: probe.firstTreeRowMs ?? null,
|
||||
logicalTreeReadyMs: probe.logicalTreeReadyMs ?? null,
|
||||
firstDiffRenderMs: probe.firstDiffRenderMs ?? null,
|
||||
stableReadyMs: probe.stableReadyMs ?? null,
|
||||
responseStartMs: resource ? resource.responseStart - startedAt : null,
|
||||
responseEndMs: resource ? resource.responseEnd - startedAt : null,
|
||||
responseToStableMs: resource ? probe.stableReadyMs! - (resource.responseEnd - startedAt) : null,
|
||||
treeRows: final.treeRows,
|
||||
logicalRows: final.logicalRows,
|
||||
fileRows: final.fileRows,
|
||||
diffLines: final.diffLines,
|
||||
samples: probe.samples.length,
|
||||
maxFrameGapMs: Math.max(0, ...frameGaps),
|
||||
longTaskCount: longTasks.length,
|
||||
longTaskTotalMs: longTasks.reduce((sum, entry) => sum + entry.duration, 0),
|
||||
maxLongTaskMs: Math.max(0, ...longTasks.map((entry) => entry.duration)),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async function installReviewPaneScalingProbe(page: Page, input: { expectedFile: string; expectedRows: number }) {
|
||||
await page.evaluate(
|
||||
({ expectedFile, expectedRows, stableFrames }) => {
|
||||
let running = true
|
||||
let readyStreak = 0
|
||||
const basename = expectedFile.split("/").at(-1)!
|
||||
const longTaskObserver = PerformanceObserver.supportedEntryTypes.includes("longtask")
|
||||
? new PerformanceObserver((list) => {
|
||||
probe.longTasks.push(
|
||||
...list.getEntries().map((entry) => ({ startTime: entry.startTime, duration: entry.duration })),
|
||||
)
|
||||
})
|
||||
: undefined
|
||||
const probe: ReviewPaneScalingProbe = {
|
||||
samples: [],
|
||||
frameTimesMs: [],
|
||||
longTasks: [],
|
||||
stop: () => {
|
||||
running = false
|
||||
longTaskObserver?.disconnect()
|
||||
},
|
||||
}
|
||||
|
||||
const sample = (time: number) => {
|
||||
if (!running || probe.startedAt === undefined) return
|
||||
const panel = document.querySelector<HTMLElement>("#review-panel")
|
||||
const tree = panel?.querySelector<HTMLElement>('[data-component="file-tree-v2"]')
|
||||
const rows = panel?.querySelectorAll('[data-slot="file-tree-v2-row"]') ?? []
|
||||
const fileRows = panel?.querySelectorAll('button[data-slot="file-tree-v2-row"]') ?? []
|
||||
const header =
|
||||
panel?.querySelector<HTMLElement>('[data-slot="session-review-v2-file-header"]')?.textContent?.trim() ?? ""
|
||||
const viewers = panel
|
||||
? [...panel.querySelectorAll<HTMLElement>('[data-component="file"][data-mode="diff"]')]
|
||||
: []
|
||||
const diffLines = viewers.reduce(
|
||||
(sum, viewer) =>
|
||||
sum + (viewer.querySelector("diffs-container")?.shadowRoot?.querySelectorAll("[data-line]").length ?? 0),
|
||||
0,
|
||||
)
|
||||
const observedAtMs = time - probe.startedAt
|
||||
const logicalRows = Number(tree?.dataset.totalRows ?? rows.length)
|
||||
const ready =
|
||||
logicalRows === expectedRows &&
|
||||
fileRows.length > 0 &&
|
||||
header.includes(basename) &&
|
||||
viewers.length === 1 &&
|
||||
diffLines > 0
|
||||
const previous = probe.samples.at(-1)
|
||||
const stable =
|
||||
ready &&
|
||||
previous?.ready === true &&
|
||||
previous.logicalRows === logicalRows &&
|
||||
previous.treeRows === rows.length &&
|
||||
previous.fileRows === fileRows.length &&
|
||||
previous.diffLines === diffLines &&
|
||||
previous.header === header
|
||||
|
||||
probe.frameTimesMs.push(observedAtMs)
|
||||
probe.samples.push({
|
||||
observedAtMs,
|
||||
logicalRows,
|
||||
treeRows: rows.length,
|
||||
fileRows: fileRows.length,
|
||||
diffLines,
|
||||
header,
|
||||
ready,
|
||||
})
|
||||
if (probe.firstTreeRowMs === undefined && rows.length > 0) probe.firstTreeRowMs = observedAtMs
|
||||
if (probe.logicalTreeReadyMs === undefined && logicalRows === expectedRows)
|
||||
probe.logicalTreeReadyMs = observedAtMs
|
||||
if (probe.firstDiffRenderMs === undefined && diffLines > 0) probe.firstDiffRenderMs = observedAtMs
|
||||
readyStreak = !ready ? 0 : stable ? readyStreak + 1 : 1
|
||||
if (readyStreak === stableFrames) probe.stableReadyMs = observedAtMs
|
||||
if (probe.stableReadyMs === undefined) requestAnimationFrame(sample)
|
||||
}
|
||||
|
||||
longTaskObserver?.observe({ type: "longtask", buffered: true })
|
||||
document.addEventListener(
|
||||
"click",
|
||||
(event) => {
|
||||
const toggle = event.target instanceof Element ? event.target.closest("button") : undefined
|
||||
if (!toggle?.hasAttribute("data-review-pane-scaling-toggle")) return
|
||||
probe.startedAt = performance.now()
|
||||
performance.mark("opencode.review-pane-scaling.click")
|
||||
requestAnimationFrame(sample)
|
||||
},
|
||||
{ capture: true, once: true },
|
||||
)
|
||||
;(window as Window & { __reviewPaneScalingProbe?: ReviewPaneScalingProbe }).__reviewPaneScalingProbe = probe
|
||||
},
|
||||
{ ...input, stableFrames: readyFrames },
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
import type { Page } from "@playwright/test"
|
||||
import { expectSessionTitle } from "../../utils/waits"
|
||||
import { mockOpenCodeServer } from "../../utils/mock-server"
|
||||
import { benchmark, expect, withBenchmarkPage } from "../benchmark"
|
||||
import { fixture } from "./session-timeline-stress.fixture"
|
||||
import { installStressSessionTabs, stressSessionHref } from "./timeline-test-helpers"
|
||||
import { measureSessionSwitch, waitForStableTimeline } from "./session-tab-switch-probe"
|
||||
|
||||
type ParentHydrationBenchmarkMode = "natural" | "candidate"
|
||||
|
||||
const mode = process.env.SESSION_PARENT_HYDRATION_BENCHMARK_MODE ?? "natural"
|
||||
if (mode !== "natural" && mode !== "candidate") throw new Error(`Unknown parent hydration benchmark mode: ${mode}`)
|
||||
const userID = "msg_parent_hydration_user"
|
||||
const user = {
|
||||
...fixture.messages[fixture.targetID][0]!,
|
||||
info: { ...fixture.messages[fixture.targetID][0]!.info, id: userID, time: { created: 1700001000000 } },
|
||||
parts: fixture.messages[fixture.targetID][0]!.parts.map((part, index) => ({
|
||||
...part,
|
||||
id: `prt_parent_hydration_user_${index}`,
|
||||
messageID: userID,
|
||||
})),
|
||||
}
|
||||
const assistantSeed = fixture.messages[fixture.targetID][3]!
|
||||
const assistants = Array.from({ length: 14 }, (_, index) => {
|
||||
const messageID = `msg_parent_hydration_${String(index).padStart(2, "0")}`
|
||||
return {
|
||||
...assistantSeed,
|
||||
info: {
|
||||
...assistantSeed.info,
|
||||
id: messageID,
|
||||
parentID: userID,
|
||||
time: { created: 1700001001000 + index * 1_000, completed: 1700001001500 + index * 1_000 },
|
||||
},
|
||||
parts: assistantSeed.parts.map((part, partIndex) => ({
|
||||
...part,
|
||||
id: `prt_parent_hydration_${String(index).padStart(2, "0")}_${partIndex}`,
|
||||
messageID,
|
||||
})),
|
||||
}
|
||||
})
|
||||
const messages = [user, ...assistants]
|
||||
const target = fixture.sessions.find((session) => session.id === fixture.targetID)!
|
||||
const lastID = userID
|
||||
const lastPartID = assistants.at(-1)!.parts.at(-1)!.id
|
||||
|
||||
benchmark("hydrates an orphaned latest turn after a cold session click", async ({ browser, report }, testInfo) => {
|
||||
benchmark.setTimeout(180_000)
|
||||
const results = [] as Awaited<ReturnType<typeof trial>>[]
|
||||
for (let run = 0; run < 5; run++) {
|
||||
results.push(
|
||||
await withBenchmarkPage(
|
||||
browser,
|
||||
`session-parent-hydration-${mode}-${run}`,
|
||||
(page) => trial(page, mode),
|
||||
testInfo,
|
||||
),
|
||||
)
|
||||
}
|
||||
const timing = results.map((result) => result.metrics.firstCorrectObservedMs!).sort((a, b) => a - b)
|
||||
report(
|
||||
{
|
||||
results: results.map((result) => ({ ...result.metrics, historyGateCount: result.historyGateCount })),
|
||||
summary: {
|
||||
firstCorrectObservedMs: { min: timing[0], median: timing[2], max: timing.at(-1) },
|
||||
blankSamples: results.map((result) => result.metrics.blankSamples),
|
||||
requestCounts: {
|
||||
list: results.map((result) => result.requestCounts.list),
|
||||
parent: results.map((result) => result.requestCounts.parent),
|
||||
},
|
||||
historyGateCount: results.map((result) => result.historyGateCount),
|
||||
},
|
||||
},
|
||||
{ mode },
|
||||
)
|
||||
})
|
||||
|
||||
async function trial(page: Page, mode: ParentHydrationBenchmarkMode) {
|
||||
const requests: { type: "list" | "parent"; before?: string }[] = []
|
||||
const history = mode === "candidate" ? Promise.withResolvers<void>() : undefined
|
||||
let historyGates = 0
|
||||
await mockOpenCodeServer(page, {
|
||||
sessions: fixture.sessions.filter((session) => session.id === fixture.sourceID),
|
||||
provider: fixture.provider,
|
||||
directory: fixture.directory,
|
||||
project: fixture.project,
|
||||
messageDelay: 50,
|
||||
onMessages: (request) => {
|
||||
if (request.sessionID === fixture.targetID && request.phase === "start")
|
||||
requests.push({ type: "list", before: request.before })
|
||||
},
|
||||
beforeMessagesResponse: (request) => {
|
||||
if (mode !== "candidate" || request.sessionID !== fixture.targetID || !request.before) return Promise.resolve()
|
||||
historyGates++
|
||||
return history!.promise
|
||||
},
|
||||
onMessage: (request) => {
|
||||
if (request.sessionID === fixture.targetID && request.messageID === userID) requests.push({ type: "parent" })
|
||||
},
|
||||
message: (sessionID, messageID) => {
|
||||
if (sessionID !== fixture.targetID || messageID !== userID) return
|
||||
return user
|
||||
},
|
||||
pageMessages: (sessionID, limit, before) => {
|
||||
const items = sessionID === fixture.targetID ? messages : fixture.messages[fixture.sourceID]
|
||||
const end = before ? items.findIndex((message) => message.info.id === before) : items.length
|
||||
const start = Math.max(0, end - limit)
|
||||
return { items: items.slice(start, end), cursor: start > 0 ? items[start]!.info.id : undefined }
|
||||
},
|
||||
})
|
||||
await page.route(`**/session/${fixture.targetID}`, (route) =>
|
||||
route.fulfill({ status: 200, contentType: "application/json", body: JSON.stringify(target) }),
|
||||
)
|
||||
await installStressSessionTabs(page, { sessionIDs: [fixture.sourceID] })
|
||||
await page.goto(stressSessionHref(fixture.sourceID))
|
||||
await expectSessionTitle(page, fixture.expected.sourceTitle)
|
||||
await waitForStableTimeline(page, fixture.expected.sourceMessageIDs.at(-1)!)
|
||||
|
||||
const href = stressSessionHref(fixture.targetID)
|
||||
await page.evaluate(
|
||||
({ href, title }) => {
|
||||
const link = document.createElement("a")
|
||||
link.id = "parent-hydration-target"
|
||||
link.href = href
|
||||
link.textContent = title
|
||||
document.body.append(link)
|
||||
},
|
||||
{ href, title: target.title },
|
||||
)
|
||||
const metrics = await measureSessionSwitch(page, {
|
||||
destinationIDs: messages.map((message) => message.info.id),
|
||||
sourceIDs: fixture.messages[fixture.sourceID].map((message) => message.info.id),
|
||||
lastID,
|
||||
requiredPartID: lastPartID,
|
||||
requireBottomAnchor: false,
|
||||
href,
|
||||
switch: async () => {
|
||||
await page.locator("#parent-hydration-target").click()
|
||||
await expectSessionTitle(page, target.title)
|
||||
},
|
||||
}).finally(() => history?.resolve())
|
||||
expect(metrics.firstCorrectObservedMs).not.toBeNull()
|
||||
const requestCounts = {
|
||||
list: requests.filter((request) => request.type === "list").length,
|
||||
parent: requests.filter((request) => request.type === "parent").length,
|
||||
}
|
||||
if (mode === "candidate") {
|
||||
expect(requestCounts.parent).toBe(1)
|
||||
expect(historyGates).toBe(1)
|
||||
}
|
||||
return { metrics, requestCounts, historyGateCount: historyGates }
|
||||
}
|
||||
@@ -2,7 +2,13 @@ import type { Page } from "@playwright/test"
|
||||
import { expectSessionTitle } from "../../utils/waits"
|
||||
import { benchmark, expect, withBenchmarkPage } from "../benchmark"
|
||||
import { fixture } from "./session-timeline-stress.fixture"
|
||||
import { installStressSessionTabs, mockStressTimeline, stressSessionHref } from "./timeline-test-helpers"
|
||||
import {
|
||||
createReviewDiffs,
|
||||
installStressSessionTabs,
|
||||
installTimelineSettings,
|
||||
mockStressTimeline,
|
||||
stressSessionHref,
|
||||
} from "./timeline-test-helpers"
|
||||
import { measureSessionSwitch, waitForStableTimeline } from "./session-tab-switch-probe"
|
||||
|
||||
type Result = Awaited<ReturnType<typeof measureSessionSwitch>>
|
||||
@@ -20,8 +26,41 @@ benchmark("benchmarks cold and hot session tab switching", async ({ browser, rep
|
||||
report({ results, summary: summarize(results) })
|
||||
})
|
||||
|
||||
async function trial(page: Page, mode: "cold" | "hot") {
|
||||
await mockStressTimeline(page)
|
||||
benchmark(
|
||||
"benchmarks v2 session tab switching with and without the review pane",
|
||||
async ({ browser, report }, testInfo) => {
|
||||
benchmark.setTimeout(360_000)
|
||||
const runs = Number(process.env.SESSION_TAB_SWITCH_RUNS ?? 5)
|
||||
const results = {
|
||||
closed: { cold: [] as Result[], hot: [] as Result[] },
|
||||
open: { cold: [] as Result[], hot: [] as Result[] },
|
||||
}
|
||||
for (const reviewPane of ["closed", "open"] as const) {
|
||||
for (const mode of ["cold", "hot"] as const) {
|
||||
for (let run = 0; run < runs; run++) {
|
||||
results[reviewPane][mode].push(
|
||||
await withBenchmarkPage(
|
||||
browser,
|
||||
`session-tab-switch-v2-${reviewPane}-${mode}-${run}`,
|
||||
(page) => trial(page, mode, { newLayoutDesigns: true, reviewPane }),
|
||||
testInfo,
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
report({ results, summary: summarizeReviewPane(results) }, { runs, reviewDiffs: createReviewDiffs().length })
|
||||
},
|
||||
)
|
||||
|
||||
async function trial(
|
||||
page: Page,
|
||||
mode: "cold" | "hot",
|
||||
options?: { newLayoutDesigns?: boolean; reviewPane?: "closed" | "open" },
|
||||
) {
|
||||
const reviewDiffs = options?.newLayoutDesigns ? createReviewDiffs() : undefined
|
||||
await mockStressTimeline(page, { vcsDiff: reviewDiffs })
|
||||
if (options?.newLayoutDesigns) await installTimelineSettings(page)
|
||||
await installStressSessionTabs(page)
|
||||
if (mode === "hot") {
|
||||
await page.goto(stressSessionHref(fixture.targetID))
|
||||
@@ -33,6 +72,10 @@ async function trial(page: Page, mode: "cold" | "hot") {
|
||||
await expectSessionTitle(page, fixture.expected.sourceTitle)
|
||||
}
|
||||
await waitForStableTimeline(page, fixture.expected.sourceMessageIDs.at(-1)!)
|
||||
if (options?.reviewPane === "open") {
|
||||
await openReviewPane(page)
|
||||
await waitForStableTimeline(page, fixture.expected.sourceMessageIDs.at(-1)!)
|
||||
}
|
||||
|
||||
const destinationIDs = fixture.messages[fixture.targetID].map((message) => message.info.id)
|
||||
const sourceIDs = fixture.messages[fixture.sourceID].map((message) => message.info.id)
|
||||
@@ -70,6 +113,15 @@ function summarize(results: Record<"cold" | "hot", Result[]>) {
|
||||
)
|
||||
}
|
||||
|
||||
function summarizeReviewPane(results: Record<"closed" | "open", Record<"cold" | "hot", Result[]>>) {
|
||||
return Object.fromEntries(
|
||||
Object.entries(results).map(([reviewPane, values]) => [
|
||||
reviewPane,
|
||||
summarize(values as Record<"cold" | "hot", Result[]>),
|
||||
]),
|
||||
)
|
||||
}
|
||||
|
||||
async function switchSession(page: Page, sessionID: string, title: string) {
|
||||
const href = stressSessionHref(sessionID)
|
||||
const tab = page.locator(`[data-slot="titlebar-tabs"] a[href="${href}"]`).first()
|
||||
@@ -77,3 +129,16 @@ async function switchSession(page: Page, sessionID: string, title: string) {
|
||||
await tab.click()
|
||||
await expectSessionTitle(page, title)
|
||||
}
|
||||
|
||||
async function openReviewPane(page: Page) {
|
||||
await page.getByRole("button", { name: "Toggle review" }).click()
|
||||
const panel = page.locator("#review-panel")
|
||||
await expect(panel).toBeVisible()
|
||||
// Text-based readiness works across review implementations; the legacy list mounts
|
||||
// diff viewers lazily while V2 mounts the active preview eagerly.
|
||||
await page.waitForFunction(() => {
|
||||
const panel = document.querySelector<HTMLElement>("#review-panel")
|
||||
const text = panel?.textContent ?? ""
|
||||
return text.includes("generated-000.ts") && text.includes("+3")
|
||||
})
|
||||
}
|
||||
|
||||
@@ -4,7 +4,15 @@ export type SessionSwitchSample = {
|
||||
source: string[]
|
||||
hasVisibleRows: boolean
|
||||
last: boolean
|
||||
requiredPartVisible?: boolean
|
||||
bottomAnchorRequired?: boolean
|
||||
bottomErrorPx?: number
|
||||
review?: {
|
||||
fileHost: boolean
|
||||
fileHostReplaced: boolean
|
||||
header: string
|
||||
replacedLevels: string[]
|
||||
}
|
||||
}
|
||||
|
||||
export function classifySessionSwitch(samples: SessionSwitchSample[]) {
|
||||
@@ -23,6 +31,10 @@ export function classifySessionSwitch(samples: SessionSwitchSample[]) {
|
||||
(sample) => sample.hasVisibleRows && sample.destination.length === 0 && sample.source.length === 0,
|
||||
).length,
|
||||
sourceSamples: samples.filter((sample) => sample.source.length > 0).length,
|
||||
reviewFileHostMissingSamples: samples.filter((sample) => sample.review && !sample.review.fileHost).length,
|
||||
reviewFileHostReplacedSamples: samples.filter((sample) => sample.review?.fileHostReplaced).length,
|
||||
reviewHeaders: [...new Set(samples.flatMap((sample) => (sample.review ? [sample.review.header] : [])))],
|
||||
reviewReplacedLevels: [...new Set(samples.flatMap((sample) => sample.review?.replacedLevels ?? []))],
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,7 +43,8 @@ export function isCorrectDestination(sample: SessionSwitchSample) {
|
||||
sample.destination.length > 0 &&
|
||||
sample.source.length === 0 &&
|
||||
sample.last &&
|
||||
Math.abs(sample.bottomErrorPx ?? Infinity) <= 1
|
||||
sample.requiredPartVisible !== false &&
|
||||
(sample.bottomAnchorRequired === false || Math.abs(sample.bottomErrorPx ?? Infinity) <= 1)
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -8,19 +8,56 @@ type SessionSwitchProbe = {
|
||||
|
||||
async function installSessionSwitchProbe(
|
||||
page: Page,
|
||||
input: { destinationIDs: string[]; sourceIDs: string[]; lastID: string; href: string },
|
||||
input: {
|
||||
destinationIDs: string[]
|
||||
sourceIDs: string[]
|
||||
lastID: string
|
||||
requiredPartID?: string
|
||||
requireBottomAnchor?: boolean
|
||||
href: string
|
||||
},
|
||||
) {
|
||||
await page.evaluate(({ destinationIDs, sourceIDs, lastID, href }) => {
|
||||
await page.evaluate(({ destinationIDs, sourceIDs, lastID, requiredPartID, requireBottomAnchor, href }) => {
|
||||
const destination = new Set(destinationIDs)
|
||||
const source = new Set(sourceIDs)
|
||||
const samples: SessionSwitchSample[] = []
|
||||
let started: number | undefined
|
||||
let running = true
|
||||
const reviewLevels: Record<string, string> = {
|
||||
panel: "#review-panel",
|
||||
tabs: '#review-panel [data-component="tabs"]',
|
||||
body: '#review-panel [data-slot="session-review-v2-body"]',
|
||||
review: '#review-panel [data-component="session-review-v2"]',
|
||||
preview: '#review-panel [data-slot="session-review-v2-preview"]',
|
||||
scroll: '#review-panel [data-slot="session-review-v2-diff-scroll"]',
|
||||
file: '#review-panel [data-component="file"][data-mode="diff"]',
|
||||
}
|
||||
const initialReviewNodes: Record<string, Element | null> = {}
|
||||
const sample = () => {
|
||||
if (!running || started === undefined) return
|
||||
setTimeout(() => {
|
||||
if (!running || started === undefined) return
|
||||
const observedAtMs = performance.now() - started
|
||||
const reviewPanel = document.querySelector<HTMLElement>("#review-panel")
|
||||
const reviewFile = reviewPanel?.querySelector('[data-component="file"][data-mode="diff"]')
|
||||
const initialReviewFile = initialReviewNodes.file
|
||||
const replacedLevels = Object.entries(reviewLevels).flatMap(([name, selector]) => {
|
||||
const initial = initialReviewNodes[name]
|
||||
if (!initial) return []
|
||||
const current = document.querySelector(selector)
|
||||
return current && current !== initial ? [name] : []
|
||||
})
|
||||
const review = reviewPanel
|
||||
? {
|
||||
fileHost: !!reviewFile,
|
||||
fileHostReplaced: !!initialReviewFile && !!reviewFile && reviewFile !== initialReviewFile,
|
||||
header:
|
||||
reviewPanel
|
||||
.querySelector<HTMLElement>('[data-slot="session-review-v2-file-header"]')
|
||||
?.textContent?.trim() ?? "",
|
||||
replacedLevels,
|
||||
}
|
||||
: undefined
|
||||
const root = [...document.querySelectorAll<HTMLElement>(".scroll-view__viewport")].find((element) =>
|
||||
element.querySelector("[data-timeline-row]"),
|
||||
)
|
||||
@@ -36,6 +73,13 @@ async function installSessionSwitchProbe(
|
||||
const rect = element.getBoundingClientRect()
|
||||
return rect.bottom > view.top && rect.top < view.bottom
|
||||
})
|
||||
const requiredPartVisible = requiredPartID
|
||||
? [...root.querySelectorAll<HTMLElement>("[data-timeline-part-id]")].some((element) => {
|
||||
if (element.dataset.timelinePartId !== requiredPartID) return false
|
||||
const rect = element.getBoundingClientRect()
|
||||
return rect.width > 0 && rect.height > 0 && rect.bottom > view.top && rect.top < view.bottom
|
||||
})
|
||||
: undefined
|
||||
const spacer = root.querySelector<HTMLElement>('[data-timeline-row="bottom-spacer"]')?.getBoundingClientRect()
|
||||
samples.push({
|
||||
observedAtMs,
|
||||
@@ -43,10 +87,22 @@ async function installSessionSwitchProbe(
|
||||
source: visible.filter((id) => source.has(id)),
|
||||
hasVisibleRows,
|
||||
last: visible.includes(lastID),
|
||||
requiredPartVisible,
|
||||
bottomAnchorRequired: requireBottomAnchor !== false,
|
||||
bottomErrorPx: spacer ? spacer.bottom - view.bottom : undefined,
|
||||
review,
|
||||
})
|
||||
} else {
|
||||
samples.push({ observedAtMs, destination: [], source: [], hasVisibleRows: false, last: false })
|
||||
samples.push({
|
||||
observedAtMs,
|
||||
destination: [],
|
||||
source: [],
|
||||
hasVisibleRows: false,
|
||||
last: false,
|
||||
requiredPartVisible: requiredPartID ? false : undefined,
|
||||
bottomAnchorRequired: requireBottomAnchor !== false,
|
||||
review,
|
||||
})
|
||||
}
|
||||
requestAnimationFrame(sample)
|
||||
}, 0)
|
||||
@@ -57,6 +113,9 @@ async function installSessionSwitchProbe(
|
||||
const link = event.target instanceof Element ? event.target.closest("a") : undefined
|
||||
if (link?.getAttribute("href") !== href) return
|
||||
started = performance.now()
|
||||
for (const [name, selector] of Object.entries(reviewLevels)) {
|
||||
initialReviewNodes[name] = document.querySelector(selector)
|
||||
}
|
||||
requestAnimationFrame(sample)
|
||||
},
|
||||
{ capture: true, once: true },
|
||||
@@ -83,7 +142,8 @@ async function waitForStableSessionSwitch(page: Page) {
|
||||
sample.destination.length > 0 &&
|
||||
sample.source.length === 0 &&
|
||||
sample.last &&
|
||||
Math.abs(sample.bottomErrorPx ?? Infinity) <= 1,
|
||||
sample.requiredPartVisible !== false &&
|
||||
(sample.bottomAnchorRequired === false || Math.abs(sample.bottomErrorPx ?? Infinity) <= 1),
|
||||
)
|
||||
)
|
||||
})
|
||||
@@ -101,13 +161,27 @@ async function collectSessionSwitchResult(page: Page) {
|
||||
|
||||
export async function measureSessionSwitch(
|
||||
page: Page,
|
||||
input: { destinationIDs: string[]; sourceIDs: string[]; lastID: string; href: string; switch: () => Promise<void> },
|
||||
input: {
|
||||
destinationIDs: string[]
|
||||
sourceIDs: string[]
|
||||
lastID: string
|
||||
requiredPartID?: string
|
||||
requireBottomAnchor?: boolean
|
||||
href: string
|
||||
switch: () => Promise<void>
|
||||
},
|
||||
) {
|
||||
const { switch: run, ...probe } = input
|
||||
await installSessionSwitchProbe(page, probe)
|
||||
await run()
|
||||
await waitForStableSessionSwitch(page)
|
||||
return collectSessionSwitchResult(page)
|
||||
try {
|
||||
await run()
|
||||
await waitForStableSessionSwitch(page)
|
||||
return await collectSessionSwitchResult(page)
|
||||
} finally {
|
||||
await page.evaluate(() => {
|
||||
;(window as Window & { __sessionSwitchProbe?: SessionSwitchProbe }).__sessionSwitchProbe?.stop()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export async function waitForStableTimeline(page: Page, lastID: string) {
|
||||
|
||||
@@ -93,36 +93,53 @@ const assistantMessage = {
|
||||
parts: [editPart],
|
||||
}
|
||||
|
||||
export async function setupTimelineBenchmark(page: Page, options: { historyTurns: number; eventBatch: number }) {
|
||||
export async function setupTimelineBenchmark(
|
||||
page: Page,
|
||||
options: {
|
||||
historyTurns: number
|
||||
eventBatch: number
|
||||
newLayoutDesigns?: boolean
|
||||
vcsDiff?: unknown[]
|
||||
turnDiffs?: unknown[]
|
||||
},
|
||||
) {
|
||||
const events: EventPayload[] = []
|
||||
let eventBatch = options.eventBatch
|
||||
const currentUserMessage = options.turnDiffs
|
||||
? { ...userMessage, info: { ...userMessage.info, summary: { diffs: options.turnDiffs } } }
|
||||
: userMessage
|
||||
await mockOpenCodeServer(page, {
|
||||
directory,
|
||||
project: project(),
|
||||
provider: provider(),
|
||||
sessions: [session()],
|
||||
vcsDiff: options.vcsDiff,
|
||||
pageMessages: () => ({
|
||||
items: [
|
||||
...Array.from({ length: options.historyTurns }, (_, index) => performanceTurn(index)).flat(),
|
||||
userMessage,
|
||||
currentUserMessage,
|
||||
assistantMessage,
|
||||
],
|
||||
}),
|
||||
events: () => events.splice(0, eventBatch),
|
||||
eventRetry: 16,
|
||||
})
|
||||
await page.addInitScript(() => {
|
||||
localStorage.setItem(
|
||||
"settings.v3",
|
||||
JSON.stringify({
|
||||
general: {
|
||||
editToolPartsExpanded: true,
|
||||
shellToolPartsExpanded: true,
|
||||
showReasoningSummaries: true,
|
||||
},
|
||||
}),
|
||||
)
|
||||
})
|
||||
await page.addInitScript(
|
||||
(input) => {
|
||||
localStorage.setItem(
|
||||
"settings.v3",
|
||||
JSON.stringify({
|
||||
general: {
|
||||
newLayoutDesigns: input.newLayoutDesigns,
|
||||
editToolPartsExpanded: true,
|
||||
shellToolPartsExpanded: true,
|
||||
showReasoningSummaries: true,
|
||||
},
|
||||
}),
|
||||
)
|
||||
},
|
||||
{ newLayoutDesigns: options.newLayoutDesigns ?? false },
|
||||
)
|
||||
await page.setViewportSize({ width: 1366, height: 768 })
|
||||
const scroller = page.locator(".scroll-view__viewport", { has: page.locator("[data-timeline-row]") })
|
||||
const text = page.locator(`[data-timeline-part-id="${textPartID}"]`).first()
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { Page } from "@playwright/test"
|
||||
import { benchmark, benchmarkDiagnostics, expect } from "../benchmark"
|
||||
import {
|
||||
buildInitialStreamEvent,
|
||||
@@ -6,80 +7,300 @@ import {
|
||||
textPartID,
|
||||
} from "./session-timeline-benchmark.fixture"
|
||||
import { startTimelineProfile } from "./session-timeline-profile"
|
||||
import { createReviewDiffs } from "./timeline-test-helpers"
|
||||
import {
|
||||
collectTimelineStreamMetrics,
|
||||
installTimelineStreamProbe,
|
||||
startTimelineStreamProbe,
|
||||
} from "./session-timeline-stream-probe"
|
||||
|
||||
type TimelineStreamOptions = {
|
||||
newLayoutDesigns?: boolean
|
||||
reviewDiffs?: boolean
|
||||
reviewPane?: boolean
|
||||
}
|
||||
|
||||
type ReviewPaneSample = {
|
||||
observedAtMs: number
|
||||
panelVisible: boolean
|
||||
header: string
|
||||
diffViewers: number
|
||||
diffLines: number
|
||||
codeBlocks: number
|
||||
ready: boolean
|
||||
}
|
||||
|
||||
type ReviewPaneProbe = {
|
||||
samples: ReviewPaneSample[]
|
||||
start: () => void
|
||||
stop: () => void
|
||||
}
|
||||
|
||||
const reviewReadyStreak = 3
|
||||
|
||||
benchmark.describe("performance: session timeline streaming", () => {
|
||||
benchmark("streams assistant text without remounting or oscillating", async ({ page, report }) => {
|
||||
benchmark.setTimeout(480_000)
|
||||
const cpuThrottle = Number(process.env.TIMELINE_CPU_THROTTLE ?? 30)
|
||||
const deltaCount = Number(process.env.TIMELINE_DELTA_COUNT ?? 160)
|
||||
const historyTurns = Number(process.env.TIMELINE_HISTORY_TURNS ?? 320)
|
||||
const eventBatch = Number(process.env.TIMELINE_EVENT_BATCH ?? 1)
|
||||
const minimal = process.env.TIMELINE_MINIMAL === "1"
|
||||
const profileCPU = process.env.TIMELINE_CPU_PROFILE === "1"
|
||||
const profileVisual = !minimal && profileCPU && process.env.TIMELINE_VISUAL_PROFILE !== "0"
|
||||
benchmark.setTimeout(Number(process.env.TIMELINE_COMPLETION_TIMEOUT_MS ?? 420_000) + 60_000)
|
||||
const result = await runTimelineStreamBenchmark(page, {})
|
||||
report(result.metrics, result.context)
|
||||
})
|
||||
|
||||
benchmark("streams assistant text in v2 with review pane closed", async ({ page, report }) => {
|
||||
benchmark.setTimeout(Number(process.env.TIMELINE_COMPLETION_TIMEOUT_MS ?? 420_000) + 60_000)
|
||||
const result = await runTimelineStreamBenchmark(page, { newLayoutDesigns: true })
|
||||
report(result.metrics, result.context)
|
||||
})
|
||||
|
||||
benchmark("streams assistant text in v2 with review diffs and pane closed", async ({ page, report }) => {
|
||||
benchmark.setTimeout(Number(process.env.TIMELINE_COMPLETION_TIMEOUT_MS ?? 420_000) + 60_000)
|
||||
const result = await runTimelineStreamBenchmark(page, { newLayoutDesigns: true, reviewDiffs: true })
|
||||
report(result.metrics, result.context)
|
||||
})
|
||||
|
||||
benchmark("streams assistant text in v2 with review pane open", async ({ page, report }) => {
|
||||
benchmark.setTimeout(Number(process.env.TIMELINE_COMPLETION_TIMEOUT_MS ?? 420_000) + 60_000)
|
||||
const result = await runTimelineStreamBenchmark(page, { newLayoutDesigns: true, reviewPane: true })
|
||||
report(result.metrics, result.context)
|
||||
})
|
||||
})
|
||||
|
||||
benchmark.describe("performance: review pane", () => {
|
||||
benchmark("loads v2 review diffs and switches active files", async ({ page, report }) => {
|
||||
benchmark.setTimeout(240_000)
|
||||
const historyTurns = Number(process.env.REVIEW_PANE_HISTORY_TURNS ?? 72)
|
||||
const diffs = createReviewDiffs()
|
||||
const fixture = await setupTimelineBenchmark(page, {
|
||||
historyTurns,
|
||||
eventBatch,
|
||||
eventBatch: 1,
|
||||
newLayoutDesigns: true,
|
||||
vcsDiff: diffs,
|
||||
})
|
||||
|
||||
fixture.transport.enqueue(buildInitialStreamEvent(deltaCount))
|
||||
const contentStart = performance.now()
|
||||
fixture.transport.enqueue(buildInitialStreamEvent(1))
|
||||
await expect(fixture.text).toBeVisible()
|
||||
await expect(fixture.text).toContainText("Implementation plan")
|
||||
const initialContentObservedMs = performance.now() - contentStart
|
||||
await fixture.scrollToBottom()
|
||||
await fixture.waitForStableGeometry()
|
||||
|
||||
const profile = await startTimelineProfile(page, { cpuThrottle, profileCPU })
|
||||
await installTimelineStreamProbe(page, { textPartID, finalIndex: deltaCount, profileVisual, minimal })
|
||||
const deltas = buildStreamDeltaEvents(deltaCount)
|
||||
await startTimelineStreamProbe(page)
|
||||
fixture.transport.enqueue(deltas)
|
||||
|
||||
await page.waitForFunction(
|
||||
(finalIndex) =>
|
||||
(
|
||||
window as Window & {
|
||||
__timelineStreamBenchmark?: { applied: { index: number }[] }
|
||||
}
|
||||
).__timelineStreamBenchmark?.applied.some((value) => value.index === finalIndex),
|
||||
deltaCount,
|
||||
{ timeout: 420_000 },
|
||||
)
|
||||
await expect(fixture.text).toContainText("benchmark-complete")
|
||||
await expect(fixture.text).toContainText("Streaming")
|
||||
await fixture.waitForStableGeometry()
|
||||
const metrics = await collectTimelineStreamMetrics(page, {
|
||||
textPartID,
|
||||
finalIndex: deltaCount,
|
||||
navigations: benchmarkDiagnostics(page).navigations,
|
||||
})
|
||||
const delivered = deltas.length - fixture.transport.pendingCount()
|
||||
await profile.stop()
|
||||
const open = await measureReviewPaneLoad(page, diffs[0]!.file)
|
||||
const switches = []
|
||||
for (const diff of diffs.slice(1, 4)) switches.push(await measureReviewNextFile(page, diff.file))
|
||||
|
||||
report(
|
||||
{
|
||||
endToEndInitialContentObservedMs: initialContentObservedMs,
|
||||
...metrics,
|
||||
deliveredDeltas: delivered,
|
||||
pendingDeltas: fixture.transport.pendingCount(),
|
||||
open,
|
||||
switches,
|
||||
},
|
||||
{
|
||||
cpuThrottle,
|
||||
profileCPU,
|
||||
profileVisual,
|
||||
minimal,
|
||||
queuedDeltas: deltas.length,
|
||||
historyTurns,
|
||||
eventBatch,
|
||||
reviewDiffs: diffs.length,
|
||||
},
|
||||
)
|
||||
|
||||
await profile.reset()
|
||||
})
|
||||
})
|
||||
|
||||
async function runTimelineStreamBenchmark(page: Page, options: TimelineStreamOptions) {
|
||||
const completionTimeoutMs = Number(process.env.TIMELINE_COMPLETION_TIMEOUT_MS ?? 420_000)
|
||||
const cpuThrottle = Number(process.env.TIMELINE_CPU_THROTTLE ?? 30)
|
||||
const deltaCount = Number(process.env.TIMELINE_DELTA_COUNT ?? 160)
|
||||
const historyTurns = Number(process.env.TIMELINE_HISTORY_TURNS ?? 320)
|
||||
const eventBatch = Number(process.env.TIMELINE_EVENT_BATCH ?? 1)
|
||||
const minimal = process.env.TIMELINE_MINIMAL === "1"
|
||||
const profileCPU = process.env.TIMELINE_CPU_PROFILE === "1"
|
||||
const profileVisual = !minimal && profileCPU && process.env.TIMELINE_VISUAL_PROFILE !== "0"
|
||||
const diffs = options.reviewDiffs || options.reviewPane ? createReviewDiffs() : undefined
|
||||
const fixture = await setupTimelineBenchmark(page, {
|
||||
historyTurns,
|
||||
eventBatch,
|
||||
newLayoutDesigns: options.newLayoutDesigns,
|
||||
// Turn diffs exercise timeline data cost; the pane-open scenario serves the same
|
||||
// diffs through the default git mode so it works across review implementations.
|
||||
turnDiffs: options.reviewDiffs ? diffs : undefined,
|
||||
vcsDiff: options.reviewPane ? diffs : undefined,
|
||||
})
|
||||
|
||||
fixture.transport.enqueue(buildInitialStreamEvent(deltaCount))
|
||||
const contentStart = performance.now()
|
||||
await expect(fixture.text).toBeVisible()
|
||||
await expect(fixture.text).toContainText("Implementation plan")
|
||||
const initialContentObservedMs = performance.now() - contentStart
|
||||
await fixture.scrollToBottom()
|
||||
await fixture.waitForStableGeometry()
|
||||
|
||||
const reviewPane = options.reviewPane && diffs ? await measureReviewPaneLoad(page, diffs[0]!.file) : undefined
|
||||
if (reviewPane) await fixture.waitForStableGeometry()
|
||||
|
||||
const profile = await startTimelineProfile(page, { cpuThrottle, profileCPU })
|
||||
await installTimelineStreamProbe(page, { textPartID, finalIndex: deltaCount, profileVisual, minimal })
|
||||
const deltas = buildStreamDeltaEvents(deltaCount)
|
||||
await startTimelineStreamProbe(page)
|
||||
fixture.transport.enqueue(deltas)
|
||||
|
||||
await page.waitForFunction(
|
||||
(finalIndex) =>
|
||||
(
|
||||
window as Window & {
|
||||
__timelineStreamBenchmark?: { applied: { index: number }[] }
|
||||
}
|
||||
).__timelineStreamBenchmark?.applied.some((value) => value.index === finalIndex),
|
||||
deltaCount,
|
||||
{ timeout: completionTimeoutMs },
|
||||
)
|
||||
await expect(fixture.text).toContainText("benchmark-complete")
|
||||
await expect(fixture.text).toContainText("Streaming")
|
||||
await fixture.waitForStableGeometry()
|
||||
const metrics = await collectTimelineStreamMetrics(page, {
|
||||
textPartID,
|
||||
finalIndex: deltaCount,
|
||||
navigations: benchmarkDiagnostics(page).navigations,
|
||||
})
|
||||
const delivered = deltas.length - fixture.transport.pendingCount()
|
||||
await profile.stop()
|
||||
|
||||
const result = {
|
||||
metrics: {
|
||||
endToEndInitialContentObservedMs: initialContentObservedMs,
|
||||
...metrics,
|
||||
deliveredDeltas: delivered,
|
||||
pendingDeltas: fixture.transport.pendingCount(),
|
||||
reviewPane: reviewPane ?? null,
|
||||
},
|
||||
context: {
|
||||
cpuThrottle,
|
||||
profileCPU,
|
||||
profileVisual,
|
||||
minimal,
|
||||
queuedDeltas: deltas.length,
|
||||
historyTurns,
|
||||
eventBatch,
|
||||
newLayoutDesigns: options.newLayoutDesigns === true,
|
||||
reviewPane: options.reviewPane === true ? "open" : "closed",
|
||||
reviewDiffs: diffs?.length ?? 0,
|
||||
},
|
||||
}
|
||||
|
||||
await profile.reset()
|
||||
return result
|
||||
}
|
||||
|
||||
async function measureReviewPaneLoad(page: Page, file: string) {
|
||||
// Default git mode reads the mocked /vcs/diff data, so opening the pane is enough
|
||||
// and the flow works across review pane implementations.
|
||||
await installReviewPaneProbe(page, { file })
|
||||
await startReviewPaneProbe(page)
|
||||
await page.getByRole("button", { name: "Toggle review" }).click()
|
||||
await expect(page.locator("#review-panel")).toBeVisible()
|
||||
return collectReviewPaneProbe(page)
|
||||
}
|
||||
|
||||
async function measureReviewNextFile(page: Page, file: string) {
|
||||
await installReviewPaneProbe(page, { file })
|
||||
await startReviewPaneProbe(page)
|
||||
await page.getByRole("button", { name: "Next file" }).click()
|
||||
return collectReviewPaneProbe(page)
|
||||
}
|
||||
|
||||
async function installReviewPaneProbe(page: Page, input: { file: string }) {
|
||||
await page.evaluate((input) => {
|
||||
const samples: ReviewPaneSample[] = []
|
||||
const basename = input.file.split(/[\\/]/).at(-1) ?? input.file
|
||||
let started: number | undefined
|
||||
let running = true
|
||||
|
||||
const paneState = () => {
|
||||
const panel = document.querySelector<HTMLElement>("#review-panel")
|
||||
const review = panel?.querySelector<HTMLElement>('[data-component="session-review-v2"]')
|
||||
const rect = (review ?? panel)?.getBoundingClientRect()
|
||||
const text = panel?.textContent ?? ""
|
||||
const previewHeader = panel?.querySelector<HTMLElement>(
|
||||
'[data-slot="session-review-v2-file-header"]',
|
||||
)?.textContent
|
||||
const header = previewHeader ?? text
|
||||
const viewers = panel ? [...panel.querySelectorAll<HTMLElement>('[data-component="file"][data-mode="diff"]')] : []
|
||||
const codeBlocks = panel?.querySelectorAll("code").length ?? 0
|
||||
const diffLines = viewers.reduce(
|
||||
(sum, viewer) =>
|
||||
sum +
|
||||
(viewer.shadowRoot?.querySelectorAll("[data-line]").length ?? viewer.querySelectorAll("[data-line]").length),
|
||||
0,
|
||||
)
|
||||
const panelVisible =
|
||||
!!panel && panel.getAttribute("aria-hidden") !== "true" && !!rect && rect.width > 0 && rect.height > 0
|
||||
return {
|
||||
panelVisible,
|
||||
header: header.slice(0, 500),
|
||||
diffViewers: viewers.length,
|
||||
diffLines,
|
||||
codeBlocks,
|
||||
ready:
|
||||
panelVisible &&
|
||||
header.includes(basename) &&
|
||||
(viewers.length > 0 || text.includes("+3") || diffLines > 0 || codeBlocks > 0),
|
||||
}
|
||||
}
|
||||
|
||||
const sample = () => {
|
||||
if (!running || started === undefined) return
|
||||
requestAnimationFrame(() => {
|
||||
setTimeout(() => {
|
||||
if (!running || started === undefined) return
|
||||
samples.push({ observedAtMs: performance.now() - started, ...paneState() })
|
||||
if (performance.now() - started < 10_000) sample()
|
||||
}, 0)
|
||||
})
|
||||
}
|
||||
|
||||
;(window as Window & { __reviewPaneProbe?: ReviewPaneProbe }).__reviewPaneProbe = {
|
||||
samples,
|
||||
start: () => {
|
||||
started = performance.now()
|
||||
performance.mark("opencode.review-pane.click")
|
||||
sample()
|
||||
},
|
||||
stop: () => {
|
||||
running = false
|
||||
},
|
||||
}
|
||||
}, input)
|
||||
}
|
||||
|
||||
async function startReviewPaneProbe(page: Page) {
|
||||
await page.evaluate(() => {
|
||||
;(window as Window & { __reviewPaneProbe?: ReviewPaneProbe }).__reviewPaneProbe!.start()
|
||||
})
|
||||
}
|
||||
|
||||
async function collectReviewPaneProbe(page: Page) {
|
||||
await page.waitForFunction((streak) => {
|
||||
const samples = (window as Window & { __reviewPaneProbe?: ReviewPaneProbe }).__reviewPaneProbe?.samples
|
||||
if (!samples) return false
|
||||
return samples.some((_, index) => {
|
||||
const stable = samples.slice(index, index + streak)
|
||||
return stable.length === streak && stable.every((sample) => sample.ready)
|
||||
})
|
||||
}, reviewReadyStreak)
|
||||
|
||||
const samples = await page.evaluate(() => {
|
||||
const probe = (window as Window & { __reviewPaneProbe?: ReviewPaneProbe }).__reviewPaneProbe!
|
||||
probe.stop()
|
||||
return probe.samples
|
||||
})
|
||||
return { summary: summarizeReviewPaneSamples(samples), samples }
|
||||
}
|
||||
|
||||
function summarizeReviewPaneSamples(samples: ReviewPaneSample[]) {
|
||||
const firstReady = samples.find((sample) => sample.ready)
|
||||
const stableIndex = samples.findIndex((_, index) => {
|
||||
const stable = samples.slice(index, index + reviewReadyStreak)
|
||||
return stable.length === reviewReadyStreak && stable.every((sample) => sample.ready)
|
||||
})
|
||||
return {
|
||||
samples: samples.length,
|
||||
firstReadyObservedMs: firstReady?.observedAtMs ?? null,
|
||||
stableReadyObservedMs: stableIndex === -1 ? null : samples[stableIndex + reviewReadyStreak - 1]!.observedAtMs,
|
||||
notReadySamples: samples.filter((sample) => !sample.ready).length,
|
||||
maxDiffViewers: Math.max(0, ...samples.map((sample) => sample.diffViewers)),
|
||||
maxDiffLines: Math.max(0, ...samples.map((sample) => sample.diffLines)),
|
||||
maxCodeBlocks: Math.max(0, ...samples.map((sample) => sample.codeBlocks)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,7 +21,10 @@ export async function installTimelineSettings(page: Page) {
|
||||
|
||||
export function mockStressTimeline(
|
||||
page: Page,
|
||||
input?: { onMessages?: (input: { sessionID: string; before?: string; phase: "start" | "end" }) => void },
|
||||
input?: {
|
||||
onMessages?: (input: { sessionID: string; before?: string; phase: "start" | "end" }) => void
|
||||
vcsDiff?: unknown[]
|
||||
},
|
||||
) {
|
||||
return mockOpenCodeServer(page, {
|
||||
sessions: fixture.sessions,
|
||||
@@ -30,6 +33,7 @@ export function mockStressTimeline(
|
||||
project: fixture.project,
|
||||
pageMessages,
|
||||
onMessages: input?.onMessages,
|
||||
vcsDiff: input?.vcsDiff,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -78,3 +82,53 @@ export function stressDraftHref(draftID: string) {
|
||||
function stressServer() {
|
||||
return `http://${process.env.PLAYWRIGHT_SERVER_HOST ?? "127.0.0.1"}:${process.env.PLAYWRIGHT_SERVER_PORT ?? "4096"}`
|
||||
}
|
||||
|
||||
export function createReviewDiffs() {
|
||||
return Array.from({ length: Number(process.env.REVIEW_PANE_DIFF_COUNT ?? 72) }, (_, index) => {
|
||||
const lines = index % 3 === 0 ? 300 : index % 3 === 1 ? 120 : 38
|
||||
const file = `src/review/generated-${String(index).padStart(3, "0")}.ts`
|
||||
const before = reviewSource(index, lines)
|
||||
const after = before
|
||||
.replace(`value_${index}_4`, `updated_${index}_4`)
|
||||
.replace(
|
||||
`value_${index}_${Math.max(8, Math.floor(lines / 2))}`,
|
||||
`updated_${index}_${Math.max(8, Math.floor(lines / 2))}`,
|
||||
)
|
||||
.replace(`value_${index}_${lines - 4}`, `updated_${index}_${lines - 4}`)
|
||||
return {
|
||||
file,
|
||||
patch: reviewPatch(file, before, after),
|
||||
additions: 3,
|
||||
deletions: 3,
|
||||
status: "modified" as const,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function reviewSource(seed: number, lines: number) {
|
||||
return Array.from(
|
||||
{ length: lines },
|
||||
(_, index) => `export const value_${seed}_${index} = "${reviewWords(seed + index, index % 5 === 0 ? 180 : 42)}"`,
|
||||
).join("\n")
|
||||
}
|
||||
|
||||
function reviewPatch(file: string, before: string, after: string) {
|
||||
const beforeLines = before.split("\n")
|
||||
const afterLines = after.split("\n")
|
||||
return [
|
||||
`diff --git a/${file} b/${file}`,
|
||||
`--- a/${file}`,
|
||||
`+++ b/${file}`,
|
||||
`@@ -1,${beforeLines.length} +1,${afterLines.length} @@`,
|
||||
...beforeLines.flatMap((line, index) => {
|
||||
const next = afterLines[index]!
|
||||
if (line === next) return [` ${line}`]
|
||||
return [`-${line}`, `+${next}`]
|
||||
}),
|
||||
].join("\n")
|
||||
}
|
||||
|
||||
function reviewWords(seed: number, length: number) {
|
||||
const words = ["alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel", "india", "juliet"]
|
||||
return Array.from({ length: Math.ceil(length / 7) }, (_, index) => words[(seed + index * 3) % words.length]).join(" ")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import { expect, test } from "bun:test"
|
||||
import type { Page, Route } from "@playwright/test"
|
||||
import { mockOpenCodeServer } from "../../utils/mock-server"
|
||||
|
||||
test("applies message latency after a list response gate is released", async () => {
|
||||
const events: string[] = []
|
||||
const gate = Promise.withResolvers<void>()
|
||||
let handler: ((route: Route) => Promise<void>) | undefined
|
||||
const page = {
|
||||
route: (_url: string, callback: (route: Route) => Promise<void>) => {
|
||||
handler = callback
|
||||
return Promise.resolve()
|
||||
},
|
||||
} as unknown as Page
|
||||
await mockOpenCodeServer(page, {
|
||||
provider: {},
|
||||
directory: "C:/OpenCode",
|
||||
project: {},
|
||||
sessions: [{ id: "session" }],
|
||||
messageDelay: 25,
|
||||
beforeMessagesResponse: () => {
|
||||
events.push("before")
|
||||
return gate.promise
|
||||
},
|
||||
onMessages: (request) => events.push(request.phase),
|
||||
pageMessages: () => {
|
||||
events.push("page")
|
||||
return { items: [] }
|
||||
},
|
||||
})
|
||||
|
||||
const response = handler!({
|
||||
request: () => ({ url: () => "http://127.0.0.1:4096/session/session/message" }),
|
||||
fulfill: () => {
|
||||
events.push("fulfill")
|
||||
return Promise.resolve()
|
||||
},
|
||||
} as unknown as Route)
|
||||
expect(events).toEqual(["start", "before"])
|
||||
|
||||
const released = performance.now()
|
||||
gate.resolve()
|
||||
await response
|
||||
expect(performance.now() - released).toBeGreaterThanOrEqual(20)
|
||||
expect(events).toEqual(["start", "before", "page", "end", "fulfill"])
|
||||
})
|
||||
@@ -52,3 +52,35 @@ test("reports missing correctness without throwing", () => {
|
||||
expect(result.firstCorrectObservedMs).toBeNull()
|
||||
expect(result.stableObservedMs).toBeNull()
|
||||
})
|
||||
|
||||
test("requires an explicitly tracked part to be visible", () => {
|
||||
const result = classifySessionSwitch([
|
||||
{
|
||||
observedAtMs: 16,
|
||||
destination: ["destination"],
|
||||
source: [],
|
||||
hasVisibleRows: true,
|
||||
last: true,
|
||||
requiredPartVisible: false,
|
||||
bottomErrorPx: 0,
|
||||
},
|
||||
])
|
||||
|
||||
expect(result.firstCorrectObservedMs).toBeNull()
|
||||
})
|
||||
|
||||
test("can measure content correctness without requiring a bottom anchor", () => {
|
||||
const result = classifySessionSwitch([
|
||||
{
|
||||
observedAtMs: 16,
|
||||
destination: ["destination"],
|
||||
source: [],
|
||||
hasVisibleRows: true,
|
||||
last: true,
|
||||
requiredPartVisible: true,
|
||||
bottomAnchorRequired: false,
|
||||
},
|
||||
])
|
||||
|
||||
expect(result.firstCorrectObservedMs).toBe(16)
|
||||
})
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import { expect, test } from "bun:test"
|
||||
import type { Page } from "@playwright/test"
|
||||
import { measureSessionSwitch } from "../timeline/session-tab-switch-probe"
|
||||
|
||||
function testPage(waitFailure?: Error) {
|
||||
const stops: unknown[] = []
|
||||
const page = {
|
||||
evaluate: async (_callback: unknown, input?: unknown) => {
|
||||
if (input) return
|
||||
stops.push(undefined)
|
||||
},
|
||||
waitForFunction: async () => {
|
||||
if (waitFailure) throw waitFailure
|
||||
},
|
||||
} as unknown as Page
|
||||
return { page, stops }
|
||||
}
|
||||
|
||||
function input(run: () => Promise<void>) {
|
||||
return {
|
||||
destinationIDs: ["destination"],
|
||||
sourceIDs: ["source"],
|
||||
lastID: "destination",
|
||||
href: "/session/destination",
|
||||
switch: run,
|
||||
}
|
||||
}
|
||||
|
||||
test("stops sampling when the session switch fails", async () => {
|
||||
const failure = new Error("switch failed")
|
||||
const context = testPage()
|
||||
|
||||
await expect(
|
||||
measureSessionSwitch(
|
||||
context.page,
|
||||
input(async () => Promise.reject(failure)),
|
||||
),
|
||||
).rejects.toBe(failure)
|
||||
|
||||
expect(context.stops).toHaveLength(1)
|
||||
})
|
||||
|
||||
test("stops sampling when the stable wait fails", async () => {
|
||||
const failure = new Error("stable wait failed")
|
||||
const context = testPage(failure)
|
||||
|
||||
await expect(
|
||||
measureSessionSwitch(
|
||||
context.page,
|
||||
input(async () => {}),
|
||||
),
|
||||
).rejects.toBe(failure)
|
||||
|
||||
expect(context.stops).toHaveLength(1)
|
||||
})
|
||||
@@ -0,0 +1,392 @@
|
||||
import { expect, test } from "bun:test"
|
||||
import {
|
||||
analyzeVisualStability,
|
||||
analyzeVisualStabilityByMarker,
|
||||
type VisualStabilityTrace,
|
||||
} from "../../utils/visual-stability"
|
||||
import { analyzeVisualObservations } from "../../utils/visual-stability/analyzer"
|
||||
import { legacyVisualPlan, visualPlan, type VisualInvariant } from "../../utils/visual-stability/invariant"
|
||||
import { defineVisualRegions, mapVisualRegions } from "../../utils/visual-stability/regions"
|
||||
|
||||
function trace(samples: VisualStabilityTrace["samples"]): VisualStabilityTrace {
|
||||
return { markers: [], samples }
|
||||
}
|
||||
|
||||
test("accepts continuous visible motion", () => {
|
||||
expect(
|
||||
analyzeVisualStability(
|
||||
trace([
|
||||
frame(0, region({ width: 80, bottom: 40 }), region({ top: 40, bottom: 60 })),
|
||||
frame(16, region({ width: 75, bottom: 45 }), region({ top: 45, bottom: 65 })),
|
||||
frame(32, region({ width: 70, bottom: 50 }), region({ top: 50, bottom: 70 })),
|
||||
]),
|
||||
{ flow: ["changing", "following"] },
|
||||
),
|
||||
).toEqual([])
|
||||
})
|
||||
|
||||
test("reports repeated geometry reversals", () => {
|
||||
const issues = analyzeVisualStability(
|
||||
trace([
|
||||
frame(0, region({ width: 80 })),
|
||||
frame(16, region({ width: 60 })),
|
||||
frame(32, region({ width: 78 })),
|
||||
frame(48, region({ width: 62 })),
|
||||
]),
|
||||
)
|
||||
|
||||
expect(issues.some((issue) => issue.includes("changing width reversed 2 times"))).toBe(true)
|
||||
})
|
||||
|
||||
test("reports visible blanking, label reversal, and overlap", () => {
|
||||
const issues = analyzeVisualStability(
|
||||
trace([
|
||||
frame(0, region({ label: "Exploring", opacity: 1, bottom: 40 }), region({ top: 40, bottom: 60 })),
|
||||
frame(16, region({ label: "Explored", opacity: 0.2, bottom: 50 }), region({ top: 49, bottom: 69 })),
|
||||
frame(32, region({ label: "Exploring", opacity: 1, bottom: 50 }), region({ top: 50, bottom: 70 })),
|
||||
]),
|
||||
{ flow: ["changing", "following"] },
|
||||
)
|
||||
|
||||
expect(issues.some((issue) => issue.includes("opacity fell to 0.2"))).toBe(true)
|
||||
expect(issues.some((issue) => issue.includes("label reverted"))).toBe(true)
|
||||
expect(issues.some((issue) => issue.includes("overlapped following by 1px"))).toBe(true)
|
||||
})
|
||||
|
||||
test("reports duplicate regions and unexpected remounts", () => {
|
||||
const issues = analyzeVisualStability(
|
||||
trace([frame(0, region({ node: 1 })), frame(16, region({ node: 2, count: 2 })), frame(32, region({ node: 2 }))]),
|
||||
{ stable: ["changing"], unique: ["changing"] },
|
||||
)
|
||||
|
||||
expect(issues.some((issue) => issue.includes("changing appeared 2 times"))).toBe(true)
|
||||
expect(issues.some((issue) => issue.includes("changing remounted"))).toBe(true)
|
||||
})
|
||||
|
||||
test("reports bottom anchor loss but permits movement while scrolled away", () => {
|
||||
const anchored = analyzeVisualStability(
|
||||
trace([
|
||||
{ at: 0, regions: { changing: region() }, viewport: viewport(0) },
|
||||
{ at: 16, regions: { changing: region() }, viewport: viewport(24) },
|
||||
]),
|
||||
{ preserveBottomAnchor: true },
|
||||
)
|
||||
const away = analyzeVisualStability(
|
||||
trace([
|
||||
{ at: 0, regions: { changing: region() }, viewport: viewport(80) },
|
||||
{ at: 16, regions: { changing: region() }, viewport: viewport(104) },
|
||||
]),
|
||||
{ preserveBottomAnchor: true },
|
||||
)
|
||||
|
||||
expect(anchored.some((issue) => issue.includes("bottom anchor moved to 24px"))).toBe(true)
|
||||
expect(away).toEqual([])
|
||||
})
|
||||
|
||||
test("reports up down up movement while preserving a bottom anchor", () => {
|
||||
const issues = analyzeVisualStability(
|
||||
trace([
|
||||
{ at: 0, regions: { changing: region({ top: 200, bottom: 240 }) }, viewport: viewport(0) },
|
||||
{ at: 16, regions: { changing: region({ top: 180, bottom: 220 }) }, viewport: viewport(0) },
|
||||
{ at: 32, regions: { changing: region({ top: 196, bottom: 236 }) }, viewport: viewport(0) },
|
||||
{ at: 48, regions: { changing: region({ top: 176, bottom: 216 }) }, viewport: viewport(0) },
|
||||
]),
|
||||
{ preserveBottomAnchor: true, maxPositionReversals: 0 },
|
||||
)
|
||||
|
||||
expect(issues.some((issue) => issue.includes("changing top reversed 2 times"))).toBe(true)
|
||||
expect(issues.some((issue) => issue.includes("changing bottom reversed 2 times"))).toBe(true)
|
||||
})
|
||||
|
||||
test("accepts monotonic upward movement while preserving a bottom anchor", () => {
|
||||
expect(
|
||||
analyzeVisualStability(
|
||||
trace([
|
||||
{ at: 0, regions: { changing: region({ top: 200, bottom: 240 }) }, viewport: viewport(0) },
|
||||
{ at: 16, regions: { changing: region({ top: 190, bottom: 230 }) }, viewport: viewport(0) },
|
||||
{ at: 32, regions: { changing: region({ top: 180, bottom: 220 }) }, viewport: viewport(0) },
|
||||
]),
|
||||
{ preserveBottomAnchor: true, maxPositionReversals: 0 },
|
||||
),
|
||||
).toEqual([])
|
||||
})
|
||||
|
||||
test("ignores overlap entirely outside the clipped timeline viewport", () => {
|
||||
expect(
|
||||
analyzeVisualStability(
|
||||
trace([
|
||||
{
|
||||
at: 0,
|
||||
regions: {
|
||||
changing: region({ top: -200, bottom: -100 }),
|
||||
following: region({ top: -150, bottom: -50 }),
|
||||
},
|
||||
viewport: viewport(0),
|
||||
},
|
||||
]),
|
||||
{ flow: ["changing", "following"] },
|
||||
),
|
||||
).toEqual([])
|
||||
})
|
||||
|
||||
test("reports visible anchor movement while allowing virtual scrollbar movement", () => {
|
||||
const issues = analyzeVisualStability(
|
||||
trace([
|
||||
{ at: 0, regions: { anchor: region({ top: 100, bottom: 120 }) }, viewport: { ...viewport(100), scrollTop: 40 } },
|
||||
{ at: 16, regions: { anchor: region({ top: 100, bottom: 120 }) }, viewport: { ...viewport(120), scrollTop: 60 } },
|
||||
]),
|
||||
{ fixed: ["anchor"] },
|
||||
)
|
||||
|
||||
expect(issues).toEqual([])
|
||||
|
||||
const moved = analyzeVisualStability(
|
||||
trace([
|
||||
{ at: 0, regions: { anchor: region({ top: 100, bottom: 120 }) }, viewport: viewport(100) },
|
||||
{ at: 16, regions: { anchor: region({ top: 112, bottom: 132 }) }, viewport: viewport(100) },
|
||||
]),
|
||||
{ fixed: ["anchor"] },
|
||||
)
|
||||
expect(moved.some((issue) => issue.includes("anchor moved 12px in the viewport"))).toBe(true)
|
||||
})
|
||||
|
||||
test("analyzes each marked event independently", () => {
|
||||
const input: VisualStabilityTrace = {
|
||||
markers: [
|
||||
{ at: 10, label: "grow" },
|
||||
{ at: 40, label: "shrink" },
|
||||
],
|
||||
samples: [
|
||||
frame(0, region({ top: 100 })),
|
||||
frame(16, region({ top: 90 })),
|
||||
frame(32, region({ top: 80 })),
|
||||
frame(48, region({ top: 90 })),
|
||||
frame(64, region({ top: 100 })),
|
||||
],
|
||||
}
|
||||
|
||||
expect(analyzeVisualStability(input, { maxPositionReversals: 0 })).toContain("changing top reversed 1 times")
|
||||
expect(
|
||||
analyzeVisualStabilityByMarker(input, {
|
||||
maxPositionReversals: 0,
|
||||
motion: ["changing"],
|
||||
aggregateMotion: false,
|
||||
}),
|
||||
).toEqual([])
|
||||
})
|
||||
|
||||
test("reports cross-event motion reversals by default", () => {
|
||||
const input: VisualStabilityTrace = {
|
||||
markers: [
|
||||
{ at: 10, label: "up" },
|
||||
{ at: 40, label: "down" },
|
||||
],
|
||||
samples: [
|
||||
frame(0, region({ top: 100 })),
|
||||
frame(16, region({ top: 90 })),
|
||||
frame(32, region({ top: 80 })),
|
||||
frame(48, region({ top: 90 })),
|
||||
frame(64, region({ top: 100 })),
|
||||
],
|
||||
}
|
||||
|
||||
expect(analyzeVisualStabilityByMarker(input, { maxPositionReversals: 0 })).toContain("changing top reversed 1 times")
|
||||
})
|
||||
|
||||
test("reports regions rendered in the wrong flow order", () => {
|
||||
const issues = analyzeVisualStability(
|
||||
trace([frame(0, region({ top: 100, bottom: 120 }), region({ top: 60, bottom: 80 }))]),
|
||||
{ flow: ["changing", "following"] },
|
||||
)
|
||||
|
||||
expect(issues.some((issue) => issue.includes("changing rendered after following"))).toBe(true)
|
||||
})
|
||||
|
||||
test("uses painted bounds instead of clipped layout overflow", () => {
|
||||
expect(
|
||||
analyzeVisualStability(
|
||||
trace([
|
||||
frame(
|
||||
0,
|
||||
region({ top: 100, bottom: 140, height: 40, layoutTop: 100, layoutBottom: 300 }),
|
||||
region({ top: 140, bottom: 180 }),
|
||||
),
|
||||
]),
|
||||
{ flow: ["changing", "following"] },
|
||||
),
|
||||
).toEqual([])
|
||||
})
|
||||
|
||||
test("does not report disappearance when a present row moves outside the viewport", () => {
|
||||
expect(
|
||||
analyzeVisualStability(
|
||||
trace([
|
||||
frame(0, region({ visible: true })),
|
||||
frame(16, region({ visible: false, inViewport: false, top: -100, bottom: -80 })),
|
||||
frame(32, region({ visible: true })),
|
||||
]),
|
||||
),
|
||||
).toEqual([])
|
||||
})
|
||||
|
||||
test("reports an in-viewport transparent frame between visible frames", () => {
|
||||
const issues = analyzeVisualStability(
|
||||
trace([
|
||||
frame(0, region()),
|
||||
frame(16, region({ visible: false, opacity: 0, inViewport: true })),
|
||||
frame(32, region()),
|
||||
]),
|
||||
)
|
||||
|
||||
expect(issues.some((issue) => issue.includes("blanked between visible frames"))).toBe(true)
|
||||
})
|
||||
|
||||
test("reports an in-viewport display-none frame between visible frames", () => {
|
||||
const issues = analyzeVisualStability(
|
||||
trace([
|
||||
frame(0, region()),
|
||||
frame(16, region({ visible: false, width: 0, height: 0, inViewport: true, cssHidden: true })),
|
||||
frame(32, region()),
|
||||
]),
|
||||
)
|
||||
|
||||
expect(issues.some((issue) => issue.includes("blanked between visible frames"))).toBe(true)
|
||||
})
|
||||
|
||||
test("can limit motion analysis to unaffected regions", () => {
|
||||
expect(
|
||||
analyzeVisualStability(
|
||||
trace([
|
||||
frame(0, region({ height: 20 }), region()),
|
||||
frame(16, region({ height: 40 }), region()),
|
||||
frame(32, region({ height: 30 }), region()),
|
||||
]),
|
||||
{ motion: ["following"] },
|
||||
),
|
||||
).toEqual([])
|
||||
})
|
||||
|
||||
test("reports a blank frame across replacement surfaces", () => {
|
||||
const issues = analyzeVisualStability(
|
||||
{
|
||||
markers: [],
|
||||
samples: [
|
||||
{ at: 0, regions: { thinking: region(), error: region({ present: false, visible: false }) } },
|
||||
{
|
||||
at: 16,
|
||||
regions: {
|
||||
thinking: region({ present: false, visible: false }),
|
||||
error: region({ present: false, visible: false }),
|
||||
},
|
||||
},
|
||||
{ at: 32, regions: { thinking: region({ present: false, visible: false }), error: region() } },
|
||||
],
|
||||
},
|
||||
{ continuousAny: [["thinking", "error"]] },
|
||||
)
|
||||
|
||||
expect(issues.some((issue) => issue.includes("thinking | error blanked"))).toBe(true)
|
||||
})
|
||||
|
||||
test("reports failure to acquire the bottom anchor", () => {
|
||||
const issues = analyzeVisualStability(
|
||||
trace([
|
||||
{ at: 0, regions: { changing: region() }, viewport: viewport(600) },
|
||||
{ at: 16, regions: { changing: region() }, viewport: viewport(120) },
|
||||
]),
|
||||
{ acquireBottomAnchor: true },
|
||||
)
|
||||
|
||||
expect(issues.some((issue) => issue.includes("did not acquire bottom anchor"))).toBe(true)
|
||||
})
|
||||
|
||||
test("reports a required region that never renders", () => {
|
||||
const issues = analyzeVisualStability(
|
||||
trace([
|
||||
{
|
||||
at: 0,
|
||||
regions: { changing: region({ present: false, visible: false }) },
|
||||
},
|
||||
]),
|
||||
{ required: ["changing"] },
|
||||
)
|
||||
|
||||
expect(issues).toContain("changing never rendered")
|
||||
})
|
||||
|
||||
test("preserves typed region names while mapping definitions", () => {
|
||||
const regions = defineVisualRegions({
|
||||
changing: { selector: "[data-changing]" },
|
||||
following: { selector: "[data-following]", closest: "[data-row]" },
|
||||
})
|
||||
const selectors = mapVisualRegions(regions, (region) => region.selector)
|
||||
|
||||
expect(selectors).toEqual({ changing: "[data-changing]", following: "[data-following]" })
|
||||
const name: keyof typeof selectors = "changing"
|
||||
expect(name).toBe("changing")
|
||||
})
|
||||
|
||||
test("evaluates the typed invariant algebra over explicit observations", () => {
|
||||
const regions = defineVisualRegions({
|
||||
changing: { selector: "[data-changing]" },
|
||||
following: { selector: "[data-following]" },
|
||||
})
|
||||
const invariants = [
|
||||
{ type: "required", regions: ["changing"] },
|
||||
{ type: "flow", regions: ["changing", "following"] },
|
||||
] satisfies VisualInvariant<keyof typeof regions>[]
|
||||
// @ts-expect-error Plans reject names that are not in the region definition.
|
||||
const invalid = { type: "required", regions: ["missing"] } satisfies VisualInvariant<keyof typeof regions>
|
||||
const plan = visualPlan(regions, invariants, { perMarker: true })
|
||||
|
||||
expect(invalid.regions).toEqual(["missing"])
|
||||
expect(plan.perMarker).toBe(true)
|
||||
expect(analyzeVisualObservations([frame(0, region({ bottom: 50 }), region({ top: 49, bottom: 69 }))], plan)).toEqual([
|
||||
"changing overlapped following by 1px at 0ms",
|
||||
])
|
||||
})
|
||||
|
||||
test("legacy plan adapter preserves analyzer messages and order", () => {
|
||||
const input = trace([
|
||||
frame(0, region({ label: "Exploring", opacity: 1, bottom: 40 }), region({ top: 40, bottom: 60 })),
|
||||
frame(16, region({ label: "Explored", opacity: 0.2, bottom: 50 }), region({ top: 49, bottom: 69 })),
|
||||
frame(32, region({ label: "Exploring", opacity: 1, bottom: 50 }), region({ top: 50, bottom: 70 })),
|
||||
])
|
||||
const options = { flow: ["changing", "following"], stable: ["changing"] }
|
||||
|
||||
expect(analyzeVisualObservations(input.samples, legacyVisualPlan(options))).toEqual(
|
||||
analyzeVisualStability(input, options),
|
||||
)
|
||||
})
|
||||
|
||||
function frame(
|
||||
at: number,
|
||||
changing: VisualStabilityTrace["samples"][number]["regions"][string],
|
||||
following?: VisualStabilityTrace["samples"][number]["regions"][string],
|
||||
) {
|
||||
return { at, regions: { changing, ...(following ? { following } : {}) } }
|
||||
}
|
||||
|
||||
function region(input: Partial<VisualStabilityTrace["samples"][number]["regions"][string]> = {}) {
|
||||
return {
|
||||
present: true,
|
||||
visible: true,
|
||||
inViewport: true,
|
||||
top: 0,
|
||||
bottom: 20,
|
||||
width: 100,
|
||||
height: 20,
|
||||
opacity: 1,
|
||||
count: 1,
|
||||
node: 1,
|
||||
label: "",
|
||||
text: "",
|
||||
layoutTop: input.top ?? 0,
|
||||
layoutBottom: input.bottom ?? 20,
|
||||
...input,
|
||||
}
|
||||
}
|
||||
|
||||
function viewport(distanceFromBottom: number) {
|
||||
return { top: 0, bottom: 400, scrollTop: 100, scrollHeight: 500, clientHeight: 400, distanceFromBottom }
|
||||
}
|
||||
Reference in New Issue
Block a user