From 31de2dd712edcc9664891fe2332b57b2b928c3b6 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Thu, 12 Nov 2026 17:48:24 +0530 Subject: [PATCH 01/31] test-companion: fixes from running every job as the GitHub App - Ignore comments from bots, our own included. The App's own comments quote "@browserstack go", so they started runs, got a "write access" reply, and could push a real request out of the queue. The starter workflow also skips bot comments, so no runner starts. - Jobs whose output isn't code (generate, rca, verdict, validate) never commit to the branch, and BrowserStack tooling's by-products such as browserstack.err are never committed by any job. - Work handed back as a pull request after a request on a pull request now targets that pull request's branch, as a draft, titled for it. Before, it targeted the default branch and carried the whole PR. - A run that broke no longer offers "go" or leaves its check waiting for approval; it is reported like any other run, and the comment says why (the CLI's error line, e.g. a missing project or a credit limit). - Acting as the App, commits are made as its bot account, so they show its name and logo, and the loop guard still recognises them. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/answer.mjs | 28 ++++++++-- test-companion/src/main.mjs | 46 +++++++++++----- test-companion/src/outcome.mjs | 3 ++ test-companion/src/plan-run.mjs | 3 ++ test-companion/src/push-work.mjs | 52 +++++++++++++++--- test-companion/src/report.mjs | 15 ++++++ test-companion/src/sandbox.mjs | 4 +- test-companion/starter-workflow.yml | 8 +-- test-companion/test/answer.test.mjs | 30 +++++++++++ test-companion/test/outcome.test.mjs | 13 ++++- test-companion/test/plan-run.test.mjs | 21 ++++++++ test-companion/test/push-work.test.mjs | 74 ++++++++++++++++++++++++++ test-companion/test/report.test.mjs | 17 ++++++ test-companion/test/starter.test.mjs | 4 ++ 14 files changed, 287 insertions(+), 31 deletions(-) diff --git a/test-companion/src/answer.mjs b/test-companion/src/answer.mjs index 95f4f53..f72b838 100644 --- a/test-companion/src/answer.mjs +++ b/test-companion/src/answer.mjs @@ -12,9 +12,16 @@ export function createAnswerCapture() { // Discarding the rest of a line that overran MAX_PARTIAL_LINE. let skipping = false let answer + let error const read = (line) => { const t = line.trim() + // The CLI prints why it stopped as `Error: …` on stderr. + const plain = /^Error:\s*(.+)$/.exec(t) + if (plain) { + error = plain[1] + return + } if (!t.startsWith("{") || !t.endsWith("}")) return let event try { @@ -22,7 +29,12 @@ export function createAnswerCapture() { } catch { return } - if (!event || typeof event !== "object" || event.event !== "answer") return + if (!event || typeof event !== "object") return + if (event.event === "error" && typeof event.message === "string" && event.message.trim()) { + error = event.message.trim() + return + } + if (event.event !== "answer") return const text = typeof event.text === "string" ? event.text.trim() : "" if (text) answer = text } @@ -51,6 +63,7 @@ export function createAnswerCapture() { pending = "" }, answer: () => answer, + error: () => error, pending: () => pending.length, } } @@ -59,17 +72,24 @@ export function createAnswerCapture() { export function runCapturingAnswer(command, args, childEnv) { return new Promise((resolve) => { const capture = createAnswerCapture() - const child = spawn(command, args, { stdio: ["inherit", "pipe", "inherit"], env: childEnv }) + const errors = createAnswerCapture() + const child = spawn(command, args, { stdio: ["inherit", "pipe", "pipe"], env: childEnv }) child.stdout.on("data", (chunk) => { process.stdout.write(chunk) capture.push(chunk) }) + child.stderr.on("data", (chunk) => { + process.stderr.write(chunk) + errors.push(chunk) + }) child.stdout.on("error", () => {}) - child.on("error", () => resolve({ status: null, answer: undefined })) + child.stderr.on("error", () => {}) + child.on("error", () => resolve({ status: null, answer: undefined, error: undefined })) // `close`, not `exit`: stdout may still be draining at exit. child.on("close", (status) => { capture.end() - resolve({ status, answer: capture.answer() }) + errors.end() + resolve({ status, answer: capture.answer(), error: errors.error() ?? capture.error() }) }) }) } diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index a93827c..d39c3f9 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -21,7 +21,7 @@ import { appApi, installationLookupMessage, } from "./app-auth.mjs" -import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail, approvalWasSpent } from "./outcome.mjs" +import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail, approvalWasSpent, runDidNotWork } from "./outcome.mjs" import { renderJobSummary, renderCheckSummary, @@ -69,6 +69,9 @@ import { ghEnvFrom, gitEnvFrom, GIT_SAFE_ARGS, + NO_CODE_OUTPUT, + commitIdentity, + handedOffPrRequest, } from "./push-work.mjs" const env = process.env @@ -135,6 +138,9 @@ function gh(args, body) { } } +// Who the Action's commits are by; the App's bot account once we act as it. +let identity = commitIdentity() + // The login our comments appear under: github-actions[bot], or [bot] with the App. // Used to tell our proposals from another bot's. let botLogin = "github-actions[bot]" @@ -225,7 +231,12 @@ async function appInstallationToken() { // Mask before first use. console.log(`::add-mask::${token}`) - if (appLogin) botLogin = appLogin + if (appLogin) { + botLogin = appLogin + // The bot account's id makes the noreply email GitHub links to the App. + const user = runGh(["api", `users/${encodeURIComponent(appLogin)}`, "--jq", ".id"], { token }) + identity = commitIdentity({ appSlug: found.data?.app_slug, appUserId: user.status === 0 ? String(user.stdout ?? "").trim() : "" }) + } return token } @@ -663,11 +674,13 @@ async function main() { sameRepo: isSameRepo(prForTicket?.head?.repo?.full_name, REPO), branch: plan.context.branch, headAuthorEmail: git(["log", "-1", "--format=%ae"]).out, + ourEmails: [identity.email], // The earlier checkout is advisory, so confirm we are on the reviewed commit before pushing. onWantedCommit: !needsCheckout({ headSha: git(["rev-parse", "HEAD"]).out, wantSha: plan.context.sha }), defaultBranch: defaultBranchName(), + workflow: plan.workflow, }) - if (plan.context.branch && !plan.preview && !saving.ok) { + if (plan.context.branch && !plan.preview && !saving.ok && !NO_CODE_OUTPUT.has(String(plan.workflow ?? ""))) { notice(`Nothing will be saved to ${plan.context.branch}: ${saving.why}.`) } @@ -803,9 +816,12 @@ async function main() { // file is the fallback, only for ANSWER_JOBS (for generate it holds test cases). answer: run.answer || (hasResult && ANSWER_JOBS.has(plan.workflow ?? summary.workflow) ? readResult() : undefined), failedTests, + // Only where the run didn't work, so a stray error line never colours a success. + errorLine: runDidNotWork(outcome) ? run.error : undefined, } - // Proposing: post what it would do and stop. Nothing has been written. - if (proposing) { + // Proposing: post what it would do and stop. A run that broke has nothing to approve, + // so it is reported like any other run, which also completes its check. + if (proposing && !runDidNotWork(outcome)) { const sha = checkSha(plan.context, summary, env.GITHUB_SHA) addToJobSummary(renderJobSummary(view)) const size = describeChangeSize(pull) @@ -924,7 +940,7 @@ function saveWork(saving, plan) { `Written by Test Companion for ${plan.context.pr ? `#${plan.context.pr}` : plan.context.sha}.`, `Run: ${RUN_URL}`, ].join("\n") - const committed = git(["commit", "-m", body]) + const committed = git(["-c", `user.name=${identity.name}`, "-c", `user.email=${identity.email}`, "commit", "-m", body]) if (!committed.ok) { return renderPushNote({ branch, @@ -956,7 +972,7 @@ function isolateAgent(plan) { let why = can.why if (can.ok) { try { - const layout = prepareSandbox({ repoDir: process.cwd(), tcPath: TC_PATH }) + const layout = prepareSandbox({ repoDir: process.cwd(), tcPath: TC_PATH, identity }) notice("Test Companion: the agent runs as a separate, locked-down user with no access to this job's tokens.") return layout } catch (e) { @@ -1134,12 +1150,16 @@ function openHandedOffPr(path, plan) { : `**Could not open the pull request:** ${pushed.err || "git push failed"}.` } // actions/checkout doesn't record the remote HEAD, so the CLI's base is only a fallback. - const opened = ghRaw(["api", `repos/${REPO}/pulls`, "--method", "POST", "--input", "-"], { - head: req.branch, - base: defaultBranchName() || req.base, - title: req.title, - body: req.body, - }) + const opened = ghRaw( + ["api", `repos/${REPO}/pulls`, "--method", "POST", "--input", "-"], + handedOffPrRequest({ + req, + workflow: plan.workflow, + pr: plan.context.pr, + prBranch: plan.context.branch, + defaultBranch: defaultBranchName(), + }), + ) if (opened.status !== 0) { return /already exists/i.test(`${opened.stdout}${opened.stderr}`) ? `**Already done:** a pull request for \`${req.branch}\` is already open.` diff --git a/test-companion/src/outcome.mjs b/test-companion/src/outcome.mjs index 7db4142..bacf282 100644 --- a/test-companion/src/outcome.mjs +++ b/test-companion/src/outcome.mjs @@ -101,6 +101,9 @@ export const BUILD_EFFECTS = Object.freeze({ const RUN_DID_NOT_WORK = new Set([OUTCOMES.RUN_BROKE, OUTCOMES.COULD_NOT_RUN, OUTCOMES.STOPPED_AT_LIMIT]) +/** The run broke, never started or hit a limit: there is nothing to approve. */ +export const runDidNotWork = (outcome) => RUN_DID_NOT_WORK.has(outcome) + /** * Whether the Action's step goes red. The check is unaffected. Unknown values * act as the default, not `report`; `off` is a legacy synonym for `report`. diff --git a/test-companion/src/plan-run.mjs b/test-companion/src/plan-run.mjs index cfa55d7..673d54d 100644 --- a/test-companion/src/plan-run.mjs +++ b/test-companion/src/plan-run.mjs @@ -142,6 +142,9 @@ function route({ eventName, payload, inputs, repo, sha, ref }) { function fromComment(payload, inputs) { if (payload.action !== "created") return skip("comment-edited", payload.action) if (!payload.issue?.pull_request) return skip("not-a-pull-request") + // Our own comments quote "@browserstack go"; reacting to them loops and can push a real request out of the queue. + const author = payload.comment?.user + if (author?.type === "Bot" || /\[bot\]$/i.test(author?.login ?? "")) return skip("bot-comment", author?.login) const body = payload.comment?.body ?? "" const addressed = MENTION.test(body) diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index 7539566..37cc658 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -6,18 +6,35 @@ /** Must match the identity action.yml configures. */ export const BOT_EMAIL = "test-companion[bot]@users.noreply.github.com" -export function authoredByUs(email) { - return ( - String(email ?? "") - .trim() - .toLowerCase() === BOT_EMAIL - ) +export const BOT_NAME = "Test Companion" + +/** + * The identity commits are made under. As the GitHub App, its bot account, so commits + * show its name and logo; otherwise the fixed Test Companion identity. + */ +export function commitIdentity({ appSlug, appUserId } = {}) { + const slug = String(appSlug ?? "").trim() + const id = String(appUserId ?? "").trim() + if (!slug || !/^\d+$/.test(id)) return { name: BOT_NAME, email: BOT_EMAIL } + return { name: `${slug}[bot]`, email: `${id}+${slug}[bot]@users.noreply.github.com` } +} + +export function authoredByUs(email, ours = []) { + const e = String(email ?? "") + .trim() + .toLowerCase() + return e === BOT_EMAIL || ours.some((o) => String(o ?? "").toLowerCase() === e && e !== "") } +/** Jobs whose result is a report or Test Management cases, never code on the branch. */ +export const NO_CODE_OUTPUT = new Set(["generate", "rca", "verdict", "validate"]) + /** May this run push? Every check fails closed. */ -export function shouldPushWork({ preview, sameRepo, branch, headAuthorEmail, onWantedCommit, defaultBranch } = {}) { +export function shouldPushWork({ preview, sameRepo, branch, headAuthorEmail, onWantedCommit, defaultBranch, workflow, ourEmails = [] } = {}) { // A shell command the agent ran during preview is outside the brain's write refusal. if (preview) return { ok: false, why: "this is a preview run, which writes nothing" } + // Anything such a job leaves in the checkout is a by-product, not work to save. + if (NO_CODE_OUTPUT.has(String(workflow ?? ""))) return { ok: false, why: `${workflow} does not change code` } // Never push to a fork's branch with pull_request_target credentials. Unknown counts as no. if (sameRepo !== true) return { ok: false, why: "the change does not come from this repository" } if (!String(branch ?? "").trim()) return { ok: false, why: "there is no branch to push to" } @@ -31,7 +48,7 @@ export function shouldPushWork({ preview, sameRepo, branch, headAuthorEmail, onW return { ok: false, why: "that is the repository's default branch" } } // Loop guard: never push in reaction to our own commit. - if (authoredByUs(headAuthorEmail)) return { ok: false, why: "this run is reacting to our own commit" } + if (authoredByUs(headAuthorEmail, ourEmails)) return { ok: false, why: "this run is reacting to our own commit" } return { ok: true } } @@ -90,6 +107,9 @@ const NEVER_COMMIT = [ /^\.testcompanion$/i, /^blob-report$/i, /\.log$/i, + // Written by BrowserStack tooling the agent runs; never part of the work. + /^browserstack\.err$/i, + /^\.browserstack/i, ] /** @@ -297,3 +317,19 @@ export const GIT_SAFE_ARGS = Object.freeze([ "-c", "core.attributesFile=/dev/null", ]) + +/** + * The pull request to open for work the CLI handed back. Asked on a pull request, + * it targets that pull request's branch, so it carries only the new change. + */ +export function handedOffPrRequest({ req, workflow, pr, prBranch, defaultBranch } = {}) { + const onAPr = Boolean(pr && String(prBranch ?? "").trim()) + const job = String(workflow ?? "").trim() || "change" + return { + head: req?.branch, + base: onAPr ? prBranch : defaultBranch || req?.base, + title: onAPr ? `Test Companion ${job} for #${pr}` : req?.title, + body: onAPr ? `${String(req?.body ?? "").trim()}\n\nRequested on #${pr}.`.trim() : req?.body, + draft: true, + } +} diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index 7764d19..79325f9 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -85,11 +85,18 @@ export function renderJobSummary({ workNote, extraChecks = [], warnings = [], + errorLine, } = {}) { const lines = [] lines.push(`## Test Companion — ${withMark(outcome)}`) lines.push("") + const why = oneLine(errorLine) + if (why) { + lines.push(`> **Why:** ${inertMarkers(why)}`) + lines.push("") + } + if (String(approvalNote ?? "").trim()) { lines.push(`>${inertMarkers(approvalNote)}`) lines.push("") @@ -457,3 +464,11 @@ export function inFlightCheckId(checks, checkName = "Test Companion") { if (ours.length === 0) return undefined return ours.reduce((a, b) => ((a.id ?? 0) >= (b.id ?? 0) ? a : b)).id } + +/** One line of at most 300 characters, for an error quoted in a comment. */ +function oneLine(text) { + const line = String(text ?? "") + .replace(/\s+/g, " ") + .trim() + return line.length > 300 ? `${line.slice(0, 299)}…` : line +} diff --git a/test-companion/src/sandbox.mjs b/test-companion/src/sandbox.mjs index eae47b6..c4203a6 100644 --- a/test-companion/src/sandbox.mjs +++ b/test-companion/src/sandbox.mjs @@ -38,7 +38,7 @@ function must(r, what) { return r } -export function prepareSandbox({ repoDir, tcPath }) { +export function prepareSandbox({ repoDir, tcPath, identity = { name: "Test Companion", email: "test-companion[bot]@users.noreply.github.com" } }) { const layout = sandboxLayout() const me = userInfo().username @@ -90,7 +90,7 @@ export function prepareSandbox({ repoDir, tcPath }) { // safe.directory: git refuses a repository owned by another user. writeAs( `${layout.home}/.gitconfig`, - `[safe]\n\tdirectory = ${layout.work}\n[user]\n\tname = Test Companion\n\temail = test-companion[bot]@users.noreply.github.com\n`, + `[safe]\n\tdirectory = ${layout.work}\n[user]\n\tname = ${identity.name}\n\temail = ${identity.email}\n`, `${AGENT_USER}:${AGENT_USER}`, "644", ) diff --git a/test-companion/starter-workflow.yml b/test-companion/starter-workflow.yml index a157de4..37e534b 100644 --- a/test-companion/starter-workflow.yml +++ b/test-companion/starter-workflow.yml @@ -53,12 +53,14 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 35 # A loose first filter so unrelated comments don't boot a runner; the Action makes - # the real decision. The last two lines catch a bare "go" approval (case-insensitive). + # the real decision. Bot comments, our own included, never start one. The last two + # lines catch a bare "go" approval (case-insensitive). if: >- github.event_name != 'issue_comment' - || contains(github.event.comment.body, '@browserstack') + || (github.event.comment.user.type != 'Bot' + && (contains(github.event.comment.body, '@browserstack') || startsWith(github.event.comment.body, 'go') - || startsWith(github.event.comment.body, 'approve') + || startsWith(github.event.comment.body, 'approve'))) # Keep this block even with the App: the workflow token is the fallback. permissions: contents: write # branches and commits for fix / reconcile jobs diff --git a/test-companion/test/answer.test.mjs b/test-companion/test/answer.test.mjs index 44a46e8..de4a1e3 100644 --- a/test-companion/test/answer.test.mjs +++ b/test-companion/test/answer.test.mjs @@ -174,3 +174,33 @@ describe("runCapturingAnswer", () => { assert.equal(r.answer, undefined) }) }) + +describe("the CLI's reason for stopping is kept", () => { + const node = process.execPath + + test("from an `Error:` line on stderr", async () => { + const r = await runCapturingAnswer(node, ["-e", `process.stderr.write('Error: project "x" not found\\n'); process.exit(1)`], process.env) + assert.equal(r.error, 'project "x" not found') + }) + + test("from an error event on stdout", async () => { + const ev = JSON.stringify({ event: "error", fatal: true, message: "You've hit your Free credit limit." }) + const r = await runCapturingAnswer(node, ["-e", `process.stdout.write(${JSON.stringify(ev)} + "\\n"); process.exit(1)`], process.env) + assert.equal(r.error, "You've hit your Free credit limit.") + }) + + test("stderr wins: it is the CLI's own last word", async () => { + const ev = JSON.stringify({ event: "error", message: "api_request_failed" }) + const r = await runCapturingAnswer( + node, + ["-e", `process.stdout.write(${JSON.stringify(ev)} + "\\n"); process.stderr.write("Error: task failed: credit limit\\n"); process.exit(1)`], + process.env, + ) + assert.equal(r.error, "task failed: credit limit") + }) + + test("a clean run has none", async () => { + const r = await runCapturingAnswer(node, ["-e", "process.exit(0)"], process.env) + assert.equal(r.error, undefined) + }) +}) diff --git a/test-companion/test/outcome.test.mjs b/test-companion/test/outcome.test.mjs index 7ee60ba..28ba86a 100644 --- a/test-companion/test/outcome.test.mjs +++ b/test-companion/test/outcome.test.mjs @@ -1,6 +1,6 @@ import { test, describe } from "node:test" import assert from "node:assert/strict" -import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail, approvalWasSpent } from "../src/outcome.mjs" +import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail, approvalWasSpent, runDidNotWork } from "../src/outcome.mjs" describe("parseSummary", () => { test("reads the CLI's key: value report", () => { @@ -173,3 +173,14 @@ describe("approvalWasSpent", () => { } }) }) + +describe("runDidNotWork — there is nothing to approve", () => { + test("broke, never started, or hit a limit", () => { + for (const o of [OUTCOMES.RUN_BROKE, OUTCOMES.COULD_NOT_RUN, OUTCOMES.STOPPED_AT_LIMIT]) assert.equal(runDidNotWork(o), true) + }) + + test("succeeded or failed tests is a real result", () => { + assert.equal(runDidNotWork(OUTCOMES.SUCCEEDED), false) + assert.equal(runDidNotWork(OUTCOMES.TESTS_FAILED), false) + }) +}) diff --git a/test-companion/test/plan-run.test.mjs b/test-companion/test/plan-run.test.mjs index a9917d8..63ce06b 100644 --- a/test-companion/test/plan-run.test.mjs +++ b/test-companion/test/plan-run.test.mjs @@ -636,3 +636,24 @@ describe("the @browserstack mention", () => { assert.match(out.context.instructions, /test this PR/) }) }) + +describe("comments from bots, our own included, are ignored", () => { + const fromUser = (user) => ({ + action: "created", + issue: { number: 7, pull_request: {} }, + comment: { body: "Reply **go** to run this, or **@browserstack go** with your edits.", user }, + }) + const run = (user) => planRun({ eventName: "issue_comment", repo: "acme/shop", payload: fromUser(user) }) + + test("the App's own comment", () => { + assert.equal(run({ login: "browserstack[bot]", type: "Bot" }).skip?.reason, "bot-comment") + }) + + test("a bot login without a type", () => { + assert.equal(run({ login: "github-actions[bot]" }).skip?.reason, "bot-comment") + }) + + test("a person saying the same thing still counts", () => { + assert.equal(run({ login: "an-sdet", type: "User" }).skip, undefined) + }) +}) diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index d07c6eb..25d8a77 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -23,6 +23,8 @@ import { GIT_SAFE_ARGS, pushUrlFor, BOT_EMAIL, + handedOffPrRequest, + commitIdentity, } from "../src/push-work.mjs" const live = { preview: false, sameRepo: true, branch: "feat/x", headAuthorEmail: "dev@example.com", onWantedCommit: true } @@ -645,3 +647,75 @@ describe("splitPushable refuses .github and symlinks", () => { assert.deepEqual(skip, ["tests/fixtures/link"]) }) }) + +describe("jobs that don't produce code never push", () => { + const live = { preview: false, sameRepo: true, branch: "feature/x", onWantedCommit: true, defaultBranch: "main" } + + for (const workflow of ["generate", "rca", "verdict", "validate"]) { + test(`${workflow} saves nothing to the branch`, () => { + const out = shouldPushWork({ ...live, workflow }) + assert.equal(out.ok, false) + assert.match(out.why, /does not change code/) + }) + } + + test("a job that writes code still may", () => { + assert.equal(shouldPushWork({ ...live, workflow: "test-pr" }).ok, true) + assert.equal(shouldPushWork({ ...live, workflow: "fix" }).ok, true) + }) +}) + +describe("BrowserStack tooling's by-products are never committed", () => { + test("browserstack.err and .browserstack files are skipped", () => { + const { push, skip } = splitPushable(["browserstack.err", ".browserstack-local.json", "src/a.test.ts", "nested/browserstack.err"]) + assert.deepEqual(push, ["src/a.test.ts"]) + assert.deepEqual(skip, ["browserstack.err", ".browserstack-local.json", "nested/browserstack.err"]) + }) +}) + +describe("handedOffPrRequest — a fix asked for on a pull request targets that pull request", () => { + const req = { branch: "tc/fix/abc", base: "main", title: "tc fix: abc", body: "What changed." } + + test("on a pull request: base is its branch, a draft, a title that names it", () => { + const out = handedOffPrRequest({ req, workflow: "fix", pr: "42", prBranch: "feat/cart", defaultBranch: "main" }) + assert.equal(out.base, "feat/cart") + assert.equal(out.head, "tc/fix/abc") + assert.equal(out.draft, true) + assert.equal(out.title, "Test Companion fix for #42") + assert.match(out.body, /Requested on #42\./) + }) + + test("without a pull request: the default branch, the CLI's own title", () => { + const out = handedOffPrRequest({ req, workflow: "fix", defaultBranch: "main" }) + assert.equal(out.base, "main") + assert.equal(out.title, "tc fix: abc") + assert.equal(out.draft, true) + }) + + test("no default branch known: the CLI's base", () => { + assert.equal(handedOffPrRequest({ req, workflow: "automate" }).base, "main") + }) +}) + +describe("commitIdentity — commits are credited to the App when acting as it", () => { + test("the App's bot account, with the noreply email GitHub links to it", () => { + assert.deepEqual(commitIdentity({ appSlug: "browserstack", appUserId: "355331011" }), { + name: "browserstack[bot]", + email: "355331011+browserstack[bot]@users.noreply.github.com", + }) + }) + + test("without the App, or without its id, the fixed identity", () => { + assert.deepEqual(commitIdentity(), { name: "Test Companion", email: BOT_EMAIL }) + assert.deepEqual(commitIdentity({ appSlug: "browserstack", appUserId: "" }), { name: "Test Companion", email: BOT_EMAIL }) + assert.deepEqual(commitIdentity({ appSlug: "browserstack", appUserId: "not-a-number" }), { name: "Test Companion", email: BOT_EMAIL }) + }) + + test("the loop guard knows the App's commits are ours too", () => { + const app = "355331011+browserstack[bot]@users.noreply.github.com" + assert.equal(authoredByUs(app, [app]), true) + assert.equal(authoredByUs(BOT_EMAIL), true) + assert.equal(authoredByUs("dev@example.com", [app]), false) + assert.equal(authoredByUs("", [""]), false) + }) +}) diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs index 1d8b016..7dfe007 100644 --- a/test-companion/test/report.test.mjs +++ b/test-companion/test/report.test.mjs @@ -1244,3 +1244,20 @@ describe("a plan cannot choose which job its approval runs", () => { assert.equal(isApprovableJob("generate", undefined), false) }) }) + +describe("a run that didn't work says why", () => { + test("the error line leads the comment, on one line", () => { + const md = renderJobSummary({ ...base, outcome: OUTCOMES.RUN_BROKE, errorLine: 'project "rag_assignment"\nnot found' }) + assert.match(md, /> \*\*Why:\*\* project "rag_assignment" not found/) + }) + + test("a very long reason is cut", () => { + const md = renderJobSummary({ ...base, outcome: OUTCOMES.RUN_BROKE, errorLine: "x".repeat(1000) }) + const why = md.split("\n").find((l) => l.startsWith("> **Why:**")) + assert.ok(why.length < 330) + }) + + test("no error line, no Why", () => { + assert.doesNotMatch(renderJobSummary(base), /\*\*Why:\*\*/) + }) +}) diff --git a/test-companion/test/starter.test.mjs b/test-companion/test/starter.test.mjs index 18dfc98..a373977 100644 --- a/test-companion/test/starter.test.mjs +++ b/test-companion/test/starter.test.mjs @@ -200,6 +200,10 @@ describe("a comment that is not for us does not start a job", () => { assert.match(job, /startsWith\(github\.event\.comment\.body, 'approve'\)/) }) + test("a bot's comment, the App's own included, never boots a runner", () => { + assert.match(job, /github\.event\.comment\.user\.type != 'Bot'/) + }) + test("every other event is untouched, so pushes and schedules still run", () => { // The filter opens with the non-comment case for exactly this reason. const mentionCheck = job.indexOf("contains(github.event.comment.body, '@browserstack')") From 566c2829b2406b3afbfb4dcae0fa41a0ab55ab0a Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Thu, 12 Nov 2026 17:56:04 +0530 Subject: [PATCH 02/31] test-companion: leave tool by-products out of a handed-back PR A fix or automate branch that also carried a file Test Companion never commits, such as browserstack.err written by BrowserStack tooling, was refused outright, so a real fix could be blocked by a by-product. Now only files under .github/ and symlinks refuse the branch. Other never-committed files are left out: one commit on top of the agent's branch puts them back as they were, built in a scratch index so the checkout is untouched, and the comment says what was left out. If they were the branch's only changes, no pull request is opened. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/main.mjs | 57 +++++++++++++++++++++--- test-companion/src/push-work.mjs | 15 +++++++ test-companion/test/push-work.test.mjs | 60 +++++++++++++++++++++++++- 3 files changed, 125 insertions(+), 7 deletions(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index d39c3f9..5976400 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -56,6 +56,7 @@ import { shouldPushWork, changedPaths, splitPushable, + splitHandoffPaths, renderPushNote, needsCheckout, isSameRepo, @@ -1137,13 +1138,23 @@ function openHandedOffPr(path, plan) { paths.push(fields[i + 1]) if (meta[1] === "120000") links.add(fields[i + 1]) } - const { skip } = splitPushable(paths, { isSymlink: (p) => links.has(p) }) - if (skip.length) { - return `**Did not open a pull request:** the branch changes files Test Companion never pushes: ${skip.map((f) => `\`${f}\``).join(", ")}.` + const { refused, dropped } = splitHandoffPaths(paths, { isSymlink: (p) => links.has(p) }) + if (refused.length) { + return `**Did not open a pull request:** the branch changes files Test Companion never pushes: ${refused.map((f) => `\`${f}\``).join(", ")}.` } + let head = tip.out + if (dropped.length) { + const left = leaveOut(from.out, tip.out, dropped) + if (left.error) return `**Could not open the pull request:** ${left.error}.` + if (left.empty) { + return `**Did not open a pull request:** the run's only changes were files Test Companion never commits: ${dropped.map((f) => `\`${f}\``).join(", ")}.` + } + head = left.sha + } + const leftOutNote = dropped.length ? ` Left out ${dropped.map((f) => `\`${f}\``).join(", ")}, which Test Companion never commits.` : "" const target = pushUrlFor(REPO, env.GITHUB_SERVER_URL) if (!target) return `**Could not open the pull request:** bad repository name "${REPO}".` - const pushed = git(["push", target, `${tip.out}:refs/heads/${req.branch}`], pushCredentialEnv(env.GH_TOKEN, target)) + const pushed = git(["push", target, `${head}:refs/heads/${req.branch}`], pushCredentialEnv(env.GH_TOKEN, target)) if (!pushed.ok) { return /rejected|non-fast-forward|already exists/i.test(pushed.err) ? `**Already done:** \`${req.branch}\` already exists, so this job's pull request was opened by an earlier run.` @@ -1171,7 +1182,43 @@ function openHandedOffPr(path, plan) { } catch { // The link is a nicety. } - return `**Opened a pull request** with this run's changes${url ? `: ${url}` : ` from \`${req.branch}\``}.` + return `**Opened a pull request** with this run's changes${url ? `: ${url}` : ` from \`${req.branch}\``}.${leftOutNote}` +} + +/** + * A commit on top of `tip` that puts `paths` back as they were at `base`, built in a + * scratch index so the checkout is untouched. Returns { sha }, { empty } when nothing + * else changed, or { error }. + */ +function leaveOut(base, tip, paths) { + const index = join(REPORT_DIR, `tc-handoff-index-${process.pid}`) + const scratch = { GIT_INDEX_FILE: index } + try { + if (!git(["read-tree", tip], scratch).ok) return { error: "could not read the branch" } + // Resetting to `base` restores a changed file and removes an added one. + if (!git(["reset", "-q", base, "--", ...paths], scratch).ok) return { error: "could not leave out the by-products" } + const tree = git(["write-tree"], scratch) + if (!tree.ok) return { error: "could not write the branch's files" } + if (tree.out === git(["rev-parse", `${base}^{tree}`]).out) return { empty: true } + const commit = git( + [ + "-c", + `user.name=${identity.name}`, + "-c", + `user.email=${identity.email}`, + "commit-tree", + tree.out, + "-p", + tip, + "-m", + `Leave out files Test Companion never commits\n\n${paths.join("\n")}`, + ], + scratch, + ) + return commit.ok && commit.out ? { sha: commit.out } : { error: "could not commit without the by-products" } + } finally { + rmSync(index, { force: true }) + } } /** Create or update this run's check on `sha`. */ diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index 37cc658..a80ebd2 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -130,6 +130,21 @@ export function splitPushable(paths, { isSymlink = () => false } = {}) { return { push, skip } } +/** + * For a branch the agent handed back: files under `.github/` and symlinks are + * refused outright; other never-committed files (tool by-products, logs, .env) + * are dropped from the pull request instead of blocking it. + */ +export function splitHandoffPaths(paths, { isSymlink = () => false } = {}) { + const refused = [] + const dropped = [] + for (const path of splitPushable(paths, { isSymlink }).skip) { + const dangerous = isSymlink(path) || String(path).split("/").some((s) => s.toLowerCase() === ".github") + ;(dangerous ? refused : dropped).push(path) + } + return { refused, dropped } +} + /** The pull request note about the push, including when it failed. */ export function renderPushNote({ branch, sha, files = [], skipped = [], error } = {}) { const list = files.map((f) => `- \`${f}\``).join("\n") diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index 25d8a77..36b0f49 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -1,13 +1,14 @@ import { test, describe } from "node:test" import assert from "node:assert/strict" -import { execFileSync } from "node:child_process" -import { mkdtempSync, rmSync, readdirSync, readFileSync } from "node:fs" +import { execFileSync, spawnSync } from "node:child_process" +import { mkdtempSync, rmSync, readdirSync, readFileSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" import { shouldPushWork, changedPaths, splitPushable, + splitHandoffPaths, renderPushNote, authoredByUs, needsCheckout, @@ -719,3 +720,58 @@ describe("commitIdentity — commits are credited to the App when acting as it", assert.equal(authoredByUs("", [""]), false) }) }) + +describe("splitHandoffPaths — a handed-back branch", () => { + test("tool by-products are dropped, not a reason to refuse", () => { + const { refused, dropped } = splitHandoffPaths(["src/a.ts", "browserstack.err", ".browserstack-cache/x", "run.log"]) + assert.deepEqual(refused, []) + assert.deepEqual(dropped, ["browserstack.err", ".browserstack-cache/x", "run.log"]) + }) + + test(".github files and symlinks still refuse the branch", () => { + const { refused, dropped } = splitHandoffPaths([".github/workflows/x.yml", "link", "browserstack.err"], { + isSymlink: (p) => p === "link", + }) + assert.deepEqual(refused, [".github/workflows/x.yml", "link"]) + assert.deepEqual(dropped, ["browserstack.err"]) + }) + + test("ordinary files are neither refused nor dropped", () => { + assert.deepEqual(splitHandoffPaths(["src/a.ts", "tests/a.spec.ts"]), { refused: [], dropped: [] }) + }) +}) + +describe("leaving by-products out of a branch with a scratch index", () => { + // A hook running these tests may export GIT_DIR and friends; they must not leak in. + const cleanGitEnv = () => Object.fromEntries(Object.entries(process.env).filter(([k]) => !k.startsWith("GIT_"))) + // The same git steps leaveOut() in main.mjs runs, on a real repository. + const run = (cwd, args, env = {}) => { + const r = spawnSync("git", args, { cwd, encoding: "utf8", env: { ...cleanGitEnv(), ...env } }) + assert.equal(r.status, 0, r.stderr) + return r.stdout.trim() + } + test("the new commit keeps the work and loses the by-product", () => { + const dir = mkdtempSync(join(tmpdir(), "tc-leaveout-")) + const id = ["-c", "user.name=t", "-c", "user.email=t@example.com"] + run(dir, ["init", "-q"]) + writeFileSync(join(dir, "a.ts"), "old\n") + run(dir, ["add", "."]) + run(dir, [...id, "commit", "-qm", "base"]) + const base = run(dir, ["rev-parse", "HEAD"]) + writeFileSync(join(dir, "a.ts"), "new\n") + writeFileSync(join(dir, "browserstack.err"), "[object Object]") + run(dir, ["add", "."]) + run(dir, [...id, "commit", "-qm", "work"]) + const tip = run(dir, ["rev-parse", "HEAD"]) + + const scratch = { GIT_INDEX_FILE: join(dir, ".git", "scratch-index") } + run(dir, ["read-tree", tip], scratch) + run(dir, ["reset", "-q", base, "--", "browserstack.err"], scratch) + const tree = run(dir, ["write-tree"], scratch) + const sha = run(dir, [...id, "commit-tree", tree, "-p", tip, "-m", "leave out"], scratch) + + assert.equal(run(dir, ["diff", "--name-only", base, sha]), "a.ts") + assert.equal(run(dir, ["show", `${sha}:a.ts`]), "new") + rmSync(dir, { recursive: true, force: true }) + }) +}) From 46723f32374b5001710596548e2c3292459eab77 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Thu, 12 Nov 2026 18:10:44 +0530 Subject: [PATCH 03/31] test-companion: keep left-out files off the branch and read file names literally - A handed-back branch with files Test Companion never commits now gets one squashed commit on the base, so those files are not in anything pushed. Before, the clean commit sat on top of the agent's commit, which still held them. - git reads every path as a plain file name (GIT_LITERAL_PATHSPECS), so a file named like ":!x" can't make git add or reset match other files. The left-out paths go to git on stdin. - The "jobs without code output never commit" rule is checked again with the job the pipeline chose, so event runs are covered too. - The App's commits count as ours even when a later run can't act as the App, so it doesn't push in reaction to them. - A pull request that can't be a draft is opened as a normal one; "already done" is only said when its pull request really is open. - The pull request targets the requesting pull request's branch only when the run started on that pull request's commit, and its title names the job from the hand-off. - File names in comments are capped at ten and rendered so a backtick in a name can't break out. - Tests now call the real functions, against real git repositories, and the starter's comment filter is evaluated rather than searched for. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/main.mjs | 123 +++++++-------- test-companion/src/push-work.mjs | 93 ++++++++++-- test-companion/src/sandbox.mjs | 3 +- test-companion/test/push-work.test.mjs | 198 ++++++++++++++++++++----- test-companion/test/starter.test.mjs | 78 +++++++++- 5 files changed, 381 insertions(+), 114 deletions(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 5976400..58ef6ea 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -13,7 +13,7 @@ import { childEnvFor } from "./child-env.mjs" import { canIsolate, prepareSandbox, stopSandbox, handToAgent, writeAgentEnv, readAgentFile } from "./sandbox.mjs" import { agentEnvFrom, asAgentCommand, checkPrHandoff, readClaimResult } from "./sandbox-policy.mjs" import { startRelay } from "./relay.mjs" -import { runCapturingAnswer } from "./answer.mjs" +import { runCapturingAnswer, redactSecrets, secretsFrom } from "./answer.mjs" import { appCredentials, signJwt, @@ -57,6 +57,10 @@ import { changedPaths, splitPushable, splitHandoffPaths, + savingFor, + leaveOut, + leaveOutMessage, + listFiles, renderPushNote, needsCheckout, isSameRepo, @@ -70,7 +74,6 @@ import { ghEnvFrom, gitEnvFrom, GIT_SAFE_ARGS, - NO_CODE_OUTPUT, commitIdentity, handedOffPrRequest, } from "./push-work.mjs" @@ -236,6 +239,7 @@ async function appInstallationToken() { botLogin = appLogin // The bot account's id makes the noreply email GitHub links to the App. const user = runGh(["api", `users/${encodeURIComponent(appLogin)}`, "--jq", ".id"], { token }) + if (user.status !== 0) warn(`Test Companion: could not look up ${appLogin}'s account, so commits use the plain Test Companion name.`) identity = commitIdentity({ appSlug: found.data?.app_slug, appUserId: user.status === 0 ? String(user.stdout ?? "").trim() : "" }) } return token @@ -290,11 +294,12 @@ const GIT_BIN = trustedGitPath({ }, }) -function git(args, extraEnv = {}) { +function git(args, extraEnv = {}, input = undefined) { // Near-empty environment and no hooks: the agent has had write access to the tree. if (!GIT_BIN) return { ok: false, out: "", raw: "", err: "no trusted git binary" } const r = spawnSync(GIT_BIN, [...GIT_SAFE_ARGS, ...args], { encoding: "utf8", + input, env: { ...gitEnvFrom(env), ...extraEnv }, }) // `raw` keeps porcelain's leading space; trimming would lose the first line's status column. @@ -681,7 +686,7 @@ async function main() { defaultBranch: defaultBranchName(), workflow: plan.workflow, }) - if (plan.context.branch && !plan.preview && !saving.ok && !NO_CODE_OUTPUT.has(String(plan.workflow ?? ""))) { + if (plan.context.branch && !plan.preview && !saving.ok && !saving.quiet) { notice(`Nothing will be saved to ${plan.context.branch}: ${saving.why}.`) } @@ -796,7 +801,8 @@ async function main() { }) // automate and fix hand back a pull request; other jobs save to this branch. - const workNote = handoffNote ?? saveWork(saving, plan) + // Re-checked with the job the pipeline chose: an event run didn't know it before the run. + const workNote = handoffNote ?? saveWork(savingFor(saving, plan.workflow ?? summary.workflow), plan) const view = { workflow: plan.workflow ?? summary.workflow, @@ -815,10 +821,13 @@ async function main() { askedAt: plan.askedAt, // rca and verdict leave no artefact: the answer is the agent's closing words. The result // file is the fallback, only for ANSWER_JOBS (for generate it holds test cases). - answer: run.answer || (hasResult && ANSWER_JOBS.has(plan.workflow ?? summary.workflow) ? readResult() : undefined), + answer: redactSecrets( + run.answer || (hasResult && ANSWER_JOBS.has(plan.workflow ?? summary.workflow) ? readResult() : undefined), + secretsFrom(env), + ), failedTests, // Only where the run didn't work, so a stray error line never colours a success. - errorLine: runDidNotWork(outcome) ? run.error : undefined, + errorLine: runDidNotWork(outcome) ? redactSecrets(run.error, secretsFrom(env)) : undefined, } // Proposing: post what it would do and stop. A run that broke has nothing to approve, // so it is reported like any other run, which also completes its check. @@ -1140,41 +1149,58 @@ function openHandedOffPr(path, plan) { } const { refused, dropped } = splitHandoffPaths(paths, { isSymlink: (p) => links.has(p) }) if (refused.length) { - return `**Did not open a pull request:** the branch changes files Test Companion never pushes: ${refused.map((f) => `\`${f}\``).join(", ")}.` + return `**Did not open a pull request:** the branch changes files Test Companion never pushes: ${listFiles(refused)}.` } let head = tip.out if (dropped.length) { - const left = leaveOut(from.out, tip.out, dropped) + const indexFile = join(REPORT_DIR, `tc-handoff-index-${process.pid}`) + let left + try { + left = leaveOut({ git, base: from.out, tip: tip.out, paths: dropped, identity, message: leaveOutMessage(req.title, dropped), indexFile }) + } finally { + rmSync(indexFile, { force: true }) + } if (left.error) return `**Could not open the pull request:** ${left.error}.` if (left.empty) { - return `**Did not open a pull request:** the run's only changes were files Test Companion never commits: ${dropped.map((f) => `\`${f}\``).join(", ")}.` + return `**Did not open a pull request:** the run's only changes were files Test Companion never commits: ${listFiles(dropped)}.` } head = left.sha } - const leftOutNote = dropped.length ? ` Left out ${dropped.map((f) => `\`${f}\``).join(", ")}, which Test Companion never commits.` : "" + const leftOutNote = dropped.length ? ` Left out ${listFiles(dropped)}, which Test Companion never commits.` : "" const target = pushUrlFor(REPO, env.GITHUB_SERVER_URL) if (!target) return `**Could not open the pull request:** bad repository name "${REPO}".` + const onReviewedCommit = String(headBeforeRun ?? "").toLowerCase() === String(plan.context.sha ?? "").toLowerCase() + const request = handedOffPrRequest({ + req, + pr: plan.context.pr, + prBranch: plan.context.branch, + defaultBranch: defaultBranchName(), + onReviewedCommit, + }) + const baseNote = + plan.context.pr && !onReviewedCommit + ? ` It targets \`${request.base}\`, not #${plan.context.pr}'s branch, because the run did not start on that pull request's commit.` + : "" const pushed = git(["push", target, `${head}:refs/heads/${req.branch}`], pushCredentialEnv(env.GH_TOKEN, target)) - if (!pushed.ok) { - return /rejected|non-fast-forward|already exists/i.test(pushed.err) - ? `**Already done:** \`${req.branch}\` already exists, so this job's pull request was opened by an earlier run.` - : `**Could not open the pull request:** ${pushed.err || "git push failed"}.` - } - // actions/checkout doesn't record the remote HEAD, so the CLI's base is only a fallback. - const opened = ghRaw( - ["api", `repos/${REPO}/pulls`, "--method", "POST", "--input", "-"], - handedOffPrRequest({ - req, - workflow: plan.workflow, - pr: plan.context.pr, - prBranch: plan.context.branch, - defaultBranch: defaultBranchName(), - }), - ) + if (!pushed.ok && !/rejected|non-fast-forward|already exists/i.test(pushed.err)) { + return `**Could not open the pull request:** ${pushed.err || "git push failed"}.` + } + // The branch may be left from an earlier run; only claim "done" if its pull request exists. + const owner = REPO.split("/")[0] + const open = gh(["api", `repos/${REPO}/pulls?head=${encodeURIComponent(`${owner}:${req.branch}`)}&state=open`]) + if (Array.isArray(open) && open.length) { + return `**Already done:** this job's pull request is already open: ${open[0].html_url}.` + } + // Push rejected and no pull request: an earlier run of this job on this commit pushed the + // branch but couldn't open its pull request, so open it now. + const create = (body) => ghRaw(["api", `repos/${REPO}/pulls`, "--method", "POST", "--input", "-"], body) + let opened = create(request) + // Draft pull requests aren't available on every plan. + if (opened.status !== 0 && /draft/i.test(`${opened.stdout}${opened.stderr}`)) { + opened = create({ ...request, draft: false }) + } if (opened.status !== 0) { - return /already exists/i.test(`${opened.stdout}${opened.stderr}`) - ? `**Already done:** a pull request for \`${req.branch}\` is already open.` - : `**Pushed \`${req.branch}\` but could not open the pull request:** ${String(opened.stderr ?? "").trim().split("\n")[0]}.` + return `**Pushed \`${req.branch}\` but could not open the pull request:** ${String(opened.stderr ?? "").trim().split("\n")[0]}.` } let url = "" try { @@ -1182,44 +1208,9 @@ function openHandedOffPr(path, plan) { } catch { // The link is a nicety. } - return `**Opened a pull request** with this run's changes${url ? `: ${url}` : ` from \`${req.branch}\``}.${leftOutNote}` + return `**Opened a pull request** with this run's changes${url ? `: ${url}` : ` from \`${req.branch}\``}.${leftOutNote}${baseNote}` } -/** - * A commit on top of `tip` that puts `paths` back as they were at `base`, built in a - * scratch index so the checkout is untouched. Returns { sha }, { empty } when nothing - * else changed, or { error }. - */ -function leaveOut(base, tip, paths) { - const index = join(REPORT_DIR, `tc-handoff-index-${process.pid}`) - const scratch = { GIT_INDEX_FILE: index } - try { - if (!git(["read-tree", tip], scratch).ok) return { error: "could not read the branch" } - // Resetting to `base` restores a changed file and removes an added one. - if (!git(["reset", "-q", base, "--", ...paths], scratch).ok) return { error: "could not leave out the by-products" } - const tree = git(["write-tree"], scratch) - if (!tree.ok) return { error: "could not write the branch's files" } - if (tree.out === git(["rev-parse", `${base}^{tree}`]).out) return { empty: true } - const commit = git( - [ - "-c", - `user.name=${identity.name}`, - "-c", - `user.email=${identity.email}`, - "commit-tree", - tree.out, - "-p", - tip, - "-m", - `Leave out files Test Companion never commits\n\n${paths.join("\n")}`, - ], - scratch, - ) - return commit.ok && commit.out ? { sha: commit.out } : { error: "could not commit without the by-products" } - } finally { - rmSync(index, { force: true }) - } -} /** Create or update this run's check on `sha`. */ function postCheck(outcome, view, sha, annotations = [], status = "completed", title = undefined) { diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index a80ebd2..231e809 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -19,22 +19,43 @@ export function commitIdentity({ appSlug, appUserId } = {}) { return { name: `${slug}[bot]`, email: `${id}+${slug}[bot]@users.noreply.github.com` } } -export function authoredByUs(email, ours = []) { - const e = String(email ?? "") - .trim() - .toLowerCase() - return e === BOT_EMAIL || ours.some((o) => String(o ?? "").toLowerCase() === e && e !== "") +/** Slugs of the GitHub App Test Companion may act as. */ +export const APP_SLUGS = Object.freeze(["browserstack"]) + +/** + * Is this commit email ours? Includes the App's noreply address whatever identity this + * run has: a run that fell back to the workflow token must still see the App's commits. + */ +export function authoredByUs(email, ours = [], { appSlugs = APP_SLUGS } = {}) { + const norm = (v) => + String(v ?? "") + .trim() + .toLowerCase() + const e = norm(email) + if (!e) return false + if (e === BOT_EMAIL || ours.map(norm).includes(e)) return true + const app = /^\d+\+([a-z0-9-]+)\[bot\]@users\.noreply\.github\.com$/.exec(e) + return Boolean(app && appSlugs.map(norm).includes(app[1])) } /** Jobs whose result is a report or Test Management cases, never code on the branch. */ export const NO_CODE_OUTPUT = new Set(["generate", "rca", "verdict", "validate"]) +/** + * The save decision once the job is known. Event runs learn their job only from the + * pipeline file, after shouldPushWork ran without it. + */ +export function savingFor(saving, job) { + if (NO_CODE_OUTPUT.has(String(job ?? ""))) return { ok: false, quiet: true, why: `${job} does not change code` } + return saving +} + /** May this run push? Every check fails closed. */ export function shouldPushWork({ preview, sameRepo, branch, headAuthorEmail, onWantedCommit, defaultBranch, workflow, ourEmails = [] } = {}) { // A shell command the agent ran during preview is outside the brain's write refusal. if (preview) return { ok: false, why: "this is a preview run, which writes nothing" } // Anything such a job leaves in the checkout is a by-product, not work to save. - if (NO_CODE_OUTPUT.has(String(workflow ?? ""))) return { ok: false, why: `${workflow} does not change code` } + if (NO_CODE_OUTPUT.has(String(workflow ?? ""))) return { ok: false, quiet: true, why: `${workflow} does not change code` } // Never push to a fork's branch with pull_request_target credentials. Unknown counts as no. if (sameRepo !== true) return { ok: false, why: "the change does not come from this repository" } if (!String(branch ?? "").trim()) return { ok: false, why: "there is no branch to push to" } @@ -145,9 +166,26 @@ export function splitHandoffPaths(paths, { isSymlink = () => false } = {}) { return { refused, dropped } } +/** A file name as markdown code, safe whatever characters the agent put in it. */ +export function codeSpan(name) { + const text = String(name ?? "").replace(/[\r\n\t]/g, " ") + const longest = Math.max(0, ...(text.match(/`+/g) ?? []).map((run) => run.length)) + const fence = "`".repeat(longest + 1) + const pad = longest > 0 || text.startsWith("`") || text.endsWith("`") ? " " : "" + return `${fence}${pad}${text}${pad}${fence}` +} + +/** File names, at most `limit` of them, then how many more. */ +export function listFiles(names, { limit = 10, render = codeSpan, separator = ", " } = {}) { + const all = names ?? [] + const shown = all.slice(0, limit).map(render) + if (all.length > limit) shown.push(`and ${all.length - limit} more`) + return shown.join(separator) +} + /** The pull request note about the push, including when it failed. */ export function renderPushNote({ branch, sha, files = [], skipped = [], error } = {}) { - const list = files.map((f) => `- \`${f}\``).join("\n") + const list = listFiles(files, { render: (f) => `- ${codeSpan(f)}`, separator: "\n", limit: 50 }) if (error) { return [ `**Could not save this work to \`${branch}\`:** ${error}`, @@ -158,7 +196,7 @@ export function renderPushNote({ branch, sha, files = [], skipped = [], error } } const lines = [`**Saved to \`${branch}\`** as \`${String(sha ?? "").slice(0, 7)}\` — ${countOf(files)}:`, "", list] if (skipped.length) { - lines.push("", `Not committed, because Test Companion never commits these: ${skipped.map((f) => `\`${f}\``).join(", ")}.`) + lines.push("", `Not committed, because Test Companion never commits these: ${listFiles(skipped)}.`) } return lines.join("\n") } @@ -226,6 +264,8 @@ export function gitEnvFrom(env = {}) { out.GIT_ASKPASS = "" out.SSH_ASKPASS = "" out.GIT_ATTR_NOSYSTEM = "1" + // File names come from the agent: `:!x` must name a file, not mean "everything but x". + out.GIT_LITERAL_PATHSPECS = "1" return out } @@ -337,9 +377,10 @@ export const GIT_SAFE_ARGS = Object.freeze([ * The pull request to open for work the CLI handed back. Asked on a pull request, * it targets that pull request's branch, so it carries only the new change. */ -export function handedOffPrRequest({ req, workflow, pr, prBranch, defaultBranch } = {}) { - const onAPr = Boolean(pr && String(prBranch ?? "").trim()) - const job = String(workflow ?? "").trim() || "change" +export function handedOffPrRequest({ req, pr, prBranch, defaultBranch, onReviewedCommit = true } = {}) { + // The changes were measured from where the run started; only that commit makes the PR's branch a fair base. + const onAPr = Boolean(pr && String(prBranch ?? "").trim() && onReviewedCommit) + const job = String(req?.workflow ?? "").trim() || "change" return { head: req?.branch, base: onAPr ? prBranch : defaultBranch || req?.base, @@ -348,3 +389,33 @@ export function handedOffPrRequest({ req, workflow, pr, prBranch, defaultBranch draft: true, } } + +/** + * One commit, on `base`, with the branch's changes except `paths` (put back as they were + * at `base`). Its parent is `base`, not `tip`, so the left-out files are not in what is + * pushed. Built in a scratch index so the checkout is untouched. + * + * `git(args, extraEnv, input)` runs git; it must apply literal pathspecs. + * Returns { sha }, { empty } when nothing else changed, or { error }. + */ +export function leaveOut({ git, base, tip, paths, identity, message, indexFile }) { + const scratch = { GIT_INDEX_FILE: indexFile, GIT_LITERAL_PATHSPECS: "1" } + if (!git(["read-tree", tip], scratch).ok) return { error: "could not read the branch" } + // Resetting to `base` restores a changed file and removes an added one. Paths on stdin: + // no argument-length limit, and no name read as an option. + const reset = git(["reset", "-q", base, "--pathspec-from-file=-", "--pathspec-file-nul"], scratch, paths.join("\0")) + if (!reset.ok) return { error: "could not leave out the by-products" } + const tree = git(["write-tree"], scratch) + if (!tree.ok || !tree.out) return { error: "could not write the branch's files" } + if (tree.out === git(["rev-parse", `${base}^{tree}`]).out) return { empty: true } + const commit = git( + ["-c", `user.name=${identity.name}`, "-c", `user.email=${identity.email}`, "commit-tree", tree.out, "-p", base, "-m", message], + scratch, + ) + return commit.ok && commit.out ? { sha: commit.out } : { error: "could not commit without the by-products" } +} + +/** The commit message for a hand-off with files left out. */ +export function leaveOutMessage(title, paths) { + return `${String(title ?? "").trim() || "Test Companion changes"}\n\nLeft out, because Test Companion never commits them:\n${listFiles(paths, { render: (p) => `- ${p}`, separator: "\n" })}` +} diff --git a/test-companion/src/sandbox.mjs b/test-companion/src/sandbox.mjs index c4203a6..b30c6d8 100644 --- a/test-companion/src/sandbox.mjs +++ b/test-companion/src/sandbox.mjs @@ -7,6 +7,7 @@ import { spawnSync } from "node:child_process" import { closeSync, constants, copyFileSync, existsSync, fstatSync, openSync, readSync, writeFileSync } from "node:fs" import { userInfo } from "node:os" import { AGENT_USER, LAUNCHER_SOURCE, sandboxLayout } from "./sandbox-policy.mjs" +import { BOT_EMAIL, BOT_NAME } from "./push-work.mjs" const SUDO = "/usr/bin/sudo" @@ -38,7 +39,7 @@ function must(r, what) { return r } -export function prepareSandbox({ repoDir, tcPath, identity = { name: "Test Companion", email: "test-companion[bot]@users.noreply.github.com" } }) { +export function prepareSandbox({ repoDir, tcPath, identity = { name: BOT_NAME, email: BOT_EMAIL } }) { const layout = sandboxLayout() const me = userInfo().username diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index 36b0f49..72ce6a7 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -1,9 +1,9 @@ import { test, describe } from "node:test" import assert from "node:assert/strict" import { execFileSync, spawnSync } from "node:child_process" -import { mkdtempSync, rmSync, readdirSync, readFileSync, writeFileSync } from "node:fs" +import { mkdtempSync, mkdirSync, rmSync, readdirSync, readFileSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" -import { join } from "node:path" +import { dirname, join } from "node:path" import { shouldPushWork, changedPaths, @@ -26,6 +26,12 @@ import { BOT_EMAIL, handedOffPrRequest, commitIdentity, + savingFor, + leaveOut, + leaveOutMessage, + codeSpan, + listFiles, + APP_SLUGS, } from "../src/push-work.mjs" const live = { preview: false, sameRepo: true, branch: "feat/x", headAuthorEmail: "dev@example.com", onWantedCommit: true } @@ -675,10 +681,10 @@ describe("BrowserStack tooling's by-products are never committed", () => { }) describe("handedOffPrRequest — a fix asked for on a pull request targets that pull request", () => { - const req = { branch: "tc/fix/abc", base: "main", title: "tc fix: abc", body: "What changed." } + const req = { workflow: "fix", branch: "tc/fix/abc", base: "main", title: "tc fix: abc", body: "What changed." } test("on a pull request: base is its branch, a draft, a title that names it", () => { - const out = handedOffPrRequest({ req, workflow: "fix", pr: "42", prBranch: "feat/cart", defaultBranch: "main" }) + const out = handedOffPrRequest({ req, pr: "42", prBranch: "feat/cart", defaultBranch: "main" }) assert.equal(out.base, "feat/cart") assert.equal(out.head, "tc/fix/abc") assert.equal(out.draft, true) @@ -686,8 +692,18 @@ describe("handedOffPrRequest — a fix asked for on a pull request targets that assert.match(out.body, /Requested on #42\./) }) + test("the title names the job from the hand-off, not the plan", () => { + const out = handedOffPrRequest({ req: { ...req, workflow: "automate" }, pr: "42", prBranch: "feat/cart" }) + assert.equal(out.title, "Test Companion automate for #42") + }) + + test("if the run didn't start on the pull request's commit, the default branch is the base", () => { + const out = handedOffPrRequest({ req, pr: "42", prBranch: "feat/cart", defaultBranch: "main", onReviewedCommit: false }) + assert.equal(out.base, "main") + }) + test("without a pull request: the default branch, the CLI's own title", () => { - const out = handedOffPrRequest({ req, workflow: "fix", defaultBranch: "main" }) + const out = handedOffPrRequest({ req, defaultBranch: "main" }) assert.equal(out.base, "main") assert.equal(out.title, "tc fix: abc") assert.equal(out.draft, true) @@ -741,37 +757,149 @@ describe("splitHandoffPaths — a handed-back branch", () => { }) }) -describe("leaving by-products out of a branch with a scratch index", () => { - // A hook running these tests may export GIT_DIR and friends; they must not leak in. - const cleanGitEnv = () => Object.fromEntries(Object.entries(process.env).filter(([k]) => !k.startsWith("GIT_"))) - // The same git steps leaveOut() in main.mjs runs, on a real repository. - const run = (cwd, args, env = {}) => { - const r = spawnSync("git", args, { cwd, encoding: "utf8", env: { ...cleanGitEnv(), ...env } }) - assert.equal(r.status, 0, r.stderr) - return r.stdout.trim() +// A real git, with the Action's own environment, for the git-level tests below. +function realGit(cwd) { + const env = gitEnvFrom(process.env) + return (args, extraEnv = {}, input = undefined) => { + const r = spawnSync("git", args, { cwd, encoding: "utf8", input, env: { ...env, ...extraEnv } }) + return { ok: r.status === 0, out: (r.stdout ?? "").trim(), err: (r.stderr ?? "").trim() } + } +} + +function repoWithBranch(files) { + const dir = mkdtempSync(join(tmpdir(), "tc-leaveout-")) + const git = realGit(dir) + const id = ["-c", "user.name=t", "-c", "user.email=t@example.com"] + const must = (args) => { + const r = git(args) + assert.ok(r.ok, `${args.join(" ")}: ${r.err}`) + return r.out } - test("the new commit keeps the work and loses the by-product", () => { - const dir = mkdtempSync(join(tmpdir(), "tc-leaveout-")) - const id = ["-c", "user.name=t", "-c", "user.email=t@example.com"] - run(dir, ["init", "-q"]) - writeFileSync(join(dir, "a.ts"), "old\n") - run(dir, ["add", "."]) - run(dir, [...id, "commit", "-qm", "base"]) - const base = run(dir, ["rev-parse", "HEAD"]) - writeFileSync(join(dir, "a.ts"), "new\n") - writeFileSync(join(dir, "browserstack.err"), "[object Object]") - run(dir, ["add", "."]) - run(dir, [...id, "commit", "-qm", "work"]) - const tip = run(dir, ["rev-parse", "HEAD"]) - - const scratch = { GIT_INDEX_FILE: join(dir, ".git", "scratch-index") } - run(dir, ["read-tree", tip], scratch) - run(dir, ["reset", "-q", base, "--", "browserstack.err"], scratch) - const tree = run(dir, ["write-tree"], scratch) - const sha = run(dir, [...id, "commit-tree", tree, "-p", tip, "-m", "leave out"], scratch) - - assert.equal(run(dir, ["diff", "--name-only", base, sha]), "a.ts") - assert.equal(run(dir, ["show", `${sha}:a.ts`]), "new") + must(["init", "-q"]) + writeFileSync(join(dir, "a.ts"), "old\n") + must(["add", "a.ts"]) + must([...id, "commit", "-qm", "base"]) + const base = must(["rev-parse", "HEAD"]) + for (const [name, content] of Object.entries(files)) { + mkdirSync(dirname(join(dir, name)), { recursive: true }) + writeFileSync(join(dir, name), content) + } + must(["add", "--", ...Object.keys(files)]) + must([...id, "commit", "-qm", "work"]) + return { dir, git, base, tip: must(["rev-parse", "HEAD"]) } +} + +describe("leaveOut — by-products never reach the pushed branch", () => { + const identity = { name: "Test Companion", email: BOT_EMAIL } + + test("one commit on the base: the work survives, the by-product is in no pushed commit", () => { + const { dir, git, base, tip } = repoWithBranch({ "a.ts": "new\n", "browserstack.err": "[object Object]", ".env": "KEY=secret" }) + const out = leaveOut({ git, base, tip, paths: ["browserstack.err", ".env"], identity, message: "m", indexFile: join(dir, "scratch") }) + assert.ok(out.sha, JSON.stringify(out)) + // Only the new commit between base and the head: the agent's commit is not reachable. + assert.deepEqual(git(["rev-list", `${base}..${out.sha}`]).out.split("\n"), [out.sha]) + assert.equal(git(["diff", "--name-only", base, out.sha]).out, "a.ts") + assert.equal(git(["show", `${out.sha}:a.ts`]).out, "new") + assert.equal(git(["cat-file", "-e", `${out.sha}:browserstack.err`]).ok, false) + assert.equal(git(["cat-file", "-e", `${out.sha}:.env`]).ok, false) + rmSync(dir, { recursive: true, force: true }) + }) + + test("a file named like pathspec magic is treated as a file name", () => { + const { dir, git, base, tip } = repoWithBranch({ "a.ts": "new\n", ":!keep/.env": "KEY=secret" }) + const out = leaveOut({ git, base, tip, paths: [":!keep/.env"], identity, message: "m", indexFile: join(dir, "scratch") }) + assert.ok(out.sha, JSON.stringify(out)) + assert.equal(git(["show", `${out.sha}:a.ts`]).out, "new", "the work was reset along with the by-product") + assert.equal(git(["diff", "--name-only", base, out.sha]).out, "a.ts") rmSync(dir, { recursive: true, force: true }) }) + + test("when the by-products were the only changes, it says so and makes no commit", () => { + const { dir, git, base, tip } = repoWithBranch({ "browserstack.err": "[object Object]" }) + assert.deepEqual( + leaveOut({ git, base, tip, paths: ["browserstack.err"], identity, message: "m", indexFile: join(dir, "scratch") }), + { empty: true }, + ) + rmSync(dir, { recursive: true, force: true }) + }) + + test("the commit message names the job and the left-out files, capped", () => { + const many = Array.from({ length: 12 }, (_, i) => `log/${i}.log`) + const msg = leaveOutMessage("Fix the cart total", many) + assert.match(msg, /^Fix the cart total\n\nLeft out/) + assert.match(msg, /and 2 more$/) + }) +}) + +describe("git reads file names literally", () => { + test("the Action's git environment turns pathspec magic off", () => { + assert.equal(gitEnvFrom({}).GIT_LITERAL_PATHSPECS, "1") + }) + + test("adding a file named `:!x` stages only that file", () => { + const dir = mkdtempSync(join(tmpdir(), "tc-literal-")) + const git = realGit(dir) + assert.ok(git(["init", "-q"]).ok) + writeFileSync(join(dir, ":!x"), "1") + writeFileSync(join(dir, ".env"), "KEY=secret") + assert.ok(git(["add", "--", ":!x"]).ok) + assert.equal(git(["diff", "--cached", "--name-only"]).out, ":!x") + rmSync(dir, { recursive: true, force: true }) + }) +}) + +describe("savingFor — the job the pipeline chose still decides", () => { + test("an event run that turned out to be generate saves nothing, quietly", () => { + const out = savingFor({ ok: true }, "generate") + assert.equal(out.ok, false) + assert.equal(out.quiet, true) + }) + + test("a job that writes code keeps the earlier decision", () => { + assert.deepEqual(savingFor({ ok: true }, "test-pr"), { ok: true }) + assert.deepEqual(savingFor({ ok: false, why: "x" }, undefined), { ok: false, why: "x" }) + }) + + test("shouldPushWork marks the no-code refusal quiet", () => { + assert.equal(shouldPushWork({ ...live, workflow: "rca" }).quiet, true) + assert.equal(shouldPushWork({ ...live, preview: true }).quiet, undefined) + }) +}) + +describe("authoredByUs — the App's commits are ours whatever this run is", () => { + test("the App's noreply email is recognised without this run's identity", () => { + assert.equal(authoredByUs("355331011+browserstack[bot]@users.noreply.github.com"), true) + assert.ok(APP_SLUGS.includes("browserstack")) + }) + + test("another app's bot is not ours", () => { + assert.equal(authoredByUs("49699333+dependabot[bot]@users.noreply.github.com"), false) + }) + + test("this run's own identity, compared trimmed and in any case", () => { + assert.equal(authoredByUs("Me@Example.com", [" me@example.com "]), true) + assert.equal(authoredByUs("", [""]), false) + }) +}) + +describe("file names in comments", () => { + test("a plain name is ordinary code", () => { + assert.equal(codeSpan("src/a.ts"), "`src/a.ts`") + }) + + test("a backtick in the name can't break out of the code span", () => { + assert.equal(codeSpan("a`b"), "`` a`b ``") + assert.equal(codeSpan("x``y"), "``` x``y ```") + }) + + test("newlines in a name are flattened", () => { + assert.equal(codeSpan("a\nb"), "`a b`") + }) + + test("lists stop at 10 names", () => { + const names = Array.from({ length: 13 }, (_, i) => `f${i}`) + const out = listFiles(names) + assert.equal(out.split(", ").length, 11) + assert.match(out, /and 3 more$/) + }) }) diff --git a/test-companion/test/starter.test.mjs b/test-companion/test/starter.test.mjs index a373977..7269383 100644 --- a/test-companion/test/starter.test.mjs +++ b/test-companion/test/starter.test.mjs @@ -200,8 +200,84 @@ describe("a comment that is not for us does not start a job", () => { assert.match(job, /startsWith\(github\.event\.comment\.body, 'approve'\)/) }) + // Evaluates the job's `if:` the way GitHub does, for the operators it uses. + const condition = job + .slice(job.indexOf("if: >-") + "if: >-".length, job.indexOf("\n #", job.indexOf("if: >-"))) + .replace(/\s+/g, " ") + .trim() + function evaluate(expr, ctx) { + const tokens = expr.match(/'[^']*'|\|\||&&|!=|==|[(),]|[A-Za-z_][\w.]*/g) + let i = 0 + const peek = () => tokens[i] + const take = () => tokens[i++] + const value = (path) => path.split(".").reduce((o, k) => (o == null ? undefined : o[k]), ctx) + const lower = (v) => String(v ?? "").toLowerCase() + function primary() { + const t = take() + if (t === "(") { + const v = or() + take() + return v + } + if (t.startsWith("'")) return t.slice(1, -1) + if (peek() === "(") { + take() + const a = or() + take() + const b = or() + take() + if (t === "contains") return lower(a).includes(lower(b)) + if (t === "startsWith") return lower(a).startsWith(lower(b)) + throw new Error(`unknown function ${t}`) + } + return value(t) + } + function compare() { + const a = primary() + if (peek() === "!=" || peek() === "==") { + const op = take() + const eq = lower(a) === lower(primary()) + return op === "==" ? eq : !eq + } + return a + } + function and() { + let v = compare() + while (peek() === "&&") { + take() + const r = compare() + v = Boolean(v) && Boolean(r) + } + return v + } + function or() { + let v = and() + while (peek() === "||") { + take() + const r = and() + v = Boolean(v) || Boolean(r) + } + return v + } + return Boolean(or()) + } + const comment = (body, type) => ({ github: { event_name: "issue_comment", event: { comment: { body, user: { type } } } } }) + test("a bot's comment, the App's own included, never boots a runner", () => { - assert.match(job, /github\.event\.comment\.user\.type != 'Bot'/) + for (const body of ["@browserstack go", "go", "approve", "Reply go to run this"]) { + assert.equal(evaluate(condition, comment(body, "Bot")), false, `a bot's "${body}" would start a runner`) + } + }) + + test("a person's request, approval or mention still boots one", () => { + for (const body of ["@browserstack explain", "go", "Go", "approve"]) { + assert.equal(evaluate(condition, comment(body, "User")), true, `"${body}" from a person was filtered out`) + } + assert.equal(evaluate(condition, comment("thanks!", "User")), false) + }) + + test("other events are never filtered", () => { + assert.equal(evaluate(condition, { github: { event_name: "pull_request", event: {} } }), true) }) test("every other event is untouched, so pushes and schedules still run", () => { From 39ec8a644b9243e65b6d7cd4a54bf77280dd589d Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Thu, 12 Nov 2026 18:10:44 +0530 Subject: [PATCH 04/31] test-companion: say the CLI's real reason, and hide credentials in comments - The reason shown for a run that broke is the CLI's own error event first, then an "Error:" line from its stderr. "Error:" lines in the agent's test output on stdout are no longer mistaken for it. - The reason and the answer are redacted before they reach a pull request comment: credential-named environment values (tokens, access keys) and user:password in URLs. GitHub masks secrets in logs, not in comments. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/answer.mjs | 48 ++++++++++++++++++---- test-companion/test/answer.test.mjs | 63 ++++++++++++++++++++++++++--- 2 files changed, 98 insertions(+), 13 deletions(-) diff --git a/test-companion/src/answer.mjs b/test-companion/src/answer.mjs index f72b838..a7a015a 100644 --- a/test-companion/src/answer.mjs +++ b/test-companion/src/answer.mjs @@ -5,21 +5,26 @@ import { StringDecoder } from "node:string_decoder" // Cap on an unterminated line; the agent's commands can print megabytes without a newline. export const MAX_PARTIAL_LINE = 1 << 20 -export function createAnswerCapture() { +/** + * `plainErrors`: also read `Error: …` lines. Only for the CLI's stderr; on stdout the + * agent's test output prints its own `Error:` lines (assertions, stack traces). + */ +export function createAnswerCapture({ plainErrors = true } = {}) { // Chunks can split a multi-byte character. const decoder = new StringDecoder("utf8") let pending = "" // Discarding the rest of a line that overran MAX_PARTIAL_LINE. let skipping = false let answer - let error + let plainError + let structuredError const read = (line) => { const t = line.trim() // The CLI prints why it stopped as `Error: …` on stderr. - const plain = /^Error:\s*(.+)$/.exec(t) + const plain = plainErrors && /^Error:\s*(.+)$/.exec(t) if (plain) { - error = plain[1] + plainError = plain[1] return } if (!t.startsWith("{") || !t.endsWith("}")) return @@ -31,7 +36,7 @@ export function createAnswerCapture() { } if (!event || typeof event !== "object") return if (event.event === "error" && typeof event.message === "string" && event.message.trim()) { - error = event.message.trim() + structuredError = event.message.trim() return } if (event.event !== "answer") return @@ -63,7 +68,9 @@ export function createAnswerCapture() { pending = "" }, answer: () => answer, - error: () => error, + structuredError: () => structuredError, + plainError: () => plainError, + error: () => structuredError ?? plainError, pending: () => pending.length, } } @@ -71,7 +78,7 @@ export function createAnswerCapture() { /** Run a command, echoing stdout live while scanning it for the answer. Never rejects. */ export function runCapturingAnswer(command, args, childEnv) { return new Promise((resolve) => { - const capture = createAnswerCapture() + const capture = createAnswerCapture({ plainErrors: false }) const errors = createAnswerCapture() const child = spawn(command, args, { stdio: ["inherit", "pipe", "pipe"], env: childEnv }) child.stdout.on("data", (chunk) => { @@ -89,7 +96,32 @@ export function runCapturingAnswer(command, args, childEnv) { child.on("close", (status) => { capture.end() errors.end() - resolve({ status, answer: capture.answer(), error: errors.error() ?? capture.error() }) + // The CLI's own structured error is the precise reason; a plain line is the fallback. + const error = capture.structuredError() ?? errors.structuredError() ?? errors.plainError() + resolve({ status, answer: capture.answer(), error }) }) }) } + +/** + * Hide credentials before text reaches a pull request comment. GitHub masks secrets in + * job logs, not in comments posted through the API. + */ +export function redactSecrets(text, secrets = []) { + if (text === undefined || text === null) return text + let out = String(text) + // Longest first, so a secret containing another is replaced whole. + const known = [...new Set(secrets.map((s) => String(s ?? "").trim()).filter((s) => s.length >= 6))].sort( + (a, b) => b.length - a.length, + ) + for (const s of known) out = out.split(s).join("***") + // user:password@ in any URL. + return out.replace(/(\/\/)[^/\s:@]+:[^/\s@]+@/g, "$1***@") +} + +/** Values of environment variables whose names mark them as credentials. */ +export function secretsFrom(env = {}) { + return Object.entries(env) + .filter(([k]) => /(TOKEN|ACCESS_KEY|PRIVATE_KEY|SECRET|PASSWORD)/i.test(k)) + .map(([, v]) => v) +} diff --git a/test-companion/test/answer.test.mjs b/test-companion/test/answer.test.mjs index de4a1e3..47721ff 100644 --- a/test-companion/test/answer.test.mjs +++ b/test-companion/test/answer.test.mjs @@ -1,6 +1,6 @@ import { test, describe } from "node:test" import assert from "node:assert/strict" -import { createAnswerCapture, runCapturingAnswer, MAX_PARTIAL_LINE } from "../src/answer.mjs" +import { createAnswerCapture, runCapturingAnswer, MAX_PARTIAL_LINE, redactSecrets, secretsFrom } from "../src/answer.mjs" // One `answer` event as the CLI prints it under --json. const said = (text) => JSON.stringify({ event: "answer", text }) + "\n" @@ -189,14 +189,28 @@ describe("the CLI's reason for stopping is kept", () => { assert.equal(r.error, "You've hit your Free credit limit.") }) - test("stderr wins: it is the CLI's own last word", async () => { - const ev = JSON.stringify({ event: "error", message: "api_request_failed" }) + test("the CLI's structured error wins over a plain Error: line", async () => { + const ev = JSON.stringify({ event: "error", message: "You've hit your Free credit limit." }) const r = await runCapturingAnswer( node, - ["-e", `process.stdout.write(${JSON.stringify(ev)} + "\\n"); process.stderr.write("Error: task failed: credit limit\\n"); process.exit(1)`], + ["-e", `process.stdout.write(${JSON.stringify(ev)} + "\\n"); process.stderr.write("Error: task failed\\n"); process.exit(1)`], process.env, ) - assert.equal(r.error, "task failed: credit limit") + assert.equal(r.error, "You've hit your Free credit limit.") + }) + + test("an Error: line in the agent's test output on stdout is not the reason", async () => { + const r = await runCapturingAnswer( + node, + ["-e", `process.stdout.write("Error: expect(received).toBe(expected)\\n"); process.stderr.write("Error: test failed\\n"); process.exit(1)`], + process.env, + ) + assert.equal(r.error, "test failed") + }) + + test("stdout alone with only test output gives no reason", async () => { + const r = await runCapturingAnswer(node, ["-e", `process.stdout.write("Error: boom\\n"); process.exit(1)`], process.env) + assert.equal(r.error, undefined) }) test("a clean run has none", async () => { @@ -204,3 +218,42 @@ describe("the CLI's reason for stopping is kept", () => { assert.equal(r.error, undefined) }) }) + +describe("redactSecrets", () => { + test("known secrets are replaced", () => { + assert.equal(redactSecrets("key abc123secret in use", ["abc123secret"]), "key *** in use") + }) + + test("credentials in a URL are hidden even when not known", () => { + assert.equal( + redactSecrets("connect https://alice:s3cr3tkey@hub-cloud.browserstack.com/wd/hub failed"), + "connect https://***@hub-cloud.browserstack.com/wd/hub failed", + ) + }) + + test("a longer secret is replaced whole before a shorter one inside it", () => { + assert.equal(redactSecrets("tok=abcdef123456", ["abcdef", "abcdef123456"]), "tok=***") + }) + + test("short or empty values are not treated as secrets", () => { + assert.equal(redactSecrets("a b c", ["", "a", undefined]), "a b c") + }) + + test("nothing to redact passes through, and undefined stays undefined", () => { + assert.equal(redactSecrets("plain"), "plain") + assert.equal(redactSecrets(undefined), undefined) + }) +}) + +describe("secretsFrom", () => { + test("credential-named variables are collected, others are not", () => { + const got = secretsFrom({ + BROWSERSTACK_ACCESS_KEY: "k1", + GH_TOKEN: "t1", + TC_ACTION_APP_PRIVATE_KEY: "p1", + BROWSERSTACK_USERNAME: "alice", + HOME: "/home/runner", + }) + assert.deepEqual(got.sort(), ["k1", "p1", "t1"]) + }) +}) From e504beacd9808334435a9f32c6781602e19b838d Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Thu, 12 Nov 2026 18:24:34 +0530 Subject: [PATCH 05/31] test-companion: always squash a handed-back branch, and never reopen closed work A handed-back fix or automate branch is now always pushed as one commit on the base, under our name, even when nothing is left out. The agent's own commits never reach GitHub, so a file it added and later deleted is not in the pushed history, and what was checked is exactly what is pushed. Pushing the branch: only an ordinary rejection means an earlier run already pushed it. A remote rejection (a protection rule, a missing permission) is reported as "could not push", never "pushed". When the branch is already there, an open pull request is reported as done, a closed or merged one is reported and not reopened, and with none a pull request is opened and said to be from the earlier run's branch. Credentials are now hidden in everything sent to GitHub (comments, replies, checks, pull requests), including the workflow token captured before the App token replaces it. The squash commit message keeps agent-chosen names on one line, so they can't add trailers, and the scratch reset no longer re-reads the checkout. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/answer.mjs | 12 ++++ test-companion/src/main.mjs | 96 ++++++++++++++++---------- test-companion/src/push-work.mjs | 70 +++++++++++++++++-- test-companion/test/answer.test.mjs | 25 ++++++- test-companion/test/push-work.test.mjs | 94 +++++++++++++++++++++++++ 5 files changed, 254 insertions(+), 43 deletions(-) diff --git a/test-companion/src/answer.mjs b/test-companion/src/answer.mjs index a7a015a..e94898f 100644 --- a/test-companion/src/answer.mjs +++ b/test-companion/src/answer.mjs @@ -97,6 +97,8 @@ export function runCapturingAnswer(command, args, childEnv) { capture.end() errors.end() // The CLI's own structured error is the precise reason; a plain line is the fallback. + // The CLI doesn't pass its child processes' stderr through, so the last plain + // `Error:` line on stderr is cobra's, i.e. the CLI's own. const error = capture.structuredError() ?? errors.structuredError() ?? errors.plainError() resolve({ status, answer: capture.answer(), error }) }) @@ -119,6 +121,16 @@ export function redactSecrets(text, secrets = []) { return out.replace(/(\/\/)[^/\s:@]+:[^/\s@]+@/g, "$1***@") } +/** redactSecrets over every string in a JSON payload about to be sent to GitHub. */ +export function redactDeep(value, secrets = []) { + if (typeof value === "string") return redactSecrets(value, secrets) + if (Array.isArray(value)) return value.map((v) => redactDeep(v, secrets)) + if (value && typeof value === "object") { + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, redactDeep(v, secrets)])) + } + return value +} + /** Values of environment variables whose names mark them as credentials. */ export function secretsFrom(env = {}) { return Object.entries(env) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 58ef6ea..1d56658 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -13,7 +13,7 @@ import { childEnvFor } from "./child-env.mjs" import { canIsolate, prepareSandbox, stopSandbox, handToAgent, writeAgentEnv, readAgentFile } from "./sandbox.mjs" import { agentEnvFrom, asAgentCommand, checkPrHandoff, readClaimResult } from "./sandbox-policy.mjs" import { startRelay } from "./relay.mjs" -import { runCapturingAnswer, redactSecrets, secretsFrom } from "./answer.mjs" +import { runCapturingAnswer, redactSecrets, redactDeep, secretsFrom } from "./answer.mjs" import { appCredentials, signJwt, @@ -76,9 +76,17 @@ import { GIT_SAFE_ARGS, commitIdentity, handedOffPrRequest, + classifyHandoffPush, + existingHandoffPr, + nextHandoffStep, + draftRejected, + handoffReport, } from "./push-work.mjs" const env = process.env +// Taken before the App token replaces GH_TOKEN and the key is deleted, so both are still hidden. +const STARTUP_SECRETS = secretsFrom(env) +const secretsNow = () => [...STARTUP_SECRETS, ...secretsFrom(env)] const REPO = env.GITHUB_REPOSITORY ?? "" const RUN_URL = `${env.GITHUB_SERVER_URL ?? "https://github.com"}/${REPO}/actions/runs/${env.GITHUB_RUN_ID ?? ""}` // RUNNER_TEMP is cleared per job; /tmp may hold an earlier job's files on a reused runner. @@ -124,7 +132,8 @@ function runGh(args, { token, input } = {}) { } function ghRaw(args, body) { - return runGh(args, { token: env.GH_TOKEN, input: body ? JSON.stringify(body) : undefined }) + // Every comment, check and pull request we send passes through here, so it's hidden once. + return runGh(args, { token: env.GH_TOKEN, input: body ? JSON.stringify(redactDeep(body, secretsNow())) : undefined }) } /** `gh api`, returning parsed JSON or null. Never throws. */ @@ -802,6 +811,7 @@ async function main() { // automate and fix hand back a pull request; other jobs save to this branch. // Re-checked with the job the pipeline chose: an event run didn't know it before the run. + // summary.workflow comes from a file the agent can write; savingFor can only make saving stricter. const workNote = handoffNote ?? saveWork(savingFor(saving, plan.workflow ?? summary.workflow), plan) const view = { @@ -823,11 +833,11 @@ async function main() { // file is the fallback, only for ANSWER_JOBS (for generate it holds test cases). answer: redactSecrets( run.answer || (hasResult && ANSWER_JOBS.has(plan.workflow ?? summary.workflow) ? readResult() : undefined), - secretsFrom(env), + secretsNow(), ), failedTests, // Only where the run didn't work, so a stray error line never colours a success. - errorLine: runDidNotWork(outcome) ? redactSecrets(run.error, secretsFrom(env)) : undefined, + errorLine: runDidNotWork(outcome) ? redactSecrets(run.error, secretsNow()) : undefined, } // Proposing: post what it would do and stop. A run that broke has nothing to approve, // so it is reported like any other run, which also completes its check. @@ -918,7 +928,7 @@ function saveWork(saving, plan) { if (unexpected.length) { return [ `**Could not save this work to \`${branch}\`:** the repository's git config carries`, - `${unexpected.map((k) => `\`${k}\``).join(", ")}, which \`actions/checkout\` does not write. Config is`, + `${listFiles(unexpected)}, which \`actions/checkout\` does not write. Config is`, 'where git keeps its "run this program" hooks and its URL rewrites, so Test Companion will not', "commit against a config it does not recognise. Nothing was committed or pushed — the run's", "changes are in the job's workspace only.", @@ -1130,7 +1140,7 @@ function openHandedOffPr(path, plan) { if (!GIT_BIN) return "**Could not open the pull request:** no trusted `git` was found on this runner." const unexpected = unexpectedConfigKeys(git(CONFIG_READ_ARGS).raw) if (unexpected.length) { - return `**Did not open a pull request:** the repository's git config carries ${unexpected.map((k) => `\`${k}\``).join(", ")}, which \`actions/checkout\` does not write.` + return `**Did not open a pull request:** the repository's git config carries ${listFiles(unexpected)}, which \`actions/checkout\` does not write.` } const tip = git(["rev-parse", "--verify", "--quiet", `refs/heads/${req.branch}^{commit}`]) if (!tip.ok || !tip.out) return `**Did not open a pull request:** the run did not leave a \`${req.branch}\` branch.` @@ -1151,21 +1161,31 @@ function openHandedOffPr(path, plan) { if (refused.length) { return `**Did not open a pull request:** the branch changes files Test Companion never pushes: ${listFiles(refused)}.` } - let head = tip.out - if (dropped.length) { - const indexFile = join(REPORT_DIR, `tc-handoff-index-${process.pid}`) - let left - try { - left = leaveOut({ git, base: from.out, tip: tip.out, paths: dropped, identity, message: leaveOutMessage(req.title, dropped), indexFile }) - } finally { - rmSync(indexFile, { force: true }) - } - if (left.error) return `**Could not open the pull request:** ${left.error}.` - if (left.empty) { - return `**Did not open a pull request:** the run's only changes were files Test Companion never commits: ${listFiles(dropped)}.` - } - head = left.sha + // Always one squashed commit on the base, under our name: the agent's own commits (and + // anything it added then deleted in them) never reach GitHub, and what was checked above is + // exactly what is pushed. + const indexFile = join(REPORT_DIR, `tc-handoff-index-${process.pid}`) + let left + try { + left = leaveOut({ + git, + base: from.out, + tip: tip.out, + paths: dropped, + identity, + message: redactSecrets(leaveOutMessage(req.title, dropped), secretsNow()), + indexFile, + }) + } finally { + rmSync(indexFile, { force: true }) + } + if (left.error) return `**Could not open the pull request:** ${left.error}.` + if (left.empty) { + return dropped.length + ? `**Did not open a pull request:** the run's only changes were files Test Companion never commits: ${listFiles(dropped)}.` + : `**Did not open a pull request:** the run's branch has no changes.` } + const head = left.sha const leftOutNote = dropped.length ? ` Left out ${listFiles(dropped)}, which Test Companion never commits.` : "" const target = pushUrlFor(REPO, env.GITHUB_SERVER_URL) if (!target) return `**Could not open the pull request:** bad repository name "${REPO}".` @@ -1182,25 +1202,27 @@ function openHandedOffPr(path, plan) { ? ` It targets \`${request.base}\`, not #${plan.context.pr}'s branch, because the run did not start on that pull request's commit.` : "" const pushed = git(["push", target, `${head}:refs/heads/${req.branch}`], pushCredentialEnv(env.GH_TOKEN, target)) - if (!pushed.ok && !/rejected|non-fast-forward|already exists/i.test(pushed.err)) { - return `**Could not open the pull request:** ${pushed.err || "git push failed"}.` + const pushKind = classifyHandoffPush(pushed) + let existing + if (pushKind === "already") { + // The branch is from an earlier run of this job on this commit; never reopen its closed PR. + const owner = REPO.split("/")[0] + existing = existingHandoffPr(gh(["api", `repos/${REPO}/pulls?head=${encodeURIComponent(`${owner}:${req.branch}`)}&state=all`])) } - // The branch may be left from an earlier run; only claim "done" if its pull request exists. - const owner = REPO.split("/")[0] - const open = gh(["api", `repos/${REPO}/pulls?head=${encodeURIComponent(`${owner}:${req.branch}`)}&state=open`]) - if (Array.isArray(open) && open.length) { - return `**Already done:** this job's pull request is already open: ${open[0].html_url}.` + const step = nextHandoffStep(pushKind, existing) + if (step.action === "stop") { + return `${handoffReport(step, { branch: req.branch })} ${pushed.err ? `${pushed.err.split("\n").filter(Boolean).pop()}.` : ""}`.trim() } - // Push rejected and no pull request: an earlier run of this job on this commit pushed the - // branch but couldn't open its pull request, so open it now. + if (step.action === "report") return handoffReport(step, { branch: req.branch }) + const create = (body) => ghRaw(["api", `repos/${REPO}/pulls`, "--method", "POST", "--input", "-"], body) let opened = create(request) - // Draft pull requests aren't available on every plan. - if (opened.status !== 0 && /draft/i.test(`${opened.stdout}${opened.stderr}`)) { - opened = create({ ...request, draft: false }) - } + if (draftRejected(opened)) opened = create({ ...request, draft: false }) if (opened.status !== 0) { - return `**Pushed \`${req.branch}\` but could not open the pull request:** ${String(opened.stderr ?? "").trim().split("\n")[0]}.` + const why = String(opened.stderr ?? "").trim().split("\n")[0] + return step.earlier + ? `**Could not open a pull request** for an earlier run's ${listFiles([req.branch])}: ${why}.` + : `**Pushed ${listFiles([req.branch])} but could not open the pull request:** ${why}.` } let url = "" try { @@ -1208,7 +1230,11 @@ function openHandedOffPr(path, plan) { } catch { // The link is a nicety. } - return `**Opened a pull request** with this run's changes${url ? `: ${url}` : ` from \`${req.branch}\``}.${leftOutNote}${baseNote}` + const where = url ? `: ${url}` : ` from ${listFiles([req.branch])}` + // This run's notes describe this run's tree, not what an earlier run left on the branch. + return step.earlier + ? `**Opened a pull request** from an earlier run's branch for this commit${where}.` + : `**Opened a pull request** with this run's changes${where}.${leftOutNote}${baseNote}` } diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index 231e809..5f3746b 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -401,10 +401,17 @@ export function handedOffPrRequest({ req, pr, prBranch, defaultBranch, onReviewe export function leaveOut({ git, base, tip, paths, identity, message, indexFile }) { const scratch = { GIT_INDEX_FILE: indexFile, GIT_LITERAL_PATHSPECS: "1" } if (!git(["read-tree", tip], scratch).ok) return { error: "could not read the branch" } - // Resetting to `base` restores a changed file and removes an added one. Paths on stdin: - // no argument-length limit, and no name read as an option. - const reset = git(["reset", "-q", base, "--pathspec-from-file=-", "--pathspec-file-nul"], scratch, paths.join("\0")) - if (!reset.ok) return { error: "could not leave out the by-products" } + if (paths.length) { + // Resetting to `base` restores a changed file and removes an added one. Paths on stdin: + // no argument-length limit, and no name read as an option. --no-refresh: don't re-read + // the checkout, which would run its clean filters. + const reset = git( + ["reset", "-q", "--no-refresh", base, "--pathspec-from-file=-", "--pathspec-file-nul"], + scratch, + paths.join("\0"), + ) + if (!reset.ok) return { error: "could not leave out the by-products" } + } const tree = git(["write-tree"], scratch) if (!tree.ok || !tree.out) return { error: "could not write the branch's files" } if (tree.out === git(["rev-parse", `${base}^{tree}`]).out) return { empty: true } @@ -415,7 +422,56 @@ export function leaveOut({ git, base, tip, paths, identity, message, indexFile } return commit.ok && commit.out ? { sha: commit.out } : { error: "could not commit without the by-products" } } -/** The commit message for a hand-off with files left out. */ -export function leaveOutMessage(title, paths) { - return `${String(title ?? "").trim() || "Test Companion changes"}\n\nLeft out, because Test Companion never commits them:\n${listFiles(paths, { render: (p) => `- ${p}`, separator: "\n" })}` +// Agent-chosen text on one line, so it can't add a trailer such as Co-authored-by. +const oneLine = (s) => String(s ?? "").replace(/[\x00-\x1f\x7f]+/g, " ").trim() + +/** The commit message for a hand-off: its title, and any files left out. */ +export function leaveOutMessage(title, paths = []) { + const head = oneLine(title) || "Test Companion changes" + if (!paths.length) return head + return `${head}\n\nLeft out, because Test Companion never commits them:\n${listFiles(paths, { render: (p) => `- ${oneLine(p)}`, separator: "\n" })}` +} + +/** + * What a hand-off branch push means. Only an ordinary rejection says the branch is already + * there; `[remote rejected]` (a protection rule, a missing permission) means nothing was pushed. + */ +export function classifyHandoffPush({ ok, err } = {}) { + if (ok) return "pushed" + return /!\s*\[rejected\][^\n]*\((non-fast-forward|fetch first)\)/i.test(String(err ?? "")) ? "already" : "failed" +} + +/** The pull request an earlier run opened for this branch, from a state=all listing. */ +export function existingHandoffPr(prs) { + const list = Array.isArray(prs) ? prs.filter(Boolean) : [] + const open = list.find((p) => p.state === "open") + if (open) return { kind: "open", url: open.html_url } + const closed = list.find((p) => p.state === "closed") + if (closed) return { kind: closed.merged_at ? "merged" : "closed", url: closed.html_url } + return { kind: "none" } +} + +/** Next step after the push: stop, report an existing pull request, or open one. */ +export function nextHandoffStep(pushKind, existing = { kind: "none" }) { + if (pushKind === "failed") return { action: "stop" } + if (pushKind === "pushed") return { action: "create", earlier: false } + if (existing.kind === "none") return { action: "create", earlier: true } + // A closed or merged one is reported, never reopened. + return { action: "report", ...existing } +} + +/** Did creating the pull request fail only because drafts aren't available on this plan? */ +export function draftRejected(result) { + return Boolean(result) && result.status !== 0 && /draft/i.test(`${result.stdout ?? ""}${result.stderr ?? ""}`) +} + +/** The comment line for a hand-off that did not end in a new pull request of this run's work. */ +export function handoffReport(step, { branch } = {}) { + if (step.action === "report" && step.kind === "open") { + return `**Already done:** this job's pull request is already open: ${step.url}.` + } + if (step.action === "report") { + return `**Already done:** an earlier run's pull request for this commit was ${step.kind}: ${step.url}. Push a new commit to run this job again.` + } + return `**Could not push ${codeSpan(branch)}.**` } diff --git a/test-companion/test/answer.test.mjs b/test-companion/test/answer.test.mjs index 47721ff..97c6ec9 100644 --- a/test-companion/test/answer.test.mjs +++ b/test-companion/test/answer.test.mjs @@ -1,6 +1,6 @@ import { test, describe } from "node:test" import assert from "node:assert/strict" -import { createAnswerCapture, runCapturingAnswer, MAX_PARTIAL_LINE, redactSecrets, secretsFrom } from "../src/answer.mjs" +import { createAnswerCapture, runCapturingAnswer, MAX_PARTIAL_LINE, redactSecrets, redactDeep, secretsFrom } from "../src/answer.mjs" // One `answer` event as the CLI prints it under --json. const said = (text) => JSON.stringify({ event: "answer", text }) + "\n" @@ -257,3 +257,26 @@ describe("secretsFrom", () => { assert.deepEqual(got.sort(), ["k1", "p1", "t1"]) }) }) + +describe("redactDeep — every string in a payload sent to GitHub", () => { + test("hides secrets in nested comment, check and pull request payloads", () => { + const payload = { + body: "ran with key abc123secret", + output: { title: "t", summary: "https://u:p4ss@hub/x", annotations: [{ message: "abc123secret" }] }, + draft: true, + n: 3, + } + assert.deepEqual(redactDeep(payload, ["abc123secret"]), { + body: "ran with key ***", + output: { title: "t", summary: "https://***@hub/x", annotations: [{ message: "***" }] }, + draft: true, + n: 3, + }) + }) + + test("the original token stays hidden after the environment changes", () => { + const startup = secretsFrom({ GH_TOKEN: "ghs_workflowtoken1" }) + const later = secretsFrom({ GH_TOKEN: "ghs_apptoken22222" }) + assert.equal(redactDeep({ body: "ghs_workflowtoken1 ghs_apptoken22222" }, [...startup, ...later]).body, "*** ***") + }) +}) diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index 72ce6a7..9289734 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -29,6 +29,11 @@ import { savingFor, leaveOut, leaveOutMessage, + classifyHandoffPush, + existingHandoffPr, + nextHandoffStep, + draftRejected, + handoffReport, codeSpan, listFiles, APP_SLUGS, @@ -831,6 +836,95 @@ describe("leaveOut — by-products never reach the pushed branch", () => { }) }) +describe("leaveOut always squashes the agent's history", () => { + const identity = { name: "Test Companion", email: BOT_EMAIL } + + test("a .env added then deleted in the agent's commits is in no pushed commit, and the author is ours", () => { + const { dir, git, base } = repoWithBranch({ "a.ts": "new\n", ".env": "KEY=secret" }) + const id = ["-c", "user.name=agent", "-c", "user.email=agent@example.com"] + assert.ok(git(["rm", "-q", "--", ".env"]).ok) + assert.ok(git([...id, "commit", "-qm", "drop env"]).ok) + const tip = git(["rev-parse", "HEAD"]).out + // Nothing to leave out in the final tree, but the history still holds .env. + const out = leaveOut({ git, base, tip, paths: [], identity, message: "m", indexFile: join(dir, "scratch") }) + assert.ok(out.sha, JSON.stringify(out)) + const range = git(["rev-list", `${base}..${out.sha}`]).out.split("\n") + assert.deepEqual(range, [out.sha]) + for (const c of range) assert.equal(git(["cat-file", "-e", `${c}:.env`]).ok, false) + assert.equal(git(["log", "-1", "--format=%an <%ae>", out.sha]).out, `Test Companion <${BOT_EMAIL}>`) + assert.equal(git(["show", `${out.sha}:a.ts`]).out, "new") + rmSync(dir, { recursive: true, force: true }) + }) + + test("a branch with no net change is reported as empty", () => { + const { dir, git, base } = repoWithBranch({ "a.ts": "new\n" }) + writeFileSync(join(dir, "a.ts"), "old\n") + assert.ok(git(["-c", "user.name=agent", "-c", "user.email=agent@example.com", "commit", "-qam", "undo"]).ok) + const tip = git(["rev-parse", "HEAD"]).out + assert.deepEqual(leaveOut({ git, base, tip, paths: [], identity, message: "m", indexFile: join(dir, "scratch") }), { empty: true }) + rmSync(dir, { recursive: true, force: true }) + }) + + test("an agent-chosen file name can't add a trailer to the commit message", () => { + const msg = leaveOutMessage("Fix\n\nCo-authored-by: x ", ["x\n\nCo-authored-by: Someone "]) + assert.equal(msg.split("\n").some((line) => /^co-authored-by:/i.test(line)), false) + assert.match(msg, /^Fix Co-authored-by: x \n/) + }) + + test("with nothing left out, the message is just the title", () => { + assert.equal(leaveOutMessage("Fix the cart total", []), "Fix the cart total") + }) +}) + +describe("handing back a pull request: what happened, and what to do next", () => { + test("an ordinary rejection means the branch is already there", () => { + const err = "To https://github.com/o/r.git\n ! [rejected] abc -> tc/fix/abc (non-fast-forward)\nerror: failed to push" + assert.equal(classifyHandoffPush({ ok: false, err }), "already") + assert.equal(classifyHandoffPush({ ok: false, err: " ! [rejected] abc -> b (fetch first)" }), "already") + }) + + test("a remote rejection (protection rule, missing permission) means nothing was pushed", () => { + const err = " ! [remote rejected] abc -> tc/fix/abc (refusing to allow a GitHub App to create or update workflow)" + assert.equal(classifyHandoffPush({ ok: false, err }), "failed") + assert.equal(classifyHandoffPush({ ok: false, err: "fatal: unable to access" }), "failed") + assert.equal(classifyHandoffPush({ ok: true }), "pushed") + }) + + test("an open pull request wins; a closed or merged one is reported, never reopened", () => { + const open = { state: "open", html_url: "https://x/1" } + const closed = { state: "closed", html_url: "https://x/2", merged_at: null } + const merged = { state: "closed", html_url: "https://x/3", merged_at: "2026-01-01" } + assert.deepEqual(existingHandoffPr([closed, open]), { kind: "open", url: "https://x/1" }) + assert.deepEqual(existingHandoffPr([closed]), { kind: "closed", url: "https://x/2" }) + assert.deepEqual(existingHandoffPr([merged]), { kind: "merged", url: "https://x/3" }) + assert.deepEqual(existingHandoffPr(null), { kind: "none" }) + }) + + test("next step for each push outcome", () => { + assert.deepEqual(nextHandoffStep("failed"), { action: "stop" }) + assert.deepEqual(nextHandoffStep("pushed"), { action: "create", earlier: false }) + assert.deepEqual(nextHandoffStep("already", { kind: "none" }), { action: "create", earlier: true }) + assert.deepEqual(nextHandoffStep("already", { kind: "closed", url: "u" }), { action: "report", kind: "closed", url: "u" }) + assert.deepEqual(nextHandoffStep("already", { kind: "open", url: "u" }), { action: "report", kind: "open", url: "u" }) + }) + + test("the report never claims a push or reopens closed work", () => { + assert.match(handoffReport({ action: "report", kind: "open", url: "u" }), /^\*\*Already done:\*\* .*already open: u\.$/) + const closed = handoffReport({ action: "report", kind: "closed", url: "u" }) + assert.match(closed, /was closed: u\. Push a new commit/) + assert.match(handoffReport({ action: "report", kind: "merged", url: "u" }), /was merged: u\./) + const stop = handoffReport({ action: "stop" }, { branch: "tc/fix/abc" }) + assert.match(stop, /^\*\*Could not push `tc\/fix\/abc`\.\*\*$/) + assert.doesNotMatch(stop, /Pushed/) + }) + + test("a draft rejection is recognised; other failures are not", () => { + assert.equal(draftRejected({ status: 1, stderr: "Draft pull requests are not supported in this repository" }), true) + assert.equal(draftRejected({ status: 1, stderr: "Validation Failed: No commits between" }), false) + assert.equal(draftRejected({ status: 0, stdout: "{}" }), false) + }) +}) + describe("git reads file names literally", () => { test("the Action's git environment turns pathspec magic off", () => { assert.equal(gitEnvFrom({}).GIT_LITERAL_PATHSPECS, "1") From 0d6c8d9b50a6872c6b4a7b0f631d778c00f183f4 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Thu, 12 Nov 2026 18:39:05 +0530 Subject: [PATCH 06/31] test-companion: say why a hand-off push was refused, and never guess after a failed lookup A refused push now reports git's actual reason (the [remote rejected] line, a remote error such as a ruleset, or a fatal line) instead of the generic "failed to push some refs". A failed lookup for an earlier run's pull request now stops with a plain message instead of being read as "no pull request" and opening one, which could have reopened closed work. leaveOut retries its reset without --no-refresh on a git too old to know the option. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/main.mjs | 4 +- test-companion/src/push-work.mjs | 36 +++++++++++++++--- test-companion/test/push-work.test.mjs | 52 +++++++++++++++++++++++++- 3 files changed, 84 insertions(+), 8 deletions(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 1d56658..8bc23ad 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -77,6 +77,7 @@ import { commitIdentity, handedOffPrRequest, classifyHandoffPush, + pushFailureReason, existingHandoffPr, nextHandoffStep, draftRejected, @@ -1211,7 +1212,8 @@ function openHandedOffPr(path, plan) { } const step = nextHandoffStep(pushKind, existing) if (step.action === "stop") { - return `${handoffReport(step, { branch: req.branch })} ${pushed.err ? `${pushed.err.split("\n").filter(Boolean).pop()}.` : ""}`.trim() + const why = step.unknown ? "" : pushFailureReason(pushed.err) + return `${handoffReport(step, { branch: req.branch })} ${why ? `${why}.` : ""}`.trim() } if (step.action === "report") return handoffReport(step, { branch: req.branch }) diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index 5f3746b..2903bcb 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -405,11 +405,11 @@ export function leaveOut({ git, base, tip, paths, identity, message, indexFile } // Resetting to `base` restores a changed file and removes an added one. Paths on stdin: // no argument-length limit, and no name read as an option. --no-refresh: don't re-read // the checkout, which would run its clean filters. - const reset = git( - ["reset", "-q", "--no-refresh", base, "--pathspec-from-file=-", "--pathspec-file-nul"], - scratch, - paths.join("\0"), - ) + const resetWith = (extra) => + git(["reset", "-q", ...extra, base, "--pathspec-from-file=-", "--pathspec-file-nul"], scratch, paths.join("\0")) + let reset = resetWith(["--no-refresh"]) + // --no-refresh needs git 2.36; an older self-hosted runner rejects the option. + if (!reset.ok && /unknown option|usage: git reset/i.test(String(reset.err ?? ""))) reset = resetWith([]) if (!reset.ok) return { error: "could not leave out the by-products" } } const tree = git(["write-tree"], scratch) @@ -441,9 +441,29 @@ export function classifyHandoffPush({ ok, err } = {}) { return /!\s*\[rejected\][^\n]*\((non-fast-forward|fetch first)\)/i.test(String(err ?? "")) ? "already" : "failed" } +/** + * The line of a failed `git push` that says why. git's last line is always the generic + * "failed to push some refs"; the reason (a ruleset, a hook, a missing permission) is earlier. + */ +export function pushFailureReason(err) { + const lines = String(err ?? "") + .split("\n") + .map((l) => l.trim()) + .filter(Boolean) + return ( + lines.find((l) => /^!\s*\[remote rejected\]/i.test(l)) ?? + lines.find((l) => /^remote:\s*(error|GH\d+)/i.test(l)) ?? + lines.find((l) => /^fatal:/i.test(l)) ?? + lines.at(-1) ?? + "" + ) +} + /** The pull request an earlier run opened for this branch, from a state=all listing. */ export function existingHandoffPr(prs) { - const list = Array.isArray(prs) ? prs.filter(Boolean) : [] + // A failed lookup is not "no pull request": treating it so could reopen closed work. + if (!Array.isArray(prs)) return { kind: "unknown" } + const list = prs.filter(Boolean) const open = list.find((p) => p.state === "open") if (open) return { kind: "open", url: open.html_url } const closed = list.find((p) => p.state === "closed") @@ -455,6 +475,7 @@ export function existingHandoffPr(prs) { export function nextHandoffStep(pushKind, existing = { kind: "none" }) { if (pushKind === "failed") return { action: "stop" } if (pushKind === "pushed") return { action: "create", earlier: false } + if (existing.kind === "unknown") return { action: "stop", unknown: true } if (existing.kind === "none") return { action: "create", earlier: true } // A closed or merged one is reported, never reopened. return { action: "report", ...existing } @@ -473,5 +494,8 @@ export function handoffReport(step, { branch } = {}) { if (step.action === "report") { return `**Already done:** an earlier run's pull request for this commit was ${step.kind}: ${step.url}. Push a new commit to run this job again.` } + if (step.unknown) { + return `**Did not open a pull request:** ${codeSpan(branch)} already exists from an earlier run, and Test Companion could not check whether that run's pull request was closed. Look for it, or push a new commit to run this job again.` + } return `**Could not push ${codeSpan(branch)}.**` } diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index 9289734..736ccc8 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -30,6 +30,7 @@ import { leaveOut, leaveOutMessage, classifyHandoffPush, + pushFailureReason, existingHandoffPr, nextHandoffStep, draftRejected, @@ -897,7 +898,7 @@ describe("handing back a pull request: what happened, and what to do next", () = assert.deepEqual(existingHandoffPr([closed, open]), { kind: "open", url: "https://x/1" }) assert.deepEqual(existingHandoffPr([closed]), { kind: "closed", url: "https://x/2" }) assert.deepEqual(existingHandoffPr([merged]), { kind: "merged", url: "https://x/3" }) - assert.deepEqual(existingHandoffPr(null), { kind: "none" }) + assert.deepEqual(existingHandoffPr([]), { kind: "none" }) }) test("next step for each push outcome", () => { @@ -997,3 +998,52 @@ describe("file names in comments", () => { assert.match(out, /and 3 more$/) }) }) + +describe("hand-off: failed lookups and push reasons", () => { + test("a failed PR lookup is unknown, and stops rather than opening a pull request", () => { + assert.deepEqual(existingHandoffPr(null), { kind: "unknown" }) + assert.deepEqual(existingHandoffPr({ message: "Not Found" }), { kind: "unknown" }) + const step = nextHandoffStep("already", existingHandoffPr(null)) + assert.deepEqual(step, { action: "stop", unknown: true }) + assert.match(handoffReport(step, { branch: "tc/fix/abc" }), /could not check whether that run's pull request was closed/) + }) + + test("a server-side refusal reports the rule, not git's generic last line", () => { + const err = [ + "remote: error: GH013: Repository rule violations found for refs/heads/tc/fix/abc.", + "To https://github.com/o/r.git", + " ! [remote rejected] abc -> tc/fix/abc (push declined due to repository rule violations)", + "error: failed to push some refs to 'https://github.com/o/r.git'", + ].join("\n") + assert.match(pushFailureReason(err), /^! \[remote rejected\].*repository rule violations/) + }) + + test("a hook refusal without [remote rejected] falls back to the remote error line", () => { + const err = "remote: error: commits must be signed\nerror: failed to push some refs to 'x'" + assert.equal(pushFailureReason(err), "remote: error: commits must be signed") + }) + + test("fatal lines and plain last lines are used when nothing better is there", () => { + assert.equal(pushFailureReason("fatal: unable to access 'x': 403\n"), "fatal: unable to access 'x': 403") + assert.equal(pushFailureReason("something odd"), "something odd") + assert.equal(pushFailureReason(""), "") + }) +}) + +describe("leaveOut on a git without --no-refresh", () => { + test("retries the reset without the option", () => { + const calls = [] + const git = (args, _env, input) => { + calls.push(args) + if (args[0] === "reset" && args.includes("--no-refresh")) return { ok: false, err: "error: unknown option `no-refresh'" } + if (args[0] === "write-tree") return { ok: true, out: "tree2" } + if (args[0] === "rev-parse") return { ok: true, out: "tree1" } + if (args.includes("commit-tree")) return { ok: true, out: "sha2" } + return { ok: true, out: "" } + } + const r = leaveOut({ git, base: "b", tip: "t", paths: ["browserstack.err"], identity: { name: "n", email: "e" }, message: "m", indexFile: "/tmp/x" }) + assert.deepEqual(r, { sha: "sha2" }) + assert.equal(calls.filter((a) => a[0] === "reset").length, 2) + assert.ok(!calls.filter((a) => a[0] === "reset")[1].includes("--no-refresh")) + }) +}) From 0f6271c165f07b56acc4e25735bcb9146e7208a8 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Thu, 12 Nov 2026 19:46:40 +0530 Subject: [PATCH 07/31] test-companion: one reply per request, newest plan wins, and say which build was read - Each request (an @browserstack comment, or a `go`) now gets its own reply with that run's report, instead of rewriting one shared comment. Automatic runs, which answer no request, keep one status comment. - A spent plan no longer blocks a newer plan for the same job on the same commit: spent markers now name the plan they spent. Older markers still spend the plans posted before them. `go #` on a spent plan says it was already approved or dropped. - A run a person asked for or approved may save on top of an earlier Test Companion commit; the loop guard now applies to automatic runs. - explain, fix and find-the-bad-PR say which failed run they read and list the other failures. A comment can name a run: its link, `run `, or a word from the workflow's name. A named run must be in this repository, on one of the pull request's commits, and red. - When nothing has failed yet but CI is still running, the reply says so and lists what is running, instead of "Nothing has failed". - Read-only jobs (explain, find the bad PR) no longer show the preview banner, which read as "nothing happened"; they say they're read-only. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/main.mjs | 107 +++++++++++--- test-companion/src/push-work.mjs | 19 ++- test-companion/src/report.mjs | 88 ++++++++--- test-companion/src/runs.mjs | 106 ++++++++++++++ test-companion/test/plan-approval.test.mjs | 83 +++++++++++ .../test/reply-per-request.test.mjs | 91 ++++++++++++ test-companion/test/run-selection.test.mjs | 137 ++++++++++++++++++ 7 files changed, 587 insertions(+), 44 deletions(-) create mode 100644 test-companion/test/plan-approval.test.mjs create mode 100644 test-companion/test/reply-per-request.test.mjs create mode 100644 test-companion/test/run-selection.test.mjs diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 8bc23ad..228b2b8 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -5,7 +5,16 @@ import { join } from "node:path" import { tmpdir } from "node:os" import { planRun, contextArgs, withPullRequestHead, TC_EVENTS } from "./plan-run.mjs" import { readIntent, INTENT } from "./intent.mjs" -import { chooseRunsToCancel, runIdsFromChecks, latestFailedRun } from "./runs.mjs" +import { + chooseRunsToCancel, + runIdsFromChecks, + ciOnCommit, + parseRunReference, + matchFailedByName, + checkNamedRun, + renderCiStillRunning, + renderBuildNote, +} from "./runs.mjs" import { decideGate, GATE, describeChangeSize, ANSWER_JOBS } from "./gate.mjs" import { mayAsk, describeTestRepoAccess } from "./access.mjs" import { findTicket } from "./ticket.mjs" @@ -26,6 +35,7 @@ import { renderJobSummary, renderCheckSummary, renderComment, + requestMarkerFor, chooseCommentAction, checkSha, checkSubjectFor, @@ -40,6 +50,7 @@ import { renderStopRemoved, proposalConsumedMarkerFor, proposalAlreadyUsed, + planSpentState, proposalNeedsDropping, flattenPages, hasApproverEdits, @@ -177,6 +188,9 @@ const ALLOW_UNISOLATED = /^true$/i.test(String(env.TC_ACTION_ALLOW_UNISOLATED ?? /** The proposal this run is carrying out, so the report can link it. */ let approvedPlan +/** Which failed run a build-reading job looked at, and what it left out. */ +let buildNote = "" + // No --depth: it would make the full clone shallow. The URL carries its own credential // because the starter sets persist-credentials: false. function fetchAndCheckout(sha, branch) { @@ -471,6 +485,8 @@ async function main() { const CARRIES_A_PLAN = new Set(["test-pr"]) // Jobs that cannot run without knowing which build failed. const NEEDS_A_BUILD = new Set(["rca", "fix", "verdict"]) + // How to ask each of those for a named run. + const ASK_WORD = { rca: "explain", fix: "fix", verdict: "find the bad PR" } const COMMENT_TRIGGERS = new Set(["issue_comment"]) if (COMMENT_TRIGGERS.has(env.GITHUB_EVENT_NAME) || plan.requiresWriteAccess) { const who = plan.commenter ?? readEvent().comment?.user?.login @@ -491,6 +507,8 @@ async function main() { let proposing = false let approvalNote = "" if (plan.event === TC_EVENTS.PR_COMMENT) { + // Kept for naming a run: an approval deletes plan.context.instructions below. + const requestText = plan.context.instructions // Fetched before reading intent: with a plan waiting, "go, and fix case 3" is an approval with an edit. const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${plan.context.pr}/comments`, "--paginate", "--slurp"])) let waiting = findProposal(comments, plan.context.sha, Date.now(), botLogin) @@ -519,11 +537,15 @@ async function main() { if (intent.kind === INTENT.GO && intent.proposalId) { waiting = findProposal(comments, plan.context.sha, Date.now(), botLogin, { id: intent.proposalId }) if (!waiting) { + const spentAs = planSpentState(comments, intent.proposalId, botLogin) replyOnPr( plan.context.pr, - `I have no plan \`#${intent.proposalId}\` still waiting on \`${String(plan.context.sha).slice(0, 7)}\`. ` + - `It may have been approved, dropped, or written against an earlier commit. ` + - `Reply **go** on its own to approve the most recent one.`, + spentAs + ? `Plan \`#${intent.proposalId}\` was already ${spentAs === "approved" ? "approved and run" : "dropped"}, so there is nothing left to approve on it. ` + + `Reply **go** on its own to approve the newest plan, or ask me again for a fresh one.` + : `I have no plan \`#${intent.proposalId}\` still waiting on \`${String(plan.context.sha).slice(0, 7)}\`. ` + + `It may have been written against an earlier commit. ` + + `Reply **go** on its own to approve the most recent one.`, ) finishSkipped("proposal-not-found") } @@ -614,7 +636,7 @@ async function main() { ) if (pending) rememberCheck(plan.context.sha, pending) // Per job, so approving one doesn't spend another job's plan. - consumedMarker = proposalConsumedMarkerFor(plan.context.sha, plan.workflow) + consumedMarker = proposalConsumedMarkerFor(plan.context.sha, plan.workflow, waiting?.id) approvedPlan = waiting?.url ? { id: waiting.id, url: waiting.url } : undefined // The approved plan, edits included, goes in a file: --context strips punctuation. @@ -647,22 +669,54 @@ async function main() { if (gate.preview || intent.preview) plan.preview = true // Last, because the job isn't final until approval is handled ("go" names none). - // rca, fix and verdict need a failed build: default to the newest red run on this commit. + // rca, fix and verdict need a failed build: one the comment names, else the newest red run on this commit. if (NEEDS_A_BUILD.has(plan.workflow) && !plan.context.build) { - const runs = gh(["api", `repos/${REPO}/actions/runs?head_sha=${plan.context.sha}&per_page=100`]) - const failed = latestFailedRun(runs?.workflow_runs, { - selfRunId: env.GITHUB_RUN_ID, - selfWorkflowName: env.TC_ACTION_SELF_WORKFLOW, - }) - if (!failed) { - replyOnPr( - plan.context.pr, - `Nothing has failed on \`${String(plan.context.sha).slice(0, 7)}\`, so there is no build for me to look at. Re-run me once something is red, or say what you want tested instead.`, - ) - finishSkipped("nothing-failed", plan.workflow) + const short = String(plan.context.sha).slice(0, 7) + const named = parseRunReference(requestText, { repo: REPO, serverUrl: env.GITHUB_SERVER_URL }) + if (named?.error) { + replyOnPr(plan.context.pr, named.error) + finishSkipped("run-not-usable", "link") } - plan.context.build = failed - notice(`Test Companion: reading the failure from ${failed}`) + if (named?.id) { + const prShas = flattenPages(gh(["api", `repos/${REPO}/pulls/${plan.context.pr}/commits?per_page=100`, "--paginate", "--slurp"])) + .map((c) => c?.sha) + .concat(plan.context.sha) + const check = checkNamedRun(gh(["api", `repos/${REPO}/actions/runs/${named.id}`]), { + prShas, + selfRunId: env.GITHUB_RUN_ID, + selfWorkflowName: env.TC_ACTION_SELF_WORKFLOW, + }) + if (!check.ok) { + replyOnPr(plan.context.pr, `I can't use run \`${named.id}\`: ${check.why}`) + finishSkipped("run-not-usable", named.id) + } + plan.context.build = check.url + buildNote = renderBuildNote(check, { ask: ASK_WORD[plan.workflow] }) + } else { + const runs = gh(["api", `repos/${REPO}/actions/runs?head_sha=${plan.context.sha}&per_page=100`]) + const ci = ciOnCommit(runs?.workflow_runs, { + selfRunId: env.GITHUB_RUN_ID, + selfWorkflowName: env.TC_ACTION_SELF_WORKFLOW, + }) + const chosen = matchFailedByName(requestText, ci.failed) ?? ci.failed[0] + if (!chosen) { + // A build still going is not "nothing failed": its failures just aren't in yet. + replyOnPr( + plan.context.pr, + ci.running.length + ? renderCiStillRunning(plan.context.sha, ci.running) + : `Nothing has failed on \`${short}\`, so there is no build for me to look at. Re-run me once something is red, or say what you want tested instead.`, + ) + finishSkipped(ci.running.length ? "ci-still-running" : "nothing-failed", plan.workflow) + } + plan.context.build = chosen.url + buildNote = renderBuildNote(chosen, { + otherFailed: ci.failed.filter((r) => r !== chosen), + running: ci.running, + ask: ASK_WORD[plan.workflow], + }) + } + notice(`Test Companion: reading the failure from ${plan.context.build}`) } } @@ -695,6 +749,8 @@ async function main() { onWantedCommit: !needsCheckout({ headSha: git(["rev-parse", "HEAD"]).out, wantSha: plan.context.sha }), defaultBranch: defaultBranchName(), workflow: plan.workflow, + // Asked for in a comment (bots are filtered out earlier) or a manual run: may build on our commit. + requestedByPerson: plan.event === TC_EVENTS.PR_COMMENT || env.GITHUB_EVENT_NAME === "workflow_dispatch", }) if (plan.context.branch && !plan.preview && !saving.ok && !saving.quiet) { notice(`Nothing will be saved to ${plan.context.branch}: ${saving.why}.`) @@ -839,6 +895,7 @@ async function main() { failedTests, // Only where the run didn't work, so a stray error line never colours a success. errorLine: runDidNotWork(outcome) ? redactSecrets(run.error, secretsNow()) : undefined, + buildNote, } // Proposing: post what it would do and stop. A run that broke has nothing to approve, // so it is reported like any other run, which also completes its check. @@ -1411,7 +1468,7 @@ function sweepExpiredProposals() { postCheck(OUTCOMES.COULD_NOT_RUN, { outcome: OUTCOMES.COULD_NOT_RUN, summary: {}, runUrl: RUN_URL }, sha) replyOnPr( pr.number, - `That proposal went unanswered, so I have dropped it. The check now says Action required — comment again and I will put up a fresh one.\n\n${proposalDroppedMarkerFor(sha, stale?.workflow)}`, + `That proposal went unanswered, so I have dropped it. The check now says Action required — comment again and I will put up a fresh one.\n\n${proposalDroppedMarkerFor(sha, stale?.workflow, stale?.id)}`, ) notice(`Test Companion: dropped a stale proposal on #${pr.number}.`) } @@ -1465,12 +1522,18 @@ function markPlanSpent(planId, marker) { gh(["api", `repos/${REPO}/issues/comments/${planId}`, "--method", "PATCH", "--input", "-"], { body: `${body}\n\n${marker}` }) } +/** + * One reply per request: a run asked for in a comment gets its own report, keyed to that + * comment. Automatic runs, with no request to answer, keep one status comment per PR updated. + */ function postComment(pr, view) { + const requestId = env.GITHUB_EVENT_NAME === "issue_comment" ? readEvent().comment?.id : undefined // --slurp: without it gh prints one array per page and JSON.parse fails. const existing = flattenPages(gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--paginate", "--slurp"])) - const choice = chooseCommentAction(existing, botLogin) + const choice = chooseCommentAction(existing, botLogin, { requestId }) // An approved proposal must stop looking like one. - const body = { body: renderComment(view) + (view.consumedMarker ? `\n${view.consumedMarker}` : "") } + const marker = requestMarkerFor(requestId) + const body = { body: renderComment(view, marker) + (view.consumedMarker ? `\n${view.consumedMarker}` : "") } if (choice.action === "update") { gh(["api", `repos/${REPO}/issues/comments/${choice.id}`, "--method", "PATCH", "--input", "-"], body) } else { diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index 2903bcb..580de0f 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -51,7 +51,17 @@ export function savingFor(saving, job) { } /** May this run push? Every check fails closed. */ -export function shouldPushWork({ preview, sameRepo, branch, headAuthorEmail, onWantedCommit, defaultBranch, workflow, ourEmails = [] } = {}) { +export function shouldPushWork({ + preview, + sameRepo, + branch, + headAuthorEmail, + onWantedCommit, + defaultBranch, + workflow, + ourEmails = [], + requestedByPerson = false, +} = {}) { // A shell command the agent ran during preview is outside the brain's write refusal. if (preview) return { ok: false, why: "this is a preview run, which writes nothing" } // Anything such a job leaves in the checkout is a by-product, not work to save. @@ -68,8 +78,11 @@ export function shouldPushWork({ preview, sameRepo, branch, headAuthorEmail, onW if (base && String(branch).trim().toLowerCase() === base) { return { ok: false, why: "that is the repository's default branch" } } - // Loop guard: never push in reaction to our own commit. - if (authoredByUs(headAuthorEmail, ourEmails)) return { ok: false, why: "this run is reacting to our own commit" } + // Loop guard: an automatic run never pushes in reaction to our own commit. A run a person + // asked for or approved can't loop, so it may build on what an earlier run saved. + if (!requestedByPerson && authoredByUs(headAuthorEmail, ourEmails)) { + return { ok: false, why: "this run is reacting to our own commit" } + } return { ok: true } } diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index 79325f9..fa39129 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -1,6 +1,7 @@ /** Rendering and decisions for the run's comment, check and job summary. No network calls. */ import { CAPABILITIES, isApprovalOnly } from "./intent.mjs" import { emojiFor } from "./outcome.mjs" +import { ANSWER_JOBS } from "./gate.mjs" const withMark = (outcome) => { const mark = emojiFor(outcome) @@ -13,9 +14,15 @@ const withMark = (outcome) => { */ export const isOurs = (comment, us) => Boolean(us) && comment?.user?.type === "Bot" && comment?.user?.login === us -// Hidden marker so later runs edit one comment instead of posting new ones. +// Hidden marker on the one status comment automatic runs (no request comment) keep up to date. export const COMMENT_MARKER = "" +/** Marks the reply to one request, so a re-run of the same job edits it and nothing else does. */ +export function requestMarkerFor(requestId) { + const id = String(requestId ?? "").replace(/\D/g, "") + return id ? `` : COMMENT_MARKER +} + const previewRun = (summary) => summary?.preview === "true" // Flags generated-but-unsaved cases loudly; bare counts read as success. @@ -86,6 +93,7 @@ export function renderJobSummary({ extraChecks = [], warnings = [], errorLine, + buildNote, } = {}) { const lines = [] lines.push(`## Test Companion — ${withMark(outcome)}`) @@ -114,10 +122,19 @@ export function renderJobSummary({ } } + if (String(buildNote ?? "").trim()) { + lines.push(`> ${inertMarkers(oneLine(buildNote))}`) + lines.push("") + } + if (previewRun(summary)) { - // Preview blocks Test Companion's own write tools, not shell commands the agent runs. + // Explain and find-the-bad-PR always run this way on purpose; the banner reads as "nothing happened". + const job = workflow ?? summary.workflow lines.push( - "> **Preview** — Test Companion refused every one of its own write tools, so it created, changed and saved nothing itself. Shell commands the agent ran are not covered by that.", + ANSWER_JOBS.has(job) + ? "> Read-only: this job explains what happened and changes nothing." + : // Preview blocks Test Companion's own write tools, not shell commands the agent runs. + "> **Preview** — Test Companion refused every one of its own write tools, so it created, changed and saved nothing itself. Shell commands the agent ran are not covered by that.", ) lines.push("") } @@ -172,8 +189,9 @@ function cutShortNote(runUrl) { return `\n\n_Cut short to fit a comment. The full version is on ${where}._` } -export function renderComment(args) { - return renderWithin(args, COMMENT_LIMIT, `\n\n${COMMENT_MARKER}`) +/** `marker` is requestMarkerFor(id) for a reply to a request; the shared status marker otherwise. */ +export function renderComment(args, marker = COMMENT_MARKER) { + return renderWithin(args, COMMENT_LIMIT, `\n\n${marker}`) } // GitHub's cap on a check run's summary; over it the check is never completed. @@ -205,9 +223,13 @@ function renderWithin(args, limit, tail) { return rest.slice(0, Math.max(0, limit - tail.length - note.length)) + note + tail } -// Only our own marked comment; a person quoting it must not capture the thread. -export function chooseCommentAction(comments, us) { - const ours = (comments ?? []).filter((c) => isOurs(c, us) && (c.body ?? "").includes(COMMENT_MARKER)) +/** + * Only our own marked comment; a person quoting it must not capture the thread. With a + * `requestId`, only that request's own reply is ever edited: one reply per request. + */ +export function chooseCommentAction(comments, us, { requestId } = {}) { + const marker = requestId ? requestMarkerFor(requestId) : COMMENT_MARKER + const ours = (comments ?? []).filter((c) => isOurs(c, us) && (c.body ?? "").includes(marker)) if (ours.length === 0) return { action: "create" } const oldest = ours.reduce((a, b) => (a.id <= b.id ? a : b)) return { action: "update", id: oldest.id } @@ -293,16 +315,38 @@ export function proposalAlreadyUsed(comments, sha, us, workflow) { return undefined } +/** + * Is this plan comment spent? A marker naming a plan spends only that plan. An older marker + * without one spends the plans of its job (or every job, if it names none) posted up to the + * comment carrying it, so a plan posted after it is still approvable. + */ +function planSpentBy(planComment, marks) { + const id = Number(planComment?.id) + const job = readMarker(planComment?.body)?.workflow ?? "" + return marks.find((m) => + m.plan ? m.plan === String(planComment?.id) : (!m.workflow || m.workflow === job) && id <= Number(m.host), + ) +} + +/** "approved" or "dropped" when the plan comment `id` was spent, else undefined. */ +export function planSpentState(comments, id, us) { + const plan = (comments ?? []).find((c) => isOurs(c, us) && String(c.id) === String(id) && readMarker(c.body)) + if (!plan) return undefined + const hit = planSpentBy( + plan, + spentPlans(comments, us).filter((m) => m.sha === readMarker(plan.body).sha), + ) + return hit ? (hit.kind === "consumed" ? "approved" : "dropped") : undefined +} + // Only our own comments count; an unreadable timestamp counts as expired. export function findProposal(comments, sha, now = Date.now(), us, opts = {}) { const want = String(sha ?? "").trim() - // Spent plans are tracked per job, so using one leaves the others on this commit waiting. + // Spent per plan, so approving one leaves a newer plan (same job or not) waiting. const spent = spentPlans(comments, us).filter((m) => m.sha === want) - if (spent.some((m) => !m.workflow)) return undefined - const spentJobs = new Set(spent.map((m) => m.workflow)) let ours = (comments ?? []).filter((c) => isOurs(c, us) && readMarker(c.body)?.sha === want) - ours = ours.filter((c) => !spentJobs.has(readMarker(c.body)?.workflow ?? "")) + ours = ours.filter((c) => !planSpentBy(c, spent)) // `go ` picks one plan when several are waiting. const named = String(opts?.id ?? "").trim() if (named) ours = ours.filter((c) => String(c.id) === named) @@ -341,12 +385,13 @@ export function renderApprovedPlan(proposalBody, replyText) { } // Spent-plan markers, so neither a second `go` nor the nightly sweep acts on a finished plan. -export function proposalConsumedMarkerFor(sha, workflow) { - return `` +// `planId` is the plan comment spent; without it the marker covers older plans (see planSpentBy). +export function proposalConsumedMarkerFor(sha, workflow, planId) { + return `` } -export function proposalDroppedMarkerFor(sha, workflow) { - return `` +export function proposalDroppedMarkerFor(sha, workflow, planId) { + return `` } function jobField(workflow) { @@ -354,15 +399,20 @@ function jobField(workflow) { return job ? ` workflow=${job}` : "" } -// A marker without `workflow=` covers every plan on the commit. -const SPENT_MARKER = //g +function planField(planId) { + const id = String(planId ?? "").replace(/\D/g, "") + return id ? ` plan=${id}` : "" +} + +const SPENT_MARKER = + //g function spentPlans(comments, us) { const out = [] for (const c of comments ?? []) { if (!isOurs(c, us)) continue for (const m of String(c.body ?? "").matchAll(SPENT_MARKER)) { - out.push({ kind: m[1], sha: m[2], workflow: m[3] }) + out.push({ kind: m[1], sha: m[2], workflow: m[3], plan: m[4], host: c.id }) } } return out diff --git a/test-companion/src/runs.mjs b/test-companion/src/runs.mjs index e51ed40..7570dc8 100644 --- a/test-companion/src/runs.mjs +++ b/test-companion/src/runs.mjs @@ -48,3 +48,109 @@ export function latestFailedRun(runs, { selfRunId, selfWorkflowName } = {}) { if (red.length === 0) return undefined return red.reduce((a, b) => ((a.id ?? 0) >= (b.id ?? 0) ? a : b)).html_url } + +const nameOf = (r) => + String(r?.name ?? "workflow") + .replace(/[[\]`<>|\r\n]/g, " ") + .trim() || "workflow" + +/** + * The other runs on a commit, newest first: finished red ones and ones still going. + * Excludes our own runs. + */ +export function ciOnCommit(runs, { selfRunId, selfWorkflowName } = {}) { + const others = (runs ?? []) + .filter((r) => r && String(r.id) !== String(selfRunId) && (!selfWorkflowName || r.name !== selfWorkflowName)) + .sort((a, b) => (b.id ?? 0) - (a.id ?? 0)) + const pick = (r) => ({ id: r.id, name: nameOf(r), url: r.html_url }) + return { + failed: others.filter((r) => r.status === "completed" && RED.has(r.conclusion) && r.html_url).map(pick), + running: others.filter((r) => IN_FLIGHT.has(r.status)).map(pick), + } +} + +/** + * A run the comment names explicitly: an Actions run link, or `run `. Returns { id }, + * { error } for a link to another host or repository, or undefined when none is named. + * Ids are digits only. + */ +export function parseRunReference(text, { repo, serverUrl } = {}) { + const s = String(text ?? "") + const link = /https?:\/\/([^/\s]+)\/([^/\s]+)\/([^/\s]+)\/actions\/runs\/(\d+)/i.exec(s) + if (link) { + let host = "github.com" + try { + host = new URL(serverUrl || "https://github.com").host + } catch { + // Default host. + } + const [, linkHost, owner, name, id] = link + if (linkHost.toLowerCase() !== host.toLowerCase() || `${owner}/${name}`.toLowerCase() !== String(repo ?? "").toLowerCase()) { + return { error: `That run link is not for ${repo || "this repository"}, so I can't use it.` } + } + return { id } + } + const bare = /\brun\s+#?(\d{6,})\b/i.exec(s) + return bare ? { id: bare[1] } : undefined +} + +// Words that say what to do, not which workflow to look at. +const NOT_A_NAME = new Set( + "browserstack explain fix find bad pr prs pull request the a an why what did does do failed failing fail failure build builds run runs this that these on in for please of to and go verdict rca me it is test tests ci check checks job jobs with my our".split( + " ", + ), +) + +/** The newest failed run whose workflow name has a word the comment uses (e.g. "explain e2e"). */ +export function matchFailedByName(text, failed) { + const said = new Set( + String(text ?? "") + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((w) => w.length >= 2 && !NOT_A_NAME.has(w)), + ) + if (said.size === 0) return undefined + return (failed ?? []).find((r) => + String(r.name) + .toLowerCase() + .split(/[^a-z0-9]+/) + .some((w) => said.has(w)), + ) +} + +/** Why a run named in a comment can't be used, or { ok, url, name }. */ +export function checkNamedRun(run, { prShas = [], selfRunId, selfWorkflowName } = {}) { + if (!run?.id || !run?.html_url) return { why: "I could not find that run in this repository." } + if (String(run.id) === String(selfRunId) || (selfWorkflowName && run.name === selfWorkflowName)) { + return { why: "That is one of Test Companion's own runs." } + } + if (!prShas.includes(run.head_sha)) return { why: "That run is not for a commit on this pull request." } + if (IN_FLIGHT.has(run.status)) return { why: "That run is still going. Ask me again when it finishes." } + if (run.status !== "completed" || !RED.has(run.conclusion)) { + return { why: `That run did not fail (it finished as ${run.conclusion ?? "unknown"}).` } + } + return { ok: true, url: run.html_url, name: nameOf(run) } +} + +const listNames = (runs, max = 5) => { + const shown = runs.slice(0, max).map((r) => `**${r.name}**`) + return runs.length > max ? `${shown.join(", ")} and ${runs.length - max} more` : shown.join(", ") +} + +/** The reply when nothing has failed yet but CI is still running on the commit. */ +export function renderCiStillRunning(sha, running, failed = []) { + const short = String(sha ?? "").slice(0, 7) + const done = failed.length ? ` Already failed: ${listNames(failed)}.` : "" + return `CI is still running on \`${short}\`: ${listNames(running)}.${done} Ask me again when it finishes, and I will look at whatever went red.` +} + +/** Which run was looked at, which other failures were not, and how to ask about one. */ +export function renderBuildNote(chosen, { otherFailed = [], running = [], ask = "explain" } = {}) { + if (!chosen) return "" + const parts = [`Looked at **${chosen.name}** ([run](${chosen.url})).`] + if (otherFailed.length) { + parts.push(`Also failed: ${listNames(otherFailed)}. To look at one, ask \`@browserstack ${ask} ${otherFailed[0].name}\` or paste its run link.`) + } + if (running.length) parts.push(`Still running: ${listNames(running)}.`) + return parts.join(" ") +} diff --git a/test-companion/test/plan-approval.test.mjs b/test-companion/test/plan-approval.test.mjs new file mode 100644 index 0000000..22c0be6 --- /dev/null +++ b/test-companion/test/plan-approval.test.mjs @@ -0,0 +1,83 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { + findProposal, + planSpentState, + proposalConsumedMarkerFor, + proposalDroppedMarkerFor, + proposalMarkerFor, +} from "../src/report.mjs" + +const US = "browserstack[bot]" +const SHA = "abc1234" +const now = Date.parse("2026-11-12T12:00:00Z") +const fresh = "2026-11-12T11:00:00Z" +const plan = (id, workflow = "generate", extra = "") => ({ + id, + user: { type: "Bot", login: US }, + created_at: fresh, + html_url: `https://x/${id}`, + body: `the plan ${id}${extra}\n\n${proposalMarkerFor(SHA, "alice", workflow)}`, +}) +const ours = (id, body) => ({ id, user: { type: "Bot", login: US }, created_at: fresh, body }) + +describe("go approves the newest unspent plan", () => { + test("a spent plan for the same job and commit doesn't block a newer one", () => { + const comments = [plan(10, "generate", `\n${proposalConsumedMarkerFor(SHA, "generate", 10)}`), plan(20)] + assert.equal(findProposal(comments, SHA, now, US)?.id, 20) + }) + + test("the spent plan itself is not approvable again", () => { + const comments = [plan(10, "generate", `\n${proposalConsumedMarkerFor(SHA, "generate", 10)}`)] + assert.equal(findProposal(comments, SHA, now, US), undefined) + assert.equal(findProposal(comments, SHA, now, US, { id: 10 }), undefined) + }) + + test("a marker on the run report names the plan, so other plans stay open", () => { + const comments = [plan(10), plan(20), ours(30, `report\n${proposalConsumedMarkerFor(SHA, "generate", 10)}`)] + assert.equal(findProposal(comments, SHA, now, US)?.id, 20) + assert.equal(findProposal(comments, SHA, now, US, { id: 20 })?.id, 20) + }) + + test("an old marker without a plan id spends only plans posted before it", () => { + const comments = [plan(10), ours(15, `ran it\n${proposalConsumedMarkerFor(SHA, "generate")}`), plan(20)] + assert.equal(findProposal(comments, SHA, now, US)?.id, 20) + assert.equal(findProposal(comments, SHA, now, US, { id: 10 }), undefined) + }) + + test("an old marker without a job still covers earlier plans of every job", () => { + const comments = [plan(10, "fix"), ours(15, proposalConsumedMarkerFor(SHA)), plan(20, "fix")] + assert.equal(findProposal(comments, SHA, now, US, { id: 10 }), undefined) + assert.equal(findProposal(comments, SHA, now, US)?.id, 20) + }) + + test("a dropped marker with a plan id drops only that plan", () => { + const comments = [plan(10), ours(12, `dropped\n${proposalDroppedMarkerFor(SHA, "generate", 10)}`), plan(20)] + assert.equal(findProposal(comments, SHA, now, US)?.id, 20) + }) + + test("a marker another account posted counts for nothing", () => { + const forged = { id: 25, user: { type: "Bot", login: "other[bot]" }, body: proposalConsumedMarkerFor(SHA, "generate", 20) } + assert.equal(findProposal([plan(20), forged], SHA, now, US)?.id, 20) + }) +}) + +describe("planSpentState", () => { + test("says whether a named plan was approved or dropped", () => { + const comments = [ + plan(10, "generate", `\n${proposalConsumedMarkerFor(SHA, "generate", 10)}`), + plan(20), + ours(22, proposalDroppedMarkerFor(SHA, "generate", 20)), + plan(30), + ] + assert.equal(planSpentState(comments, 10, US), "approved") + assert.equal(planSpentState(comments, "20", US), "dropped") + assert.equal(planSpentState(comments, 30, US), undefined) + assert.equal(planSpentState(comments, 99, US), undefined) + }) + + test("markers carry only digits for the plan id", () => { + assert.match(proposalConsumedMarkerFor(SHA, "fix", "12x3"), / plan=123 -->$/) + assert.doesNotMatch(proposalConsumedMarkerFor(SHA, "fix"), /plan=/) + }) +}) diff --git a/test-companion/test/reply-per-request.test.mjs b/test-companion/test/reply-per-request.test.mjs new file mode 100644 index 0000000..ee9b3d6 --- /dev/null +++ b/test-companion/test/reply-per-request.test.mjs @@ -0,0 +1,91 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { chooseCommentAction, requestMarkerFor, renderComment, renderJobSummary, COMMENT_MARKER } from "../src/report.mjs" +import { shouldPushWork } from "../src/push-work.mjs" + +const US = "browserstack[bot]" +const bot = (id, body) => ({ id, body, user: { type: "Bot", login: US } }) + +describe("one reply per request", () => { + test("a request gets a new reply, never the shared status comment or another request's", () => { + const comments = [bot(1, `status\n\n${COMMENT_MARKER}`), bot(2, `answer\n\n${requestMarkerFor(500)}`)] + assert.deepEqual(chooseCommentAction(comments, US, { requestId: 600 }), { action: "create" }) + }) + + test("a re-run of the same request edits its own reply", () => { + const comments = [bot(1, `status\n\n${COMMENT_MARKER}`), bot(2, `answer\n\n${requestMarkerFor(500)}`)] + assert.deepEqual(chooseCommentAction(comments, US, { requestId: 500 }), { action: "update", id: 2 }) + }) + + test("automatic runs keep one status comment, which a request reply never takes over", () => { + assert.deepEqual(chooseCommentAction([bot(2, `answer\n\n${requestMarkerFor(500)}`)], US), { action: "create" }) + assert.deepEqual(chooseCommentAction([bot(1, `status\n\n${COMMENT_MARKER}`)], US), { action: "update", id: 1 }) + }) + + test("the reply ends with its request's marker", () => { + const body = renderComment({ outcome: "Succeeded", summary: { workflow: "rca" } }, requestMarkerFor(500)) + assert.ok(body.endsWith(requestMarkerFor(500))) + assert.ok(!body.includes(COMMENT_MARKER)) + }) + + test("a request marker takes digits only; no id falls back to the status marker", () => { + assert.equal(requestMarkerFor("12") + assert.equal(requestMarkerFor(undefined), COMMENT_MARKER) + }) + + test("agent text can't forge a request marker", () => { + const body = renderComment({ outcome: "Succeeded", answer: `see ${requestMarkerFor(500)}` }, requestMarkerFor(600)) + assert.equal(body.split(requestMarkerFor(500)).length - 1, 0) + }) +}) + +describe("the preview banner", () => { + test("read-only jobs say so instead of the preview banner", () => { + for (const workflow of ["rca", "verdict"]) { + const body = renderJobSummary({ workflow, outcome: "Succeeded", summary: { preview: "true" } }) + assert.match(body, /Read-only: this job explains what happened and changes nothing/) + assert.doesNotMatch(body, /\*\*Preview\*\*/) + } + }) + + test("a real preview of a job that writes keeps the banner", () => { + const body = renderJobSummary({ workflow: "generate", outcome: "Succeeded", summary: { preview: "true" } }) + assert.match(body, /\*\*Preview\*\*/) + }) + + test("a live run shows neither", () => { + const body = renderJobSummary({ workflow: "fix", outcome: "Succeeded", summary: { preview: "false" } }) + assert.doesNotMatch(body, /Preview|Read-only/) + }) +}) + +describe("the build note", () => { + test("is shown, on one line, with markers neutralised", () => { + const body = renderJobSummary({ outcome: "Succeeded", buildNote: "Looked at **E2E**.\n" }) + assert.match(body, /> Looked at \*\*E2E\*\*\. <!-- test-companion-run -->/) + }) +}) + +describe("own-commit guard", () => { + const base = { + sameRepo: true, + branch: "feature", + onWantedCommit: true, + defaultBranch: "main", + headAuthorEmail: "355331011+browserstack[bot]@users.noreply.github.com", + ourEmails: ["355331011+browserstack[bot]@users.noreply.github.com"], + workflow: "fix", + } + test("an automatic run still won't push on top of our own commit", () => { + assert.deepEqual(shouldPushWork(base), { ok: false, why: "this run is reacting to our own commit" }) + }) + test("a run a person asked for or approved may", () => { + assert.deepEqual(shouldPushWork({ ...base, requestedByPerson: true }), { ok: true }) + }) + test("asking doesn't bypass any other refusal", () => { + assert.equal(shouldPushWork({ ...base, requestedByPerson: true, branch: "main" }).ok, false) + assert.equal(shouldPushWork({ ...base, requestedByPerson: true, sameRepo: false }).ok, false) + assert.equal(shouldPushWork({ ...base, requestedByPerson: true, preview: true }).ok, false) + assert.equal(shouldPushWork({ ...base, requestedByPerson: true, workflow: "generate" }).ok, false) + }) +}) diff --git a/test-companion/test/run-selection.test.mjs b/test-companion/test/run-selection.test.mjs new file mode 100644 index 0000000..369c27b --- /dev/null +++ b/test-companion/test/run-selection.test.mjs @@ -0,0 +1,137 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { + ciOnCommit, + parseRunReference, + matchFailedByName, + checkNamedRun, + renderCiStillRunning, + renderBuildNote, +} from "../src/runs.mjs" + +const run = (id, name, status, conclusion = null, extra = {}) => ({ + id, + name, + status, + conclusion, + html_url: `https://github.com/o/r/actions/runs/${id}`, + head_sha: "abc1234", + ...extra, +}) + +describe("ciOnCommit", () => { + test("splits red and still-running runs, newest first, leaving out our own", () => { + const ci = ciOnCommit( + [ + run(1, "Unit tests", "completed", "failure"), + run(3, "E2E", "completed", "failure"), + run(2, "Lint", "completed", "success"), + run(4, "Deploy", "in_progress"), + run(5, "Test Companion", "in_progress"), + run(6, "Other", "completed", "failure"), + ], + { selfRunId: 6, selfWorkflowName: "Test Companion" }, + ) + assert.deepEqual( + ci.failed.map((r) => r.name), + ["E2E", "Unit tests"], + ) + assert.deepEqual( + ci.running.map((r) => r.name), + ["Deploy"], + ) + }) + + test("a workflow name can't break the comment's formatting", () => { + const ci = ciOnCommit([run(1, "x`] (evil)|\nname", "completed", "failure")]) + assert.doesNotMatch(ci.failed[0].name, /[`[\]|\n]/) + }) +}) + +describe("parseRunReference", () => { + const opts = { repo: "Org/Repo", serverUrl: "https://github.com" } + + test("a run link for this repository, with or without a job", () => { + assert.deepEqual(parseRunReference("explain https://github.com/org/repo/actions/runs/123", opts), { id: "123" }) + assert.deepEqual(parseRunReference("fix https://github.com/Org/Repo/actions/runs/456/job/789", opts), { id: "456" }) + }) + + test("a link to another repository or host is refused", () => { + assert.ok(parseRunReference("explain https://github.com/evil/repo/actions/runs/1", opts).error) + assert.ok(parseRunReference("explain https://example.com/org/repo/actions/runs/1", opts).error) + }) + + test("`run ` with a long enough number", () => { + assert.deepEqual(parseRunReference("explain run 43189383014", opts), { id: "43189383014" }) + assert.deepEqual(parseRunReference("explain run #43189383014", opts), { id: "43189383014" }) + assert.equal(parseRunReference("explain run 12", opts), undefined) + }) + + test("nothing named, including a plan id in a go", () => { + assert.equal(parseRunReference("explain", opts), undefined) + assert.equal(parseRunReference("go #6824956824", opts), undefined) + assert.equal(parseRunReference(undefined, opts), undefined) + }) + + test("a GitHub Enterprise host is matched from the server URL", () => { + const ghe = { repo: "org/repo", serverUrl: "https://ghe.example.com" } + assert.deepEqual(parseRunReference("https://ghe.example.com/org/repo/actions/runs/9", ghe), { id: "9" }) + assert.ok(parseRunReference("https://github.com/org/repo/actions/runs/9", ghe).error) + }) +}) + +describe("matchFailedByName", () => { + const failed = [ + { id: 3, name: "E2E Tests", url: "u3" }, + { id: 1, name: "Unit tests", url: "u1" }, + ] + test("a word from the workflow name picks that run", () => { + assert.equal(matchFailedByName("explain e2e", failed).id, 3) + assert.equal(matchFailedByName("why did unit fail", failed).id, 1) + }) + test("job words and filler never pick a run", () => { + assert.equal(matchFailedByName("explain the failing tests please", failed), undefined) + assert.equal(matchFailedByName("browserstack fix", failed), undefined) + assert.equal(matchFailedByName("", failed), undefined) + }) +}) + +describe("checkNamedRun", () => { + const prShas = ["abc1234", "def5678"] + test("a finished red run on one of the PR's commits is used", () => { + const r = checkNamedRun(run(9, "E2E", "completed", "failure"), { prShas }) + assert.deepEqual(r, { ok: true, url: "https://github.com/o/r/actions/runs/9", name: "E2E" }) + }) + test("each reason a run can't be used", () => { + assert.match(checkNamedRun(null, { prShas }).why, /could not find/) + assert.match(checkNamedRun(run(9, "E2E", "completed", "failure", { head_sha: "zzz" }), { prShas }).why, /not for a commit/) + assert.match(checkNamedRun(run(9, "E2E", "in_progress"), { prShas }).why, /still going/) + assert.match(checkNamedRun(run(9, "E2E", "completed", "success"), { prShas }).why, /did not fail/) + assert.match(checkNamedRun(run(9, "TC", "completed", "failure"), { prShas, selfWorkflowName: "TC" }).why, /own runs/) + }) +}) + +describe("the replies", () => { + test("CI still running lists what is going", () => { + const text = renderCiStillRunning("abcdef1234", [{ name: "Unit tests" }, { name: "E2E" }]) + assert.match(text, /CI is still running on `abcdef1`: \*\*Unit tests\*\*, \*\*E2E\*\*/) + assert.match(text, /Ask me again when it finishes/) + }) + test("CI still running caps the list at five", () => { + const many = Array.from({ length: 7 }, (_, i) => ({ name: `w${i}` })) + assert.match(renderCiStillRunning("abc", many), /and 2 more/) + }) + test("the build note says which run was used and what else failed", () => { + const note = renderBuildNote( + { name: "E2E", url: "https://x/1" }, + { otherFailed: [{ name: "Unit tests" }], running: [{ name: "Deploy" }], ask: "explain" }, + ) + assert.match(note, /Looked at \*\*E2E\*\* \(\[run\]\(https:\/\/x\/1\)\)/) + assert.match(note, /Also failed: \*\*Unit tests\*\*\. To look at one, ask `@browserstack explain Unit tests`/) + assert.match(note, /Still running: \*\*Deploy\*\*/) + }) + test("no other failures, no extra text", () => { + assert.equal(renderBuildNote({ name: "E2E", url: "u" }), "Looked at **E2E** ([run](u)).") + assert.equal(renderBuildNote(undefined), "") + }) +}) From 5eabd3c10d6d852b562be2642d31376b711622d0 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Thu, 12 Nov 2026 21:20:22 +0530 Subject: [PATCH 08/31] test-companion: older plans stay spent, ask by run id, and fail closed on lookups - Approving a plan now also spends every older plan of that job on the commit. Before, approving the newer of two plans left the older one waiting: a second "go" would run it, and the nightly sweep would drop it and turn the commit's check to "action required" over a green run. - Naming a failed workflow matches its whole name first (the longest wins, so "Lint Checks" beats "Lint"), then a name whose every significant word appears; an incidental word no longer picks a run. The "also failed" hint now asks by run id, which is unambiguous. - Workflow names in replies are code spans, so a name can't mention a person or team, link or format. - A manual run may build on our own commit only when a human started it, not automation. - Run lookups fail closed and say so: a GitHub failure is no longer reported as "nothing failed" or "no such run"; a run with no commit id never matches; a pull request too long to list fully says why. - The read-only line is chosen from the job asked for, never from the agent-written summary, and says nothing the job does is saved. - Removed latestFailedRun, unused since run selection moved to ciOnCommit; its tests now cover ciOnCommit. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/main.mjs | 35 ++- test-companion/src/push-work.mjs | 11 + test-companion/src/report.mjs | 22 +- test-companion/src/runs.mjs | 119 +++++++--- .../test/reply-per-request.test.mjs | 4 +- .../test/review-round-fixes.test.mjs | 205 ++++++++++++++++++ test-companion/test/run-selection.test.mjs | 12 +- test-companion/test/runs.test.mjs | 6 +- 8 files changed, 354 insertions(+), 60 deletions(-) create mode 100644 test-companion/test/review-round-fixes.test.mjs diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 228b2b8..bbd24a0 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -14,6 +14,9 @@ import { checkNamedRun, renderCiStillRunning, renderBuildNote, + classifyLookup, + prCommitShas, + LOOKUP_FAILED_REPLY, } from "./runs.mjs" import { decideGate, GATE, describeChangeSize, ANSWER_JOBS } from "./gate.mjs" import { mayAsk, describeTestRepoAccess } from "./access.mjs" @@ -69,6 +72,7 @@ import { splitPushable, splitHandoffPaths, savingFor, + isRequestedByPerson, leaveOut, leaveOutMessage, listFiles, @@ -678,11 +682,18 @@ async function main() { finishSkipped("run-not-usable", "link") } if (named?.id) { - const prShas = flattenPages(gh(["api", `repos/${REPO}/pulls/${plan.context.pr}/commits?per_page=100`, "--paginate", "--slurp"])) - .map((c) => c?.sha) - .concat(plan.context.sha) - const check = checkNamedRun(gh(["api", `repos/${REPO}/actions/runs/${named.id}`]), { - prShas, + const lookup = classifyLookup(ghRaw(["api", `repos/${REPO}/actions/runs/${named.id}`])) + if (lookup.failed) { + replyOnPr(plan.context.pr, LOOKUP_FAILED_REPLY) + finishSkipped("lookup-failed", named.id) + } + const listed = prCommitShas( + gh(["api", `repos/${REPO}/pulls/${plan.context.pr}/commits?per_page=100`, "--paginate", "--slurp"]), + plan.context.sha, + ) + const check = checkNamedRun(lookup.found, { + prShas: listed.shas, + shasComplete: listed.complete, selfRunId: env.GITHUB_RUN_ID, selfWorkflowName: env.TC_ACTION_SELF_WORKFLOW, }) @@ -694,6 +705,11 @@ async function main() { buildNote = renderBuildNote(check, { ask: ASK_WORD[plan.workflow] }) } else { const runs = gh(["api", `repos/${REPO}/actions/runs?head_sha=${plan.context.sha}&per_page=100`]) + if (!Array.isArray(runs?.workflow_runs)) { + // A failed lookup is not "nothing failed". + replyOnPr(plan.context.pr, LOOKUP_FAILED_REPLY) + finishSkipped("lookup-failed", plan.workflow) + } const ci = ciOnCommit(runs?.workflow_runs, { selfRunId: env.GITHUB_RUN_ID, selfWorkflowName: env.TC_ACTION_SELF_WORKFLOW, @@ -749,8 +765,12 @@ async function main() { onWantedCommit: !needsCheckout({ headSha: git(["rev-parse", "HEAD"]).out, wantSha: plan.context.sha }), defaultBranch: defaultBranchName(), workflow: plan.workflow, - // Asked for in a comment (bots are filtered out earlier) or a manual run: may build on our commit. - requestedByPerson: plan.event === TC_EVENTS.PR_COMMENT || env.GITHUB_EVENT_NAME === "workflow_dispatch", + // Asked for by a person may build on our commit; automation keeps the loop guard. + requestedByPerson: isRequestedByPerson({ + isPrComment: plan.event === TC_EVENTS.PR_COMMENT, + eventName: env.GITHUB_EVENT_NAME, + sender: readEvent().sender, + }), }) if (plan.context.branch && !plan.preview && !saving.ok && !saving.quiet) { notice(`Nothing will be saved to ${plan.context.branch}: ${saving.why}.`) @@ -873,6 +893,7 @@ async function main() { const view = { workflow: plan.workflow ?? summary.workflow, + planWorkflow: plan.workflow, approvedPlan, workNote, extraChecks, diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index 580de0f..197eb89 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -50,6 +50,17 @@ export function savingFor(saving, job) { return saving } +/** + * Was this run asked for by a person? A pull request comment (bots are filtered out before + * this) or a manual run started by a human account. A manual run started by automation, such + * as a workflow dispatched with a token, doesn't count, so it keeps the own-commit loop guard. + */ +export function isRequestedByPerson({ isPrComment, eventName, sender } = {}) { + if (isPrComment) return true + if (eventName !== "workflow_dispatch") return false + return sender?.type === "User" && !/\[bot\]$/i.test(String(sender?.login ?? "")) +} + /** May this run push? Every check fails closed. */ export function shouldPushWork({ preview, diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index fa39129..eb366b6 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -94,6 +94,7 @@ export function renderJobSummary({ warnings = [], errorLine, buildNote, + planWorkflow, } = {}) { const lines = [] lines.push(`## Test Companion — ${withMark(outcome)}`) @@ -129,10 +130,10 @@ export function renderJobSummary({ if (previewRun(summary)) { // Explain and find-the-bad-PR always run this way on purpose; the banner reads as "nothing happened". - const job = workflow ?? summary.workflow + // Decided by the job asked for, never by the agent-written summary. lines.push( - ANSWER_JOBS.has(job) - ? "> Read-only: this job explains what happened and changes nothing." + ANSWER_JOBS.has(planWorkflow) + ? "> Read-only: this job explains what happened; nothing it does is saved." : // Preview blocks Test Companion's own write tools, not shell commands the agent runs. "> **Preview** — Test Companion refused every one of its own write tools, so it created, changed and saved nothing itself. Shell commands the agent ran are not covered by that.", ) @@ -316,16 +317,19 @@ export function proposalAlreadyUsed(comments, sha, us, workflow) { } /** - * Is this plan comment spent? A marker naming a plan spends only that plan. An older marker - * without one spends the plans of its job (or every job, if it names none) posted up to the - * comment carrying it, so a plan posted after it is still approvable. + * Is this plan comment spent? A marker naming a plan spends that plan and every older plan + * of the same job on the commit, never a newer one: approving a later plan must not leave an + * earlier one waiting. An older marker without a plan id spends the plans of its job (or + * every job, if it names none) posted up to the comment carrying it. */ function planSpentBy(planComment, marks) { const id = Number(planComment?.id) const job = readMarker(planComment?.body)?.workflow ?? "" - return marks.find((m) => - m.plan ? m.plan === String(planComment?.id) : (!m.workflow || m.workflow === job) && id <= Number(m.host), - ) + return marks.find((m) => { + const sameJob = !m.workflow || m.workflow === job + if (m.plan) return id === Number(m.plan) || (sameJob && id < Number(m.plan)) + return sameJob && id <= Number(m.host) + }) } /** "approved" or "dropped" when the plan comment `id` was spent, else undefined. */ diff --git a/test-companion/src/runs.mjs b/test-companion/src/runs.mjs index 7570dc8..9b81454 100644 --- a/test-companion/src/runs.mjs +++ b/test-companion/src/runs.mjs @@ -34,21 +34,6 @@ export function runIdsFromChecks(checks, { selfRunId, checkName = "Test Companio const RED = new Set(["failure", "cancelled", "timed_out", "startup_failure"]) -/** URL of the newest failed run on this commit, excluding our own runs. */ -export function latestFailedRun(runs, { selfRunId, selfWorkflowName } = {}) { - const red = (runs ?? []).filter( - (r) => - r && - r.status === "completed" && - RED.has(r.conclusion) && - r.html_url && - String(r.id) !== String(selfRunId) && - (!selfWorkflowName || r.name !== selfWorkflowName), - ) - if (red.length === 0) return undefined - return red.reduce((a, b) => ((a.id ?? 0) >= (b.id ?? 0) ? a : b)).html_url -} - const nameOf = (r) => String(r?.name ?? "workflow") .replace(/[[\]`<>|\r\n]/g, " ") @@ -101,30 +86,95 @@ const NOT_A_NAME = new Set( ), ) -/** The newest failed run whose workflow name has a word the comment uses (e.g. "explain e2e"). */ +const tokens = (text) => + String(text ?? "") + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter(Boolean) + +/** + * The failed run a comment names by workflow (e.g. "explain e2e"). The whole workflow name, + * as words, wins (longest first, so "Lint Checks" beats "Lint"); otherwise every significant + * word of a name must appear, the run with the most such words wins, ties go to the newest. + * `failed` is newest first. Returns undefined when no workflow is named. + */ export function matchFailedByName(text, failed) { - const said = new Set( - String(text ?? "") - .toLowerCase() - .split(/[^a-z0-9]+/) - .filter((w) => w.length >= 2 && !NOT_A_NAME.has(w)), - ) - if (said.size === 0) return undefined - return (failed ?? []).find((r) => - String(r.name) - .toLowerCase() - .split(/[^a-z0-9]+/) - .some((w) => said.has(w)), - ) + const said = tokens(text) + const padded = ` ${said.join(" ")} ` + const runs = failed ?? [] + let best + let bestLen = 0 + for (const r of runs) { + const name = tokens(r.name) + if (name.length && padded.includes(` ${name.join(" ")} `) && name.length > bestLen) { + best = r + bestLen = name.length + } + } + if (best) return best + const saidSet = new Set(said) + let most = 0 + for (const r of runs) { + const words = tokens(r.name).filter((w) => w.length >= 2 && !NOT_A_NAME.has(w)) + if (words.length && words.every((w) => saidSet.has(w)) && words.length > most) { + best = r + most = words.length + } + } + return best +} + +/** The reply when GitHub could not be asked about the builds; never "nothing failed". */ +export const LOOKUP_FAILED_REPLY = + "I couldn't look up this pull request's builds on GitHub just now, so I can't tell which one failed. Please ask me again in a minute." + +/** + * A `gh api` result as found, not found (HTTP 404), or failed (anything else), so a lookup + * that went wrong is never read as "no such run" or "nothing failed". + */ +export function classifyLookup(result) { + if (result && result.status === 0) { + try { + return { found: JSON.parse(result.stdout) } + } catch { + return { failed: true } + } + } + return /HTTP 404|Not Found/i.test(`${result?.stderr ?? ""}${result?.stdout ?? ""}`) ? { notFound: true } : { failed: true } +} + +// GitHub lists at most this many commits of a pull request. +export const PR_COMMIT_LIST_LIMIT = 250 + +/** + * The pull request's commit ids from a `--paginate --slurp` listing, always including the + * head. `complete` is false when the listing failed or may have been cut off at GitHub's limit. + */ +export function prCommitShas(slurped, headSha) { + const listed = Array.isArray(slurped) + ? slurped + .flat() + .map((c) => c?.sha) + .filter(Boolean) + : [] + const shas = [...new Set([...listed, headSha].filter(Boolean))] + return { shas, complete: Array.isArray(slurped) && listed.length < PR_COMMIT_LIST_LIMIT } } /** Why a run named in a comment can't be used, or { ok, url, name }. */ -export function checkNamedRun(run, { prShas = [], selfRunId, selfWorkflowName } = {}) { +export function checkNamedRun(run, { prShas = [], shasComplete = true, selfRunId, selfWorkflowName } = {}) { if (!run?.id || !run?.html_url) return { why: "I could not find that run in this repository." } if (String(run.id) === String(selfRunId) || (selfWorkflowName && run.name === selfWorkflowName)) { return { why: "That is one of Test Companion's own runs." } } - if (!prShas.includes(run.head_sha)) return { why: "That run is not for a commit on this pull request." } + // Fails closed: a run with no commit id never matches. + if (!run.head_sha || !prShas.includes(run.head_sha)) { + return { + why: shasComplete + ? "That run is not for a commit on this pull request." + : "I couldn't list all of this pull request's commits, so I can only use runs on its latest commit. Name a run from the latest commit, or ask without naming one.", + } + } if (IN_FLIGHT.has(run.status)) return { why: "That run is still going. Ask me again when it finishes." } if (run.status !== "completed" || !RED.has(run.conclusion)) { return { why: `That run did not fail (it finished as ${run.conclusion ?? "unknown"}).` } @@ -133,7 +183,8 @@ export function checkNamedRun(run, { prShas = [], selfRunId, selfWorkflowName } } const listNames = (runs, max = 5) => { - const shown = runs.slice(0, max).map((r) => `**${r.name}**`) + // A code span: a workflow name can't mention anyone, link, or format (nameOf strips backticks). + const shown = runs.slice(0, max).map((r) => `\`${r.name}\``) return runs.length > max ? `${shown.join(", ")} and ${runs.length - max} more` : shown.join(", ") } @@ -147,9 +198,9 @@ export function renderCiStillRunning(sha, running, failed = []) { /** Which run was looked at, which other failures were not, and how to ask about one. */ export function renderBuildNote(chosen, { otherFailed = [], running = [], ask = "explain" } = {}) { if (!chosen) return "" - const parts = [`Looked at **${chosen.name}** ([run](${chosen.url})).`] + const parts = [`Looked at \`${chosen.name}\` ([run](${chosen.url})).`] if (otherFailed.length) { - parts.push(`Also failed: ${listNames(otherFailed)}. To look at one, ask \`@browserstack ${ask} ${otherFailed[0].name}\` or paste its run link.`) + parts.push(`Also failed: ${listNames(otherFailed)}. To look at one, ask \`@browserstack ${ask} run ${otherFailed[0].id}\` or paste its run link.`) } if (running.length) parts.push(`Still running: ${listNames(running)}.`) return parts.join(" ") diff --git a/test-companion/test/reply-per-request.test.mjs b/test-companion/test/reply-per-request.test.mjs index ee9b3d6..f14f5f9 100644 --- a/test-companion/test/reply-per-request.test.mjs +++ b/test-companion/test/reply-per-request.test.mjs @@ -42,8 +42,8 @@ describe("one reply per request", () => { describe("the preview banner", () => { test("read-only jobs say so instead of the preview banner", () => { for (const workflow of ["rca", "verdict"]) { - const body = renderJobSummary({ workflow, outcome: "Succeeded", summary: { preview: "true" } }) - assert.match(body, /Read-only: this job explains what happened and changes nothing/) + const body = renderJobSummary({ workflow, planWorkflow: workflow, outcome: "Succeeded", summary: { preview: "true" } }) + assert.match(body, /Read-only: this job explains what happened; nothing it does is saved/) assert.doesNotMatch(body, /\*\*Preview\*\*/) } }) diff --git a/test-companion/test/review-round-fixes.test.mjs b/test-companion/test/review-round-fixes.test.mjs new file mode 100644 index 0000000..a1b9430 --- /dev/null +++ b/test-companion/test/review-round-fixes.test.mjs @@ -0,0 +1,205 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { + findProposal, + planSpentState, + proposalNeedsDropping, + proposalConsumedMarkerFor, + proposalMarkerFor, + renderJobSummary, +} from "../src/report.mjs" +import { + matchFailedByName, + renderBuildNote, + renderCiStillRunning, + checkNamedRun, + classifyLookup, + prCommitShas, + parseRunReference, + PR_COMMIT_LIST_LIMIT, +} from "../src/runs.mjs" +import { isRequestedByPerson } from "../src/push-work.mjs" +import { readIntent } from "../src/intent.mjs" + +const US = "browserstack[bot]" +const SHA = "abc1234" +const now = Date.parse("2026-11-12T12:00:00Z") +const fresh = "2026-11-12T11:00:00Z" +const stale = "2026-11-10T11:00:00Z" +const plan = (id, workflow = "fix", { created = fresh, extra = "" } = {}) => ({ + id, + user: { type: "Bot", login: US }, + created_at: created, + html_url: `https://x/${id}`, + body: `the plan ${id}${extra}\n\n${proposalMarkerFor(SHA, "alice", workflow)}`, +}) +const ours = (id, body) => ({ id, user: { type: "Bot", login: US }, created_at: fresh, body }) + +describe("approving a newer plan also spends the older ones of that job", () => { + // Two plans for the same job on one commit; the newer one was approved. + const comments = (created = fresh) => [ + plan(100, "fix", { created }), + plan(200, "fix", { created }), + ours(300, `run report\n${proposalConsumedMarkerFor(SHA, "fix", 200)}`), + ] + + test("a bare go finds no waiting plan, so the older plan can't be run", () => { + assert.equal(findProposal(comments(), SHA, now, US), undefined) + }) + + test("the nightly sweep has nothing to drop, so it never flips the check", () => { + assert.equal(proposalNeedsDropping(comments(stale), SHA, now, US), false) + }) + + test("go # is told it was already approved", () => { + assert.equal(planSpentState(comments(), 100, US), "approved") + assert.equal(planSpentState(comments(), 200, US), "approved") + }) + + test("approving the older plan leaves the newer one approvable", () => { + const c = [plan(100), plan(200), ours(300, proposalConsumedMarkerFor(SHA, "fix", 100))] + assert.equal(findProposal(c, SHA, now, US)?.id, 200) + assert.equal(planSpentState(c, 200, US), undefined) + }) + + test("an older plan for a different job stays open", () => { + const c = [plan(100, "generate"), plan(200, "fix"), ours(300, proposalConsumedMarkerFor(SHA, "fix", 200))] + assert.equal(findProposal(c, SHA, now, US)?.id, 100) + }) +}) + +describe("naming a failed workflow in the comment", () => { + const failed = [ + { id: 9, name: "Lint" }, + { id: 8, name: "CI" }, + { id: 7, name: "Build and Test" }, + { id: 6, name: "Lint Checks" }, + { id: 5, name: "Deploy main" }, + { id: 4, name: "e2e" }, + ] + const pick = (text) => matchFailedByName(text, failed)?.name + + test("a whole workflow name wins, longest first", () => { + assert.equal(pick("@browserstack explain CI"), "CI") + assert.equal(pick("@browserstack explain Build and Test"), "Build and Test") + assert.equal(pick("@browserstack explain Lint Checks"), "Lint Checks") + assert.equal(pick("@browserstack explain lint"), "Lint") + assert.equal(pick("@browserstack explain e2e please"), "e2e") + }) + + test("an incidental word does not name a workflow", () => { + assert.equal(pick("@browserstack explain the failure on main"), undefined) + assert.equal(pick("@browserstack explain why the build failed"), undefined) + assert.equal(pick("@browserstack explain"), undefined) + }) + + test("every significant word of a name counts as naming it", () => { + assert.equal(pick("@browserstack explain the main deploy"), "Deploy main") + }) +}) + +describe("the 'also failed' hint asks by run id", () => { + const note = renderBuildNote( + { id: 9, name: "Lint", url: "https://github.com/o/r/actions/runs/9" }, + { otherFailed: [{ id: 12345678, name: "CI", url: "u" }], ask: "explain" }, + ) + + test("the hint names the run by id, which the parser and intent reader understand", () => { + assert.match(note, /`@browserstack explain run 12345678`/) + assert.deepEqual(parseRunReference("@browserstack explain run 12345678", { repo: "o/r" }), { id: "12345678" }) + assert.equal(readIntent("explain run 12345678", { workflows: ["rca", "fix"] }).workflow, "rca") + }) +}) + +describe("workflow names in replies are inert", () => { + const hostile = { id: 1, name: "x @org/core #12 *y* https://e.example", url: "https://github.com/o/r/actions/runs/1" } + + test("names are code spans, so they can't mention, link or format", () => { + const note = renderBuildNote(hostile, { otherFailed: [hostile], running: [hostile] }) + assert.ok(note.includes("`x @org/core #12 *y* https://e.example`")) + assert.doesNotMatch(note, /\*\*x @org/) + const running = renderCiStillRunning("abcdef0123", [hostile], [hostile]) + assert.ok(running.includes("`x @org/core #12 *y* https://e.example`")) + }) +}) + +describe("isRequestedByPerson", () => { + test("a pull request comment is a person's request (bots are filtered earlier)", () => { + assert.equal(isRequestedByPerson({ isPrComment: true }), true) + }) + + test("a manual run counts only when a human account started it", () => { + assert.equal(isRequestedByPerson({ eventName: "workflow_dispatch", sender: { type: "User", login: "alice" } }), true) + assert.equal( + isRequestedByPerson({ eventName: "workflow_dispatch", sender: { type: "Bot", login: "browserstack[bot]" } }), + false, + ) + assert.equal( + isRequestedByPerson({ eventName: "workflow_dispatch", sender: { type: "User", login: "github-actions[bot]" } }), + false, + ) + assert.equal(isRequestedByPerson({ eventName: "workflow_dispatch" }), false) + }) + + test("automatic events never count", () => { + for (const eventName of ["push", "pull_request", "schedule", "workflow_run"]) { + assert.equal(isRequestedByPerson({ eventName, sender: { type: "User", login: "alice" } }), false) + } + }) +}) + +describe("run lookups fail closed and say so", () => { + test("a 404 is not found; any other failure is a failed lookup", () => { + assert.deepEqual(classifyLookup({ status: 0, stdout: '{"id":1}' }), { found: { id: 1 } }) + assert.deepEqual(classifyLookup({ status: 1, stderr: "gh: Not Found (HTTP 404)" }), { notFound: true }) + assert.deepEqual(classifyLookup({ status: 1, stderr: "gh: Server Error (HTTP 502)" }), { failed: true }) + assert.deepEqual(classifyLookup({ status: 0, stdout: "not json" }), { failed: true }) + assert.deepEqual(classifyLookup(undefined), { failed: true }) + }) + + test("commit ids drop missing values and always include the head", () => { + const got = prCommitShas([[{ sha: "a" }, {}, null, { sha: undefined }], [{ sha: "b" }]], "h") + assert.deepEqual(got, { shas: ["a", "b", "h"], complete: true }) + }) + + test("a failed listing keeps only the head and is marked incomplete", () => { + assert.deepEqual(prCommitShas(null, "h"), { shas: ["h"], complete: false }) + }) + + test("a listing cut off at GitHub's limit is marked incomplete", () => { + const page = Array.from({ length: PR_COMMIT_LIST_LIMIT }, (_, i) => ({ sha: `s${i}` })) + assert.equal(prCommitShas([page], "h").complete, false) + }) + + test("a run with no commit id never matches, even if the list held a gap", () => { + const run = { id: 5, html_url: "u", status: "completed", conclusion: "failure" } + assert.ok(checkNamedRun(run, { prShas: prCommitShas([[{}]], "h").shas }).why) + }) + + test("a run on an unlisted commit of a too-long pull request says why", () => { + const run = { id: 5, html_url: "u", head_sha: "old", status: "completed", conclusion: "failure" } + assert.match(checkNamedRun(run, { prShas: ["h"], shasComplete: false }).why, /couldn't list all/) + assert.match(checkNamedRun(run, { prShas: ["h"], shasComplete: true }).why, /not for a commit on this pull request/) + }) +}) + +describe("the read-only line follows the job that was asked for", () => { + const summary = { preview: "true" } + + test("explain shows the read-only line", () => { + const md = renderJobSummary({ workflow: "rca", planWorkflow: "rca", outcome: "Succeeded", summary }) + assert.match(md, /Read-only: this job explains what happened; nothing it does is saved\./) + assert.doesNotMatch(md, /\*\*Preview\*\*/) + }) + + test("an agent-written summary can't swap the preview banner for the read-only line", () => { + const md = renderJobSummary({ workflow: "rca", outcome: "Succeeded", summary: { ...summary, workflow: "rca" } }) + assert.match(md, /\*\*Preview\*\*/) + assert.doesNotMatch(md, /Read-only/) + }) + + test("a preview of a job that writes keeps the banner", () => { + const md = renderJobSummary({ workflow: "fix", planWorkflow: "fix", outcome: "Succeeded", summary }) + assert.match(md, /\*\*Preview\*\*/) + }) +}) diff --git a/test-companion/test/run-selection.test.mjs b/test-companion/test/run-selection.test.mjs index 369c27b..233a837 100644 --- a/test-companion/test/run-selection.test.mjs +++ b/test-companion/test/run-selection.test.mjs @@ -114,7 +114,7 @@ describe("checkNamedRun", () => { describe("the replies", () => { test("CI still running lists what is going", () => { const text = renderCiStillRunning("abcdef1234", [{ name: "Unit tests" }, { name: "E2E" }]) - assert.match(text, /CI is still running on `abcdef1`: \*\*Unit tests\*\*, \*\*E2E\*\*/) + assert.match(text, /CI is still running on `abcdef1`: `Unit tests`, `E2E`/) assert.match(text, /Ask me again when it finishes/) }) test("CI still running caps the list at five", () => { @@ -124,14 +124,14 @@ describe("the replies", () => { test("the build note says which run was used and what else failed", () => { const note = renderBuildNote( { name: "E2E", url: "https://x/1" }, - { otherFailed: [{ name: "Unit tests" }], running: [{ name: "Deploy" }], ask: "explain" }, + { otherFailed: [{ id: 4242424, name: "Unit tests" }], running: [{ name: "Deploy" }], ask: "explain" }, ) - assert.match(note, /Looked at \*\*E2E\*\* \(\[run\]\(https:\/\/x\/1\)\)/) - assert.match(note, /Also failed: \*\*Unit tests\*\*\. To look at one, ask `@browserstack explain Unit tests`/) - assert.match(note, /Still running: \*\*Deploy\*\*/) + assert.match(note, /Looked at `E2E` \(\[run\]\(https:\/\/x\/1\)\)/) + assert.match(note, /Also failed: `Unit tests`\. To look at one, ask `@browserstack explain run 4242424`/) + assert.match(note, /Still running: `Deploy`/) }) test("no other failures, no extra text", () => { - assert.equal(renderBuildNote({ name: "E2E", url: "u" }), "Looked at **E2E** ([run](u)).") + assert.equal(renderBuildNote({ name: "E2E", url: "u" }), "Looked at `E2E` ([run](u)).") assert.equal(renderBuildNote(undefined), "") }) }) diff --git a/test-companion/test/runs.test.mjs b/test-companion/test/runs.test.mjs index 6a9e155..b5eba12 100644 --- a/test-companion/test/runs.test.mjs +++ b/test-companion/test/runs.test.mjs @@ -1,6 +1,6 @@ import { test, describe } from "node:test" import assert from "node:assert/strict" -import { chooseRunsToCancel, runIdsFromChecks, latestFailedRun } from "../src/runs.mjs" +import { chooseRunsToCancel, runIdsFromChecks, ciOnCommit } from "../src/runs.mjs" const inProgress = (id, over = {}) => ({ id, @@ -90,7 +90,9 @@ describe("runIdsFromChecks — finding OUR in-flight runs on a commit", () => { }) }) -describe("latestFailedRun — which build 'explain the failure' means", () => { +describe("ciOnCommit — which build 'explain the failure' means", () => { + // The newest failed run is failed[0]. + const latestFailedRun = (runs, opts) => ciOnCommit(runs, opts).failed[0]?.url // `rca`, `fix` and `verdict` need to know which build failed. const run = (id, conclusion, name = "CI", status = "completed") => ({ id, From 2882be503fe38b19dafd0409793e9327fd10098c Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Thu, 12 Nov 2026 21:31:04 +0530 Subject: [PATCH 09/31] test-companion: one approval settles a job's plans, and everyday words don't name a workflow Approving a plan now spends every plan of the same job on that commit that already existed when the approval was recorded, older or newer. Approving an older plan therefore also settles a newer one that was waiting, so the nightly sweep can no longer drop it and flip the check over the approved run. A plan posted after the approval stays approvable. Dropping an expired plan still spends only it and older plans, never a newer one that may still be waiting. A marker that names a plan but no job spends only that plan. Old markers without a plan id keep their meaning. "go #" on a plan that another approval settled now says a different plan for this job was approved, instead of claiming this one ran. Picking a failed workflow by name: a name made only of everyday words (build, Test, CI, explain, fix) no longer matches, because comments use those words anyway; such a run is asked for by run id, which is what the bot's own hint suggests. Matrix suffixes in parentheses are ignored, so "explain e2e" finds "E2E Tests (linux)" again. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/main.mjs | 11 +- test-companion/src/report.mjs | 60 ++++++++-- test-companion/src/runs.mjs | 12 +- test-companion/test/plan-approval.test.mjs | 12 +- .../test/review-round-fixes.test.mjs | 104 ++++++++++++++++-- 5 files changed, 166 insertions(+), 33 deletions(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index bbd24a0..3aedc62 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -54,6 +54,7 @@ import { proposalConsumedMarkerFor, proposalAlreadyUsed, planSpentState, + unavailablePlanReply, proposalNeedsDropping, flattenPages, hasApproverEdits, @@ -542,15 +543,7 @@ async function main() { waiting = findProposal(comments, plan.context.sha, Date.now(), botLogin, { id: intent.proposalId }) if (!waiting) { const spentAs = planSpentState(comments, intent.proposalId, botLogin) - replyOnPr( - plan.context.pr, - spentAs - ? `Plan \`#${intent.proposalId}\` was already ${spentAs === "approved" ? "approved and run" : "dropped"}, so there is nothing left to approve on it. ` + - `Reply **go** on its own to approve the newest plan, or ask me again for a fresh one.` - : `I have no plan \`#${intent.proposalId}\` still waiting on \`${String(plan.context.sha).slice(0, 7)}\`. ` + - `It may have been written against an earlier commit. ` + - `Reply **go** on its own to approve the most recent one.`, - ) + replyOnPr(plan.context.pr, unavailablePlanReply(spentAs, intent.proposalId, plan.context.sha)) finishSkipped("proposal-not-found") } } diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index eb366b6..bb80907 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -317,32 +317,70 @@ export function proposalAlreadyUsed(comments, sha, us, workflow) { } /** - * Is this plan comment spent? A marker naming a plan spends that plan and every older plan - * of the same job on the commit, never a newer one: approving a later plan must not leave an - * earlier one waiting. An older marker without a plan id spends the plans of its job (or - * every job, if it names none) posted up to the comment carrying it. + * Is this plan comment spent? Returns the marker that spent it, or undefined. + * + * A marker naming a plan always spends that plan. An approval ("consumed") also spends every + * other plan of the same job on the commit that already existed when it was recorded (posted + * before the comment carrying the marker), older or newer than the approved one: one approval + * settles that job on that commit, so neither a second `go` nor the nightly sweep acts on the + * others. Plans posted after it stay approvable. Dropping an expired plan spends only it and + * older plans of its job, never a newer plan that may still be waiting. A marker naming no job + * spends only its named plan. An older marker without a plan id spends the plans of its job + * (or every job, if it names none) posted up to the comment carrying it. */ function planSpentBy(planComment, marks) { const id = Number(planComment?.id) const job = readMarker(planComment?.body)?.workflow ?? "" return marks.find((m) => { + if (m.plan) { + const named = Number(m.plan) + if (id === named) return true + if (!m.workflow || m.workflow !== job) return false + return m.kind === "consumed" ? id < Number(m.host) : id < named + } const sameJob = !m.workflow || m.workflow === job - if (m.plan) return id === Number(m.plan) || (sameJob && id < Number(m.plan)) return sameJob && id <= Number(m.host) }) } -/** "approved" or "dropped" when the plan comment `id` was spent, else undefined. */ +/** The reply to `go #` when that plan can't be approved; `spentAs` is from planSpentState. */ +export function unavailablePlanReply(spentAs, id, sha) { + const plan = `Plan \`#${String(id ?? "").replace(/\D/g, "")}\`` + if (spentAs === "superseded") { + return ( + "A different plan for this job on this commit was already approved, so this one no longer applies. " + + "Push a new commit or ask again to get a fresh plan." + ) + } + if (spentAs === "approved" || spentAs === "dropped") { + return ( + `${plan} was already ${spentAs === "approved" ? "approved and run" : "dropped"}, so there is nothing left to approve on it. ` + + "Reply **go** on its own to approve the newest plan, or ask me again for a fresh one." + ) + } + return ( + `I have no plan \`#${String(id ?? "").replace(/\D/g, "")}\` still waiting on \`${String(sha ?? "").slice(0, 7)}\`. ` + + "It may have been written against an earlier commit. Reply **go** on its own to approve the most recent one." + ) +} + +/** + * "approved" or "dropped" when the plan comment `id` itself was approved or dropped, + * "superseded" when another plan of its job on the commit was approved instead, else undefined. + */ export function planSpentState(comments, id, us) { const plan = (comments ?? []).find((c) => isOurs(c, us) && String(c.id) === String(id) && readMarker(c.body)) if (!plan) return undefined - const hit = planSpentBy( - plan, - spentPlans(comments, us).filter((m) => m.sha === readMarker(plan.body).sha), - ) - return hit ? (hit.kind === "consumed" ? "approved" : "dropped") : undefined + const marks = spentPlans(comments, us).filter((m) => m.sha === readMarker(plan.body).sha) + // Its own fate first: a marker naming this plan says exactly what happened to it. + const own = marks.find((m) => m.plan && Number(m.plan) === Number(plan.id)) + const hit = own ?? planSpentBy(plan, marks) + if (!hit) return undefined + if (hit.kind === "dropped") return "dropped" + return hit.plan && Number(hit.plan) !== Number(plan.id) ? "superseded" : "approved" } + // Only our own comments count; an unreadable timestamp counts as expired. export function findProposal(comments, sha, now = Date.now(), us, opts = {}) { const want = String(sha ?? "").trim() diff --git a/test-companion/src/runs.mjs b/test-companion/src/runs.mjs index 9b81454..36eb309 100644 --- a/test-companion/src/runs.mjs +++ b/test-companion/src/runs.mjs @@ -92,10 +92,16 @@ const tokens = (text) => .split(/[^a-z0-9]+/) .filter(Boolean) +// A matrix suffix ("E2E Tests (linux)") is not part of what people call the workflow. +const nameTokens = (name) => tokens(String(name ?? "").replace(/\([^)]*\)/g, " ")) +const significant = (words) => words.filter((w) => w.length >= 2 && !NOT_A_NAME.has(w)) + /** * The failed run a comment names by workflow (e.g. "explain e2e"). The whole workflow name, * as words, wins (longest first, so "Lint Checks" beats "Lint"); otherwise every significant * word of a name must appear, the run with the most such words wins, ties go to the newest. + * A name made only of everyday words ("build", "Test", "CI", "explain") never matches: the + * comment uses those words anyway. Such a run is asked for by `run `. * `failed` is newest first. Returns undefined when no workflow is named. */ export function matchFailedByName(text, failed) { @@ -105,8 +111,8 @@ export function matchFailedByName(text, failed) { let best let bestLen = 0 for (const r of runs) { - const name = tokens(r.name) - if (name.length && padded.includes(` ${name.join(" ")} `) && name.length > bestLen) { + const name = nameTokens(r.name) + if (significant(name).length && padded.includes(` ${name.join(" ")} `) && name.length > bestLen) { best = r bestLen = name.length } @@ -115,7 +121,7 @@ export function matchFailedByName(text, failed) { const saidSet = new Set(said) let most = 0 for (const r of runs) { - const words = tokens(r.name).filter((w) => w.length >= 2 && !NOT_A_NAME.has(w)) + const words = significant(nameTokens(r.name)) if (words.length && words.every((w) => saidSet.has(w)) && words.length > most) { best = r most = words.length diff --git a/test-companion/test/plan-approval.test.mjs b/test-companion/test/plan-approval.test.mjs index 22c0be6..39a9a56 100644 --- a/test-companion/test/plan-approval.test.mjs +++ b/test-companion/test/plan-approval.test.mjs @@ -33,10 +33,16 @@ describe("go approves the newest unspent plan", () => { assert.equal(findProposal(comments, SHA, now, US, { id: 10 }), undefined) }) - test("a marker on the run report names the plan, so other plans stay open", () => { - const comments = [plan(10), plan(20), ours(30, `report\n${proposalConsumedMarkerFor(SHA, "generate", 10)}`)] + test("an approval settles the job's plans that existed then; one posted afterwards stays open", () => { + const comments = [plan(10), plan(20), ours(30, `report\n${proposalConsumedMarkerFor(SHA, "generate", 10)}`), plan(40)] + assert.equal(findProposal(comments, SHA, now, US)?.id, 40) + assert.equal(findProposal(comments, SHA, now, US, { id: 20 }), undefined) + assert.equal(findProposal(comments, SHA, now, US, { id: 40 })?.id, 40) + }) + + test("an approval of one job leaves another job's plans open", () => { + const comments = [plan(10), plan(20, "fix"), ours(30, `report\n${proposalConsumedMarkerFor(SHA, "generate", 10)}`)] assert.equal(findProposal(comments, SHA, now, US)?.id, 20) - assert.equal(findProposal(comments, SHA, now, US, { id: 20 })?.id, 20) }) test("an old marker without a plan id spends only plans posted before it", () => { diff --git a/test-companion/test/review-round-fixes.test.mjs b/test-companion/test/review-round-fixes.test.mjs index a1b9430..df149e3 100644 --- a/test-companion/test/review-round-fixes.test.mjs +++ b/test-companion/test/review-round-fixes.test.mjs @@ -5,7 +5,9 @@ import { planSpentState, proposalNeedsDropping, proposalConsumedMarkerFor, + proposalDroppedMarkerFor, proposalMarkerFor, + unavailablePlanReply, renderJobSummary, } from "../src/report.mjs" import { @@ -51,15 +53,61 @@ describe("approving a newer plan also spends the older ones of that job", () => assert.equal(proposalNeedsDropping(comments(stale), SHA, now, US), false) }) - test("go # is told it was already approved", () => { - assert.equal(planSpentState(comments(), 100, US), "approved") + test("go # is told another plan was approved instead, not that it ran", () => { + assert.equal(planSpentState(comments(), 100, US), "superseded") assert.equal(planSpentState(comments(), 200, US), "approved") }) - test("approving the older plan leaves the newer one approvable", () => { - const c = [plan(100), plan(200), ours(300, proposalConsumedMarkerFor(SHA, "fix", 100))] - assert.equal(findProposal(c, SHA, now, US)?.id, 200) + test("approving the older plan also settles a newer one that was already waiting", () => { + const c = (created = fresh) => [ + plan(100, "fix", { created }), + plan(200, "fix", { created }), + ours(300, proposalConsumedMarkerFor(SHA, "fix", 100)), + ] + assert.equal(findProposal(c(), SHA, now, US), undefined) + assert.equal(planSpentState(c(), 200, US), "superseded") + assert.equal(planSpentState(c(), 100, US), "approved") + // So the sweep never drops the newer plan and flips the check over the approved run. + assert.equal(proposalNeedsDropping(c(stale), SHA, now, US), false) + }) + + test("a plan posted after the approval stays approvable", () => { + const c = [plan(100), ours(300, proposalConsumedMarkerFor(SHA, "fix", 100)), plan(400)] + assert.equal(findProposal(c, SHA, now, US)?.id, 400) + assert.equal(planSpentState(c, 400, US), undefined) + }) + + test("a marker that names a plan but no job spends only that plan", () => { + const c = [plan(100, "fix"), plan(150, "generate"), plan(200, "fix"), ours(300, proposalConsumedMarkerFor(SHA, "", 200))] + assert.equal(planSpentState(c, 200, US), "approved") + assert.equal(planSpentState(c, 100, US), undefined) + assert.equal(planSpentState(c, 150, US), undefined) + assert.equal(findProposal(c, SHA, now, US)?.id, 150) + }) + + test("an approval on one commit leaves plans on another commit alone", () => { + const other = { ...plan(250, "fix"), body: `p\n\n${proposalMarkerFor("def5678", "alice", "fix")}` } + const c = [plan(200, "fix"), other, ours(300, proposalConsumedMarkerFor(SHA, "fix", 200))] + assert.equal(planSpentState(c, 250, US), undefined) + assert.equal(findProposal(c, "def5678", now, US)?.id, 250) + }) + + test("dropping an expired plan never spends a newer one still waiting", () => { + const c = [ + plan(100, "fix", { created: stale }), + plan(200, "fix"), + ours(300, proposalDroppedMarkerFor(SHA, "fix", 100)), + ] + assert.equal(planSpentState(c, 100, US), "dropped") assert.equal(planSpentState(c, 200, US), undefined) + assert.equal(findProposal(c, SHA, now, US)?.id, 200) + }) + + test("an old marker without a plan id keeps its old meaning", () => { + const c = [plan(100), plan(200), ours(300, proposalConsumedMarkerFor(SHA, "fix")), plan(400)] + assert.equal(planSpentState(c, 100, US), "approved") + assert.equal(planSpentState(c, 200, US), "approved") + assert.equal(findProposal(c, SHA, now, US)?.id, 400) }) test("an older plan for a different job stays open", () => { @@ -80,13 +128,17 @@ describe("naming a failed workflow in the comment", () => { const pick = (text) => matchFailedByName(text, failed)?.name test("a whole workflow name wins, longest first", () => { - assert.equal(pick("@browserstack explain CI"), "CI") - assert.equal(pick("@browserstack explain Build and Test"), "Build and Test") assert.equal(pick("@browserstack explain Lint Checks"), "Lint Checks") assert.equal(pick("@browserstack explain lint"), "Lint") assert.equal(pick("@browserstack explain e2e please"), "e2e") }) + test("a name made only of everyday words never matches; it is asked for by run id", () => { + // The comment uses these words anyway ("why this build failed"), so they can't name a run. + assert.equal(pick("@browserstack explain CI"), undefined) + assert.equal(pick("@browserstack explain Build and Test"), undefined) + }) + test("an incidental word does not name a workflow", () => { assert.equal(pick("@browserstack explain the failure on main"), undefined) assert.equal(pick("@browserstack explain why the build failed"), undefined) @@ -98,6 +150,44 @@ describe("naming a failed workflow in the comment", () => { }) }) +describe("everyday-word workflow names and matrix suffixes", () => { + const pick = (text, failed) => matchFailedByName(text, failed)?.name + + test("a workflow called build, Test or explain is not picked by a sentence that uses the word", () => { + assert.equal(pick("explain why this build failed", [{ id: 9, name: "e2e" }, { id: 8, name: "build" }]), undefined) + assert.equal(pick("fix the failing test", [{ id: 9, name: "Lint" }, { id: 8, name: "Test" }]), undefined) + assert.equal(pick("explain", [{ id: 9, name: "Lint" }, { id: 8, name: "explain" }]), undefined) + assert.equal(pick("explain CI / build", [{ id: 9, name: "e2e" }, { id: 8, name: "CI / build" }]), undefined) + }) + + test("a matrix suffix doesn't stop the workflow from being named", () => { + assert.equal(pick("explain e2e", [{ id: 9, name: "Lint" }, { id: 8, name: "E2E Tests (linux)" }]), "E2E Tests (linux)") + assert.equal(pick("explain e2e", [{ id: 9, name: "Lint" }, { id: 8, name: "e2e (chrome, 18)" }]), "e2e (chrome, 18)") + }) + + test("between two matrix legs, ties go to the newest", () => { + assert.equal(pick("explain e2e", [{ id: 9, name: "e2e (linux)" }, { id: 8, name: "e2e (mac)" }]), "e2e (linux)") + }) +}) + +describe("the reply to go # on a plan that can't be approved", () => { + test("superseded says another plan was approved, not that this one ran", () => { + const text = unavailablePlanReply("superseded", 100, SHA) + assert.match(text, /different plan for this job on this commit was already approved/) + assert.doesNotMatch(text, /approved and run/) + }) + + test("approved, dropped and unknown keep their own replies", () => { + assert.match(unavailablePlanReply("approved", 100, SHA), /was already approved and run/) + assert.match(unavailablePlanReply("dropped", 100, SHA), /was already dropped/) + assert.match(unavailablePlanReply(undefined, 100, SHA), /I have no plan `#100` still waiting on `abc1234`/) + }) + + test("the id is digits only in the reply", () => { + assert.match(unavailablePlanReply("approved", "1`0@x", SHA), /Plan `#10`/) + }) +}) + describe("the 'also failed' hint asks by run id", () => { const note = renderBuildNote( { id: 9, name: "Lint", url: "https://github.com/o/r/actions/runs/9" }, From 8d05931ed7f89256bd8fc02268dc80cc2883908a Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Thu, 12 Nov 2026 23:45:13 +0530 Subject: [PATCH 10/31] test-companion: retry the App sign-in, and refuse a fix that rewrote history Signing in as the App now retries a network error, a 5xx or a 429 up to three times (1s, then 3s; a Retry-After wins, capped at 10s) before falling back to the workflow token. 401, 403, 404 and 422 are never retried. A single network blip had made one reply post as github-actions[bot] instead of the App. A handed-back fix is now squashed onto the commit the run started on, and only if the agent's branch builds on that commit. An agent that ran `git reset HEAD^` past it had handed back a branch whose diff, squashed onto the older merge-base, would have undone the pull request's own fix and tests. Such a branch is now refused with a plain reason, and no branch is pushed and no pull request is opened. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/app-auth.mjs | 30 +++++++++- test-companion/src/main.mjs | 18 +++++- test-companion/src/push-work.mjs | 16 ++++++ test-companion/test/app-auth.test.mjs | 79 +++++++++++++++++++++++++- test-companion/test/push-work.test.mjs | 54 ++++++++++++++++++ 5 files changed, 190 insertions(+), 7 deletions(-) diff --git a/test-companion/src/app-auth.mjs b/test-companion/src/app-auth.mjs index ed62421..696095b 100644 --- a/test-companion/src/app-auth.mjs +++ b/test-companion/src/app-auth.mjs @@ -28,7 +28,32 @@ export function jwtClaims(appId, nowSeconds) { * and GitHub accepts an App's JWT only as `Authorization: Bearer`. * Returns { ok, status, data, reason }; `reason` is GitHub's message, never the JWT. */ -export async function appApi(path, { jwt, method = "GET", body, apiUrl, fetchImpl = fetch } = {}) { +export async function appApi(path, { jwt, method = "GET", body, apiUrl, fetchImpl = fetch, attempts = 3, sleep = defaultSleep } = {}) { + let result + for (let attempt = 1; attempt <= attempts; attempt++) { + result = await appApiOnce(path, { jwt, method, body, apiUrl, fetchImpl }) + if (!isRetryable(result) || attempt === attempts) return { ...result, attempts: attempt } + await sleep(retryDelayMs(result, attempt)) + } + return { ...result, attempts } +} + +const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) + +/** A network failure, a 5xx or a 429 may pass on a second try; 401/403/404/422 never will. */ +export function isRetryable(result) { + const status = Number(result?.status) + return status === 0 || status === 429 || (status >= 500 && status <= 599) +} + +/** 1s, then 3s; a Retry-After from GitHub wins, capped at 10s. */ +export function retryDelayMs(result, attempt) { + const after = Number(result?.retryAfter) + if (Number.isFinite(after) && after > 0) return Math.min(after, 10) * 1000 + return attempt === 1 ? 1000 : 3000 +} + +async function appApiOnce(path, { jwt, method, body, apiUrl, fetchImpl }) { const base = String(apiUrl || "https://api.github.com").replace(/\/+$/, "") let response try { @@ -47,7 +72,8 @@ export async function appApi(path, { jwt, method = "GET", body, apiUrl, fetchImp } const data = await response.json().catch(() => undefined) const reason = response.ok ? "" : `${data?.message || response.statusText || "no detail from GitHub"} (HTTP ${response.status})` - return { ok: response.ok, status: response.status, data, reason } + const retryAfter = response.headers?.get?.("retry-after") ?? undefined + return { ok: response.ok, status: response.status, data, reason, ...(retryAfter ? { retryAfter } : {}) } } /** What to log when looking up the App's installation on this repository fails. */ diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 3aedc62..f43908c 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -72,6 +72,8 @@ import { changedPaths, splitPushable, splitHandoffPaths, + checkHandoffDescends, + REWROTE_HISTORY, savingFor, isRequestedByPerson, leaveOut, @@ -222,7 +224,11 @@ async function appInstallationToken() { } // The JWT authenticates the App; only the installation token can touch a repository. - const asApp = (path, opts) => appApi(path, { jwt, apiUrl: env.GITHUB_API_URL, ...opts }) + const asApp = async (path, opts) => { + const r = await appApi(path, { jwt, apiUrl: env.GITHUB_API_URL, ...opts }) + if (r.ok && r.attempts > 1) notice(`Test Companion: reaching GitHub as the App took ${r.attempts} tries.`) + return r + } const found = await asApp(`repos/${REPO}/installation`) if (!found.ok) { @@ -1216,8 +1222,14 @@ function openHandedOffPr(path, plan) { } const tip = git(["rev-parse", "--verify", "--quiet", `refs/heads/${req.branch}^{commit}`]) if (!tip.ok || !tip.out) return `**Did not open a pull request:** the run did not leave a \`${req.branch}\` branch.` - const from = git(["merge-base", headBeforeRun || "HEAD", tip.out]) - if (!from.ok) return `**Did not open a pull request:** \`${req.branch}\` does not share history with this commit.` + // The squash goes onto the commit the run started on, and only if the agent's branch builds + // on it: a branch that reset or rebased past it would hand back a diff that undoes that work. + const reviewed = git(["rev-parse", "--verify", "--quiet", `${headBeforeRun || "HEAD"}^{commit}`]) + if (!reviewed.ok || !reviewed.out) return "**Could not open the pull request:** could not read the commit the run started on." + const descends = checkHandoffDescends({ git, reviewed: reviewed.out, tip: tip.out }) + if (descends.error) return `**Could not open the pull request:** ${descends.error}.` + if (!descends.ok) return `**Did not open a pull request:** ${REWROTE_HISTORY}.` + const from = { ok: true, out: reviewed.out } // --raw gives file modes, so a symlink is caught from the commit itself. const diff = git(["-c", "core.quotePath=false", "diff", "--raw", "--no-renames", "-z", from.out, tip.out]) if (!diff.ok) return `**Could not open the pull request:** ${diff.err || "could not list its changes"}.` diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index 197eb89..8362204 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -414,6 +414,22 @@ export function handedOffPrRequest({ req, pr, prBranch, defaultBranch, onReviewe } } +export const REWROTE_HISTORY = + "the run's branch doesn't build on the commit that was reviewed (it rewrote history), so its changes can't be trusted as a fix for this pull request" + +/** + * Does the agent's branch build on `reviewed`? An agent that reset or rebased past it + * (e.g. `git reset HEAD^`) hands back a tree whose diff would undo the reviewed work. + * Returns { ok }, { refused }, or { error } when git couldn't tell. + */ +export function checkHandoffDescends({ git, reviewed, tip }) { + if (!reviewed || !tip) return { refused: true } + const r = git(["merge-base", "--is-ancestor", reviewed, tip]) + if (r.ok) return { ok: true } + // Exit 1 with nothing on stderr means "not an ancestor"; anything else is git failing. + return String(r.err ?? "").trim() ? { error: r.err.trim().split("\n")[0] } : { refused: true } +} + /** * One commit, on `base`, with the branch's changes except `paths` (put back as they were * at `base`). Its parent is `base`, not `tip`, so the left-out files are not in what is diff --git a/test-companion/test/app-auth.test.mjs b/test-companion/test/app-auth.test.mjs index 4bd6305..f4d1d82 100644 --- a/test-companion/test/app-auth.test.mjs +++ b/test-companion/test/app-auth.test.mjs @@ -7,6 +7,8 @@ import { signJwt, installationTokenRequest, appApi, + isRetryable, + retryDelayMs, installationLookupMessage, } from "../src/app-auth.mjs" @@ -107,7 +109,7 @@ describe("appApi", () => { const r = await appApi("repos/o/r/installation", { jwt: "J", fetchImpl: fakeFetch(200, { id: 7 }, seen) }) assert.equal(seen.init.headers.Authorization, "Bearer J") assert.equal(seen.url, "https://api.github.com/repos/o/r/installation") - assert.deepEqual(r, { ok: true, status: 200, data: { id: 7 }, reason: "" }) + assert.deepEqual(r, { ok: true, status: 200, data: { id: 7 }, reason: "", attempts: 1 }) }) test("uses the server's API address when given", async () => { @@ -132,7 +134,7 @@ describe("appApi", () => { }) test("a network failure is reported, not thrown", async () => { - const r = await appApi("x", { jwt: "J", fetchImpl: async () => { throw new Error("ECONNRESET") } }) + const r = await appApi("x", { jwt: "J", sleep: async () => {}, fetchImpl: async () => { throw new Error("ECONNRESET") } }) assert.equal(r.ok, false) assert.equal(r.status, 0) assert.match(r.reason, /ECONNRESET/) @@ -153,3 +155,76 @@ describe("installationLookupMessage", () => { assert.doesNotMatch(m.text, /not installed/) }) }) + +describe("appApi retries a passing failure", () => { + // A response whose status and body change call by call; `throws` makes that call a network error. + const sequence = (steps) => { + let i = 0 + const fetchImpl = async () => { + const step = steps[Math.min(i++, steps.length - 1)] + if (step.throws) throw new Error(step.throws) + return { + ok: step.status >= 200 && step.status < 300, + status: step.status, + statusText: "", + headers: { get: (h) => (h.toLowerCase() === "retry-after" ? (step.retryAfter ?? null) : null) }, + json: async () => step.body ?? {}, + } + } + return { fetchImpl, calls: () => i } + } + const noSleep = [] + const sleep = async (ms) => { + noSleep.push(ms) + } + + test("a network error, then success", async () => { + const s = sequence([{ throws: "fetch failed" }, { status: 200, body: { id: 7 } }]) + const r = await appApi("x", { jwt: "J", fetchImpl: s.fetchImpl, sleep }) + assert.equal(r.ok, true) + assert.equal(r.attempts, 2) + assert.equal(s.calls(), 2) + }) + + test("a 502, then success", async () => { + const s = sequence([{ status: 502 }, { status: 201, body: { token: "t" } }]) + const r = await appApi("x", { jwt: "J", method: "POST", body: {}, fetchImpl: s.fetchImpl, sleep }) + assert.equal(r.ok, true) + assert.equal(r.data.token, "t") + assert.equal(r.attempts, 2) + }) + + test("401, 403, 404 and 422 are never retried", async () => { + for (const status of [401, 403, 404, 422]) { + const s = sequence([{ status, body: { message: "no" } }, { status: 200 }]) + const r = await appApi("x", { jwt: "J", fetchImpl: s.fetchImpl, sleep }) + assert.equal(r.ok, false, String(status)) + assert.equal(s.calls(), 1, String(status)) + } + }) + + test("three failures in a row give up with the last failure, as before", async () => { + const s = sequence([{ throws: "fetch failed" }]) + const r = await appApi("x", { jwt: "J", fetchImpl: s.fetchImpl, sleep }) + assert.equal(r.ok, false) + assert.equal(r.status, 0) + assert.equal(r.attempts, 3) + assert.equal(s.calls(), 3) + assert.match(r.reason, /could not reach GitHub: fetch failed/) + assert.match(installationLookupMessage(r).text, /could not sign in as the GitHub App \(could not reach GitHub: fetch failed\)/) + }) + + test("waits 1s then 3s, and a Retry-After wins, capped at 10s", () => { + assert.equal(retryDelayMs({ status: 0 }, 1), 1000) + assert.equal(retryDelayMs({ status: 502 }, 2), 3000) + assert.equal(retryDelayMs({ status: 429, retryAfter: "2" }, 1), 2000) + assert.equal(retryDelayMs({ status: 429, retryAfter: "120" }, 1), 10000) + }) + + test("only network errors, 429 and 5xx are worth another try", () => { + assert.equal(isRetryable({ status: 0 }), true) + assert.equal(isRetryable({ status: 429 }), true) + assert.equal(isRetryable({ status: 503 }), true) + for (const status of [200, 201, 401, 403, 404, 422]) assert.equal(isRetryable({ status }), false, String(status)) + }) +}) diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index 736ccc8..296b657 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -28,6 +28,8 @@ import { commitIdentity, savingFor, leaveOut, + checkHandoffDescends, + REWROTE_HISTORY, leaveOutMessage, classifyHandoffPush, pushFailureReason, @@ -1047,3 +1049,55 @@ describe("leaveOut on a git without --no-refresh", () => { assert.ok(!calls.filter((a) => a[0] === "reset")[1].includes("--no-refresh")) }) }) + +describe("a hand-off branch must build on the reviewed commit", () => { + const identity = { name: "Test Companion", email: BOT_EMAIL } + const id = ["-c", "user.name=t", "-c", "user.email=t@example.com"] + + // main → R (the reviewed PR head, with the PR's own work in a.ts and tests). + function reviewedRepo() { + const { dir, git, base: main, tip: reviewed } = repoWithBranch({ "a.ts": "fixed\n", "a.test.ts": "ten tests\n" }) + const must = (args) => { + const r = git(args) + assert.ok(r.ok, `${args.join(" ")}: ${r.err}`) + return r.out + } + return { dir, git, must, main, reviewed } + } + + test("an agent that reset past the reviewed commit is refused", () => { + const { dir, git, must, main, reviewed } = reviewedRepo() + // What the agent did on the pilot: `git reset HEAD^`, then committed on its own branch. + must(["checkout", "-q", "-b", "tc/fix/x", reviewed]) + must(["reset", "-q", "--hard", main]) + writeFileSync(join(dir, "a.ts"), "old again\n") + must(["add", "a.ts"]) + must([...id, "commit", "-qm", "undo"]) + const tip = must(["rev-parse", "HEAD"]) + assert.deepEqual(checkHandoffDescends({ git, reviewed, tip }), { refused: true }) + assert.match(REWROTE_HISTORY, /doesn't build on the commit that was reviewed/) + }) + + test("an agent that builds on the reviewed commit is accepted, and the squash sits on it", () => { + const { dir, git, must, reviewed } = reviewedRepo() + must(["checkout", "-q", "-b", "tc/fix/y", reviewed]) + writeFileSync(join(dir, "a.ts"), "fixed better\n") + writeFileSync(join(dir, "browserstack.err"), "[object Object]") + must(["add", "a.ts", "browserstack.err"]) + must([...id, "commit", "-qm", "fix"]) + const tip = must(["rev-parse", "HEAD"]) + assert.deepEqual(checkHandoffDescends({ git, reviewed, tip }), { ok: true }) + const out = leaveOut({ git, base: reviewed, tip, paths: ["browserstack.err"], identity, message: "m", indexFile: join(dir, "scratch") }) + assert.ok(out.sha, JSON.stringify(out)) + // Parent is the reviewed commit itself, and the diff on top of it is only the agent's change. + assert.equal(git(["rev-parse", `${out.sha}^`]).out, reviewed) + assert.equal(git(["diff", "--name-only", reviewed, out.sha]).out, "a.ts") + assert.equal(git(["show", `${out.sha}:a.test.ts`]).out, "ten tests") + }) + + test("missing commits are refused; a git failure is reported, not taken as yes", () => { + assert.deepEqual(checkHandoffDescends({ git: () => ({ ok: true }), reviewed: "", tip: "t" }), { refused: true }) + const broken = () => ({ ok: false, err: "fatal: Not a valid commit name zzz" }) + assert.deepEqual(checkHandoffDescends({ git: broken, reviewed: "zzz", tip: "t" }), { error: "fatal: Not a valid commit name zzz" }) + }) +}) From 72d2a207f4d3508ed5c3809d6b078bd67bddd0bf Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Thu, 12 Nov 2026 23:58:11 +0530 Subject: [PATCH 11/31] =?UTF-8?q?test-companion:=20react=20=F0=9F=91=80=20?= =?UTF-8?q?to=20a=20request=20once=20it's=20picked=20up?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a person with access comments a request to @browserstack, or a go for a waiting plan, the Action adds an eyes reaction to that comment as soon as it accepts it, so they know it was seen before the reply lands. Bot comments, refused comments and a bare go with no plan to approve get no reaction. The reaction is best effort: if it can't be added, the run carries on with a notice. No new permissions: pull-requests: write already covers reactions on pull request comments. Co-Authored-By: Claude Opus 5.5 --- test-companion/README.md | 1 + test-companion/src/access.mjs | 28 +++++++ test-companion/src/main.mjs | 23 +++++- test-companion/test/pickup-reaction.test.mjs | 81 ++++++++++++++++++++ 4 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 test-companion/test/pickup-reaction.test.mjs diff --git a/test-companion/README.md b/test-companion/README.md index beb25d5..66a0e87 100644 --- a/test-companion/README.md +++ b/test-companion/README.md @@ -67,6 +67,7 @@ from the release you tested; a version without a checksum is refused. Before a job changes anything, Test Companion posts its plan and waits for someone with write access to reply `go`. Read the plan before you reply. +Test Companion reacts 👀 to a request, or a `go`, once it picks it up. Without the GitHub App, Test Companion posts as `github-actions[bot]`, the account every workflow in your repository shares, and recognises its own plans diff --git a/test-companion/src/access.mjs b/test-companion/src/access.mjs index b762037..fc60889 100644 --- a/test-companion/src/access.mjs +++ b/test-companion/src/access.mjs @@ -8,6 +8,34 @@ export function mayAsk({ hasWriteAccess, askTeam, inTeam } = {}) { return inTeam === true } +/** + * The comment to mark with 👀 once a request is picked up, or undefined. Only a comment a + * person with access wrote to us: never a bot's, never a refused one, and a bare `go` only + * when there is a plan for it to approve. + */ +export function pickupReactionTarget({ eventName, comment, plan, allowed, approvalHasPlan } = {}) { + if (eventName !== "issue_comment" || !plan || plan.skip || plan.event !== "pr_comment") return undefined + if (allowed !== true) return undefined + const author = comment?.user + if (!author || author.type === "Bot" || /\[bot\]$/i.test(author.login ?? "")) return undefined + if (plan.approvalOnly && approvalHasPlan !== true) return undefined + const id = Number(comment?.id) + return Number.isSafeInteger(id) && id > 0 ? id : undefined +} + +/** + * Add 👀 to a comment. Best-effort: `post(path, body)` is the API call; any failure, + * thrown or returned, is reported back and never stops the run. Returns true when added. + */ +export function addPickupReaction({ repo, commentId, post }) { + if (!repo || !commentId || typeof post !== "function") return false + try { + return post(`repos/${repo}/issues/comments/${commentId}/reactions`, { content: "eyes" }) === true + } catch { + return false + } +} + /** Whether this run can write to the configured tests repository. `canPush` undefined counts as no. */ export function describeTestRepoAccess({ testsRepo, thisRepo, canPush } = {}) { const target = String(testsRepo ?? "").trim() diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index f43908c..df015e4 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -19,7 +19,7 @@ import { LOOKUP_FAILED_REPLY, } from "./runs.mjs" import { decideGate, GATE, describeChangeSize, ANSWER_JOBS } from "./gate.mjs" -import { mayAsk, describeTestRepoAccess } from "./access.mjs" +import { mayAsk, describeTestRepoAccess, pickupReactionTarget, addPickupReaction } from "./access.mjs" import { findTicket } from "./ticket.mjs" import { childEnvFor } from "./child-env.mjs" import { canIsolate, prepareSandbox, stopSandbox, handToAgent, writeAgentEnv, readAgentFile } from "./sandbox.mjs" @@ -155,6 +155,24 @@ function ghRaw(args, body) { return runGh(args, { token: env.GH_TOKEN, input: body ? JSON.stringify(redactDeep(body, secretsNow())) : undefined }) } +/** 👀 on the request comment once it is picked up. Never fails the run. */ +function reactToPickup(plan, allowed, approvalHasPlan) { + const commentId = pickupReactionTarget({ + eventName: env.GITHUB_EVENT_NAME, + comment: readEvent().comment, + plan, + allowed, + approvalHasPlan, + }) + if (!commentId) return + const added = addPickupReaction({ + repo: REPO, + commentId, + post: (path, body) => ghRaw(["api", path, "--method", "POST", "--input", "-"], body).status === 0, + }) + if (!added) notice("Test Companion: could not add the 👀 reaction to the request; carrying on.") +} + /** `gh api`, returning parsed JSON or null. Never throws. */ function gh(args, body) { const r = ghRaw(args, body) @@ -512,6 +530,8 @@ async function main() { if (!plan.approvalOnly) replyOnPr(plan.context?.pr, renderNotAuthorized(who ?? "there", askTeam)) finishSkipped("not-authorized", who) } + // A request addressed to us is picked up now; a bare "go" once we know it has a plan. + reactToPickup(plan, allowed, false) } // Place the request before spawning the CLI, so one we can't place costs nothing. @@ -595,6 +615,7 @@ async function main() { } finishSkipped("not-addressed-to-us") } + if (plan.approvalOnly) reactToPickup(plan, true, true) const gate = decideGate({ intent, approval: binding?.approval, diff --git a/test-companion/test/pickup-reaction.test.mjs b/test-companion/test/pickup-reaction.test.mjs new file mode 100644 index 0000000..dd08aa4 --- /dev/null +++ b/test-companion/test/pickup-reaction.test.mjs @@ -0,0 +1,81 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { pickupReactionTarget, addPickupReaction } from "../src/access.mjs" + +const person = { id: 123456, user: { login: "dev", type: "User" } } +const request = { event: "pr_comment", approvalOnly: false } +const bareGo = { event: "pr_comment", approvalOnly: true } +const base = { eventName: "issue_comment", comment: person, plan: request, allowed: true } + +describe("pickupReactionTarget — which comment gets 👀", () => { + test("an accepted @browserstack request", () => { + assert.equal(pickupReactionTarget(base), 123456) + }) + + test("an accepted go with a plan waiting", () => { + assert.equal(pickupReactionTarget({ ...base, plan: bareGo, approvalHasPlan: true }), 123456) + }) + + test("a bare go with no plan to approve is ignored, so no reaction", () => { + assert.equal(pickupReactionTarget({ ...base, plan: bareGo, approvalHasPlan: false }), undefined) + assert.equal(pickupReactionTarget({ ...base, plan: bareGo }), undefined) + }) + + test("never a bot's comment", () => { + assert.equal(pickupReactionTarget({ ...base, comment: { id: 1, user: { login: "browserstack[bot]", type: "Bot" } } }), undefined) + assert.equal(pickupReactionTarget({ ...base, comment: { id: 1, user: { login: "renovate[bot]", type: "User" } } }), undefined) + }) + + test("never a refused comment (no write access)", () => { + assert.equal(pickupReactionTarget({ ...base, allowed: false }), undefined) + assert.equal(pickupReactionTarget({ ...base, allowed: undefined }), undefined) + }) + + test("never a comment that wasn't addressed to us", () => { + assert.equal(pickupReactionTarget({ ...base, plan: { skip: "not-addressed-to-us" } }), undefined) + assert.equal(pickupReactionTarget({ ...base, plan: undefined }), undefined) + }) + + test("never outside a pull request comment", () => { + assert.equal(pickupReactionTarget({ ...base, eventName: "pull_request" }), undefined) + assert.equal(pickupReactionTarget({ ...base, plan: { event: "pr_opened" } }), undefined) + }) + + test("no usable comment id, no reaction", () => { + assert.equal(pickupReactionTarget({ ...base, comment: { user: person.user } }), undefined) + assert.equal(pickupReactionTarget({ ...base, comment: { id: "abc", user: person.user } }), undefined) + assert.equal(pickupReactionTarget({ ...base, comment: { id: -5, user: person.user } }), undefined) + }) +}) + +describe("addPickupReaction — best effort", () => { + test("posts eyes to the comment's reactions", () => { + const calls = [] + const ok = addPickupReaction({ repo: "o/r", commentId: 42, post: (path, body) => (calls.push([path, body]), true) }) + assert.equal(ok, true) + assert.deepEqual(calls, [["repos/o/r/issues/comments/42/reactions", { content: "eyes" }]]) + }) + + test("a refused call (403/404) returns false, never throws", () => { + assert.equal(addPickupReaction({ repo: "o/r", commentId: 42, post: () => false }), false) + }) + + test("a thrown error (network) returns false, never throws", () => { + assert.equal( + addPickupReaction({ + repo: "o/r", + commentId: 42, + post: () => { + throw new Error("ECONNRESET") + }, + }), + false, + ) + }) + + test("nothing to react to, no call", () => { + let called = false + assert.equal(addPickupReaction({ repo: "o/r", commentId: undefined, post: () => (called = true) }), false) + assert.equal(called, false) + }) +}) From 4427a56fa1b17f245fd4ffb85c4605983497c4c9 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 13 Nov 2026 00:03:27 +0530 Subject: [PATCH 12/31] test-companion: Asked by shows who asked and the comment they asked in The report table's Asked by row now reads `@who | #`, the id linking to the comment that asked for the run, so a reply can always be traced back to its request. The pipe is escaped so the table keeps two columns; without a comment link the row shows just who asked. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/report.mjs | 16 ++++++++++- test-companion/test/report.test.mjs | 12 ++++++--- test-companion/test/request-link.test.mjs | 33 +++++++++++++++++++++++ 3 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 test-companion/test/request-link.test.mjs diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index bb80907..9144ee8 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -8,6 +8,20 @@ const withMark = (outcome) => { return mark ? `${mark} ${outcome}` : String(outcome ?? "") } +/** `[#]()` for the comment that asked for this run; "" unless the URL names a comment. */ +export function requestLink(url) { + const u = String(url ?? "") + const id = /^https:\/\/[^\s)]+#issuecomment-(\d+)$/.exec(u)?.[1] + return id ? `[#${id}](${u})` : "" +} + +/** `@ | [#]()`; the pipe is escaped so it stays inside the table cell. */ +export function askedByCell(askedBy, askedAt) { + if (!askedBy) return "" + const link = requestLink(askedAt) + return link ? `@${askedBy} \\| ${link}` : `@${askedBy}` +} + /** * Only comments posted by our own bot login count. Another App or workflow * could post our markers, and a `go` would then run their text as the plan. @@ -147,7 +161,7 @@ export function renderJobSummary({ ["Took", summary.duration], ["Cost", summary.credits ? `${summary.credits} credits` : ""], ["CLI", cliVersion], - ["Asked by", askedBy ? (askedAt ? `[@${askedBy}](${askedAt})` : `@${askedBy}`) : ""], + ["Asked by", askedByCell(askedBy, askedAt)], ["Plan", approvedPlan?.url ? `[#${approvedPlan.id}](${approvedPlan.url})` : ""], ].filter(([, v]) => v) diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs index 7dfe007..bc1e6e8 100644 --- a/test-companion/test/report.test.mjs +++ b/test-companion/test/report.test.mjs @@ -1133,14 +1133,20 @@ describe("the report says which request it is answering", () => { assert.ok(body.includes(asked), "the asking comment is not linked") }) - test("the link is labelled with who asked", () => { + test("Asked by reads `@who | #comment id`, the id linking to the asking comment", () => { const body = renderJobSummary(view({ askedBy: "octocat", askedAt: asked })) - assert.ok(body.includes(`[@octocat](${asked})`)) + assert.ok(body.includes(`| Asked by | @octocat \\| [#6579040229](${asked}) |`)) + }) + + test("the escaped pipe keeps the table at two columns", () => { + const body = renderJobSummary(view({ askedBy: "octocat", askedAt: asked })) + const row = body.split("\n").find((l) => l.startsWith("| Asked by |")) + assert.equal(row.replace(/\\\|/g, "").split("|").length - 2, 2) }) test("knowing who asked but not where still names them", () => { const body = renderJobSummary(view({ askedBy: "octocat" })) - assert.ok(body.includes("@octocat")) + assert.ok(body.includes("| Asked by | @octocat |")) }) // A run nobody asked for — opened on a pull request, a red build, a timer. diff --git a/test-companion/test/request-link.test.mjs b/test-companion/test/request-link.test.mjs new file mode 100644 index 0000000..0f8a8ed --- /dev/null +++ b/test-companion/test/request-link.test.mjs @@ -0,0 +1,33 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { requestLink, askedByCell } from "../src/report.mjs" + +describe("requestLink: the report names the comment that asked for it", () => { + test("a comment link becomes its id, linked", () => { + const url = "https://github.com/o/r/pull/945#issuecomment-6830219211" + assert.equal(requestLink(url), `[#6830219211](${url})`) + }) + + test("anything that isn't a comment link gives no row", () => { + assert.equal(requestLink(undefined), "") + assert.equal(requestLink(""), "") + assert.equal(requestLink("https://github.com/o/r/pull/945"), "") + assert.equal(requestLink("https://github.com/o/r/pull/945#issuecomment-12) [x](https://evil.example"), "") + assert.equal(requestLink("javascript:alert(1)#issuecomment-1"), "") + }) +}) + + +describe("askedByCell", () => { + test("who and the linked comment id, pipe escaped", () => { + const url = "https://github.com/o/r/pull/9#issuecomment-123" + assert.equal(askedByCell("octocat", url), `@octocat \\| [#123](${url})`) + }) + test("no comment link: just who", () => { + assert.equal(askedByCell("octocat", undefined), "@octocat") + assert.equal(askedByCell("octocat", "https://github.com/o/r/pull/9"), "@octocat") + }) + test("nobody asked: empty, so the row is dropped", () => { + assert.equal(askedByCell(undefined, "https://github.com/o/r/pull/9#issuecomment-1"), "") + }) +}) From acc1c5732d3e2438250f878ceb5cf2686314bb88 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 13 Nov 2026 00:19:25 +0530 Subject: [PATCH 13/31] test-companion: time-limit App calls, and never guess the reviewed commit - Each call made as the App now gives up after 10 seconds. Without a limit a stalled connection waited about five minutes, and each retry again, so the retry could hold a job for a quarter of an hour. - Opening a handed-back fix no longer falls back to HEAD when the commit the run started on wasn't recorded: after the run the agent decides where HEAD points. It refuses instead. - The Asked by comment link rejects URLs carrying a pipe, brackets or angle brackets, so it can never split the table cell. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/app-auth.mjs | 25 ++++++++++++++++++++--- test-companion/src/main.mjs | 4 +++- test-companion/src/report.mjs | 2 +- test-companion/test/app-auth.test.mjs | 22 ++++++++++++++++++++ test-companion/test/request-link.test.mjs | 7 +++++++ 5 files changed, 55 insertions(+), 5 deletions(-) diff --git a/test-companion/src/app-auth.mjs b/test-companion/src/app-auth.mjs index 696095b..9ec723e 100644 --- a/test-companion/src/app-auth.mjs +++ b/test-companion/src/app-auth.mjs @@ -28,10 +28,13 @@ export function jwtClaims(appId, nowSeconds) { * and GitHub accepts an App's JWT only as `Authorization: Bearer`. * Returns { ok, status, data, reason }; `reason` is GitHub's message, never the JWT. */ -export async function appApi(path, { jwt, method = "GET", body, apiUrl, fetchImpl = fetch, attempts = 3, sleep = defaultSleep } = {}) { +export async function appApi( + path, + { jwt, method = "GET", body, apiUrl, fetchImpl = fetch, attempts = 3, sleep = defaultSleep, timeoutMs = APP_CALL_TIMEOUT_MS } = {}, +) { let result for (let attempt = 1; attempt <= attempts; attempt++) { - result = await appApiOnce(path, { jwt, method, body, apiUrl, fetchImpl }) + result = await appApiOnce(path, { jwt, method, body, apiUrl, fetchImpl, timeoutMs }) if (!isRetryable(result) || attempt === attempts) return { ...result, attempts: attempt } await sleep(retryDelayMs(result, attempt)) } @@ -40,6 +43,9 @@ export async function appApi(path, { jwt, method = "GET", body, apiUrl, fetchImp const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) +/** One App call gives up after this long; a timeout counts as a network error, so it is retried. */ +export const APP_CALL_TIMEOUT_MS = 10_000 + /** A network failure, a 5xx or a 429 may pass on a second try; 401/403/404/422 never will. */ export function isRetryable(result) { const status = Number(result?.status) @@ -53,8 +59,20 @@ export function retryDelayMs(result, attempt) { return attempt === 1 ? 1000 : 3000 } -async function appApiOnce(path, { jwt, method, body, apiUrl, fetchImpl }) { +async function appApiOnce(path, { jwt, method, body, apiUrl, fetchImpl, timeoutMs }) { const base = String(apiUrl || "https://api.github.com").replace(/\/+$/, "") + // Without a limit a stalled connection waits ~5 minutes, and each retry again. Covers the + // body too; aborting rejects fetch, which reads below as a network error and is retried. + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(new Error(`no reply within ${timeoutMs / 1000}s`)), timeoutMs) + try { + return await appApiCall(base, path, { jwt, method, body, fetchImpl, signal: controller.signal }) + } finally { + clearTimeout(timer) + } +} + +async function appApiCall(base, path, { jwt, method, body, fetchImpl, signal }) { let response try { response = await fetchImpl(`${base}/${path}`, { @@ -66,6 +84,7 @@ async function appApiOnce(path, { jwt, method, body, apiUrl, fetchImpl }) { ...(body === undefined ? {} : { "Content-Type": "application/json" }), }, body: body === undefined ? undefined : JSON.stringify(body), + signal, }) } catch (e) { return { ok: false, status: 0, data: undefined, reason: `could not reach GitHub: ${e?.message ?? e}` } diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index df015e4..4534a84 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -1245,7 +1245,9 @@ function openHandedOffPr(path, plan) { if (!tip.ok || !tip.out) return `**Did not open a pull request:** the run did not leave a \`${req.branch}\` branch.` // The squash goes onto the commit the run started on, and only if the agent's branch builds // on it: a branch that reset or rebased past it would hand back a diff that undoes that work. - const reviewed = git(["rev-parse", "--verify", "--quiet", `${headBeforeRun || "HEAD"}^{commit}`]) + // Never fall back to HEAD here: after the run, the agent decides where HEAD points. + if (!headBeforeRun) return "**Could not open the pull request:** could not read the commit the run started on." + const reviewed = git(["rev-parse", "--verify", "--quiet", `${headBeforeRun}^{commit}`]) if (!reviewed.ok || !reviewed.out) return "**Could not open the pull request:** could not read the commit the run started on." const descends = checkHandoffDescends({ git, reviewed: reviewed.out, tip: tip.out }) if (descends.error) return `**Could not open the pull request:** ${descends.error}.` diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index 9144ee8..147c3e6 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -11,7 +11,7 @@ const withMark = (outcome) => { /** `[#]()` for the comment that asked for this run; "" unless the URL names a comment. */ export function requestLink(url) { const u = String(url ?? "") - const id = /^https:\/\/[^\s)]+#issuecomment-(\d+)$/.exec(u)?.[1] + const id = /^https:\/\/[^\s()|<>[\]]+#issuecomment-(\d+)$/.exec(u)?.[1] return id ? `[#${id}](${u})` : "" } diff --git a/test-companion/test/app-auth.test.mjs b/test-companion/test/app-auth.test.mjs index f4d1d82..369e007 100644 --- a/test-companion/test/app-auth.test.mjs +++ b/test-companion/test/app-auth.test.mjs @@ -228,3 +228,25 @@ describe("appApi retries a passing failure", () => { for (const status of [200, 201, 401, 403, 404, 422]) assert.equal(isRetryable({ status }), false, String(status)) }) }) + +describe("appApi gives each call a time limit", () => { + test("every request carries an abort signal", async () => { + let seen + await appApi("x", { jwt: "J", fetchImpl: async (_u, init) => ((seen = init), { ok: true, status: 200, json: async () => ({}) }) }) + assert.ok(seen.signal instanceof AbortSignal) + }) + + test("a call that stalls is aborted, counted as a network error and retried", async () => { + let calls = 0 + const fetchImpl = (_u, init) => + new Promise((_resolve, reject) => { + calls++ + init.signal.addEventListener("abort", () => reject(init.signal.reason)) + }) + const r = await appApi("x", { jwt: "J", fetchImpl, sleep: async () => {}, timeoutMs: 5 }) + assert.equal(r.ok, false) + assert.equal(r.status, 0) + assert.match(r.reason, /no reply within/) + assert.equal(calls, 3) + }) +}) diff --git a/test-companion/test/request-link.test.mjs b/test-companion/test/request-link.test.mjs index 0f8a8ed..5dce9d6 100644 --- a/test-companion/test/request-link.test.mjs +++ b/test-companion/test/request-link.test.mjs @@ -31,3 +31,10 @@ describe("askedByCell", () => { assert.equal(askedByCell(undefined, "https://github.com/o/r/pull/9#issuecomment-1"), "") }) }) + +describe("requestLink rejects table and markdown breakers", () => { + test("a pipe or bracket in the URL gives no link", () => { + assert.equal(requestLink("https://github.com/o/r|x/pull/9#issuecomment-1"), "") + assert.equal(requestLink("https://github.com/o/r]x/pull/9#issuecomment-1"), "") + }) +}) From aa59121c90b3241564a7472f34d457c21221f483 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 13 Nov 2026 15:45:05 +0530 Subject: [PATCH 14/31] test-companion: a plain go on a generate plan saves exactly the planned cases A generate plan is a preview that lists the cases it would create. Replying go used to start a fresh run that generated the cases again, so what reached Test Management was not what the reviewer approved. Now the plan's run keeps its cases as data (the CLI's result JSON, plus the repo and commit it was made for) in a test-companion-cases artifact, and the plan records its run id. A plain go downloads that artifact, checks it is for this repository and this commit, and saves it with `tc tm add --from-file`, which runs no AI. The reply lists each created case with its id from tm add's own output, and any that failed. A go with edits still generates again with the edits as guidance, and the reply says so. If the cases can't be found, have expired, or Test Management refuses them before saving any, nothing is saved and the reply asks for a fresh plan; it never regenerates quietly. A partial save spends the plan, so approving it again can't save the same cases twice. A plan's report and result are now attached to its run too; before, a plan attached nothing. Co-Authored-By: Claude Opus 5.5 --- test-companion/README.md | 7 + test-companion/action.yml | 12 ++ test-companion/src/approved-cases.mjs | 181 +++++++++++++++++++ test-companion/src/main.mjs | 118 ++++++++++++- test-companion/src/report.mjs | 17 +- test-companion/test/approved-cases.test.mjs | 186 ++++++++++++++++++++ 6 files changed, 511 insertions(+), 10 deletions(-) create mode 100644 test-companion/src/approved-cases.mjs create mode 100644 test-companion/test/approved-cases.test.mjs diff --git a/test-companion/README.md b/test-companion/README.md index 66a0e87..ece1a55 100644 --- a/test-companion/README.md +++ b/test-companion/README.md @@ -69,6 +69,13 @@ Before a job changes anything, Test Companion posts its plan and waits for someone with write access to reply `go`. Read the plan before you reply. Test Companion reacts 👀 to a request, or a `go`, once it picks it up. +For `generate`, a plain `go` saves exactly the test cases in the plan to Test +Management: the plan's run keeps them (the `test-companion-cases` artifact, +for 7 days) and nothing is generated again. A `@browserstack go` with changes +generates the cases again with your changes as guidance, and says so. If the +plan's cases can't be found, nothing is saved and you are asked to generate a +fresh plan. + Without the GitHub App, Test Companion posts as `github-actions[bot]`, the account every workflow in your repository shares, and recognises its own plans by that account and a hidden marker. Another workflow that copies outside text diff --git a/test-companion/action.yml b/test-companion/action.yml index 225966f..764bf6a 100644 --- a/test-companion/action.yml +++ b/test-companion/action.yml @@ -234,6 +234,18 @@ runs: path: ${{ steps.run.outputs.result-file }} if-no-files-found: ignore + # A generate plan's cases as data. Approving the plan saves exactly these, so nothing + # is generated again. Only a plan sets cases-dir. + - name: Keep the planned test cases for approval + if: always() && steps.run.outputs.cases-dir != '' + # v4.6.2, pinned to a commit: a tag can be repointed. + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: test-companion-cases + path: ${{ steps.run.outputs.cases-dir }} + if-no-files-found: ignore + retention-days: 7 + # Matters on a self-hosted runner, which outlives the job. - name: Take gh's credentials back off the runner if: always() diff --git a/test-companion/src/approved-cases.mjs b/test-companion/src/approved-cases.mjs new file mode 100644 index 0000000..e696641 --- /dev/null +++ b/test-companion/src/approved-cases.mjs @@ -0,0 +1,181 @@ +/** + * Saving the exact test cases a reviewer approved. + * + * A `generate` plan is a preview: it lists the cases it would create and saves nothing. + * Its run keeps those cases as data (the CLI's `.json`) in an artifact. A plain + * `go` then saves that file with `tc tm add --from-file`, which runs no AI: what the reviewer + * read is what reaches Test Management. Before this, `go` generated the cases again and saved + * a different set. + * + * Everything here is pure; main.mjs does the API calls and runs tc. + */ +import { hasApproverEdits } from "./report.mjs" + +/** The artifact a generate preview leaves: the cases file plus subject.json naming its repo and commit. */ +export const CASES_ARTIFACT = "test-companion-cases" +export const CASES_DATA_FILE = "result.md.json" +export const CASES_SUBJECT_FILE = "subject.json" + +/** + * How a `go` on a plan is carried out: + * - "exact": save the plan's own cases with tm add, no new run; + * - "unavailable": should be exact, but the plan's cases can't be found (say so, never regenerate); + * - "run": today's behaviour, a real run of the job. `edited` is set when that is because the + * approver changed the plan, which tm add cannot apply. + * Only generate with a Test Management destination is saved exactly; other jobs, and a generate + * with nowhere to save, run as before. + */ +export function approvedSavePlan({ workflow, replyText, binding, runId } = {}) { + if (workflow !== "generate") return { mode: "run" } + const input = binding?.input ?? {} + const project = String(input.tm_project ?? "").trim() + const folder = String(input.tm_folder ?? "").trim() + const saveSetting = String(input["save-to-tm"] ?? "").trim().toLowerCase() + if (!project || !folder || saveSetting === "false") return { mode: "run" } + if (hasApproverEdits(replyText)) return { mode: "run", edited: true } + // Plans made before the run id was recorded have nothing to fetch. + if (!/^\d+$/.test(String(runId ?? ""))) return { mode: "unavailable", reason: "no-run" } + return { mode: "exact", project, folder, runId: String(runId) } +} + +/** The plan run's cases artifact from a `GET .../runs//artifacts` reply, or why there is none. */ +export function pickCasesArtifact(list, runId) { + const want = String(runId ?? "") + const mine = (list?.artifacts ?? []).filter((a) => a?.name === CASES_ARTIFACT && String(a?.workflow_run?.id ?? "") === want) + if (mine.length === 0) return { error: "missing" } + const live = mine.find((a) => !a.expired) + return live ? { artifact: live } : { error: "expired" } +} + +/** What the plan run records about itself, so a go run can't save cases made for another repo or commit. */ +export function casesSubject({ repo, sha }) { + return { repo: String(repo ?? ""), sha: String(sha ?? "") } +} + +/** Whether a downloaded subject.json matches the repo and commit being approved; `why` when not. */ +export function checkCasesSubject(subjectText, { repo, sha }) { + let subject + try { + subject = JSON.parse(String(subjectText ?? "")) + } catch { + return { ok: false, why: "unreadable" } + } + if (!subject || subject.repo !== repo) return { ok: false, why: "other-repo" } + if (!sha || subject.sha !== sha) return { ok: false, why: "other-commit" } + return { ok: true } +} + +/** + * `tc tm add --json` output: { project, folderId, created[], failed[], skipped[] }. + * Undefined when it printed no report, which means it stopped before saving anything. + */ +export function parseTmAddReport(stdout) { + const text = String(stdout ?? "").trim() + if (!text) return undefined + // The report is the last JSON object printed. + const start = text.lastIndexOf("\n{") + const candidate = start >= 0 ? text.slice(start + 1) : text + let report + try { + report = JSON.parse(candidate) + } catch { + try { + report = JSON.parse(text) + } catch { + return undefined + } + } + if (!report || typeof report !== "object" || !Array.isArray(report.created)) return undefined + return { + project: String(report.project ?? ""), + folderId: String(report.folderId ?? ""), + created: report.created, + failed: Array.isArray(report.failed) ? report.failed : [], + skipped: Array.isArray(report.skipped) ? report.skipped : [], + } +} + +const UNAVAILABLE_WHY = { + "no-run": "this plan was made before plans kept their cases, so there is no saved list to use", + missing: "the plan's run did not keep its cases", + expired: "the plan's saved cases have expired", + "download-failed": "I could not download the plan's saved cases", + unreadable: "the plan's saved cases could not be read", + "other-repo": "the saved cases are not from this repository", + "other-commit": "the saved cases are for a different commit than this plan", + "save-failed": "Test Management refused the cases before any were saved", +} + +/** The reply when the approved cases can't be saved as they are. Never regenerates. */ +export function renderCasesUnavailable(reason, detail) { + const why = UNAVAILABLE_WHY[reason] ?? "the plan's cases could not be used" + const extra = String(detail ?? "").trim() + return [ + `I did not save anything: ${why}${extra ? ` (${extra})` : ""}.`, + "", + "I have not generated a new set in its place, because that would save cases nobody reviewed. Ask me to **generate** again and approve the fresh plan.", + ].join("\n") +} + +const oneLine = (s) => String(s ?? "").replace(/\s+/g, " ").trim() + +/** The answer text for an exact save: what was created, with ids, and anything that failed. */ +export function renderExactSave(report, { project, folder } = {}) { + const created = report?.created ?? [] + const failed = report?.failed ?? [] + const skipped = report?.skipped ?? [] + const lines = [] + const where = `${folder} in ${project}` + if (created.length > 0) { + lines.push(`Saved the ${created.length} approved test case${created.length === 1 ? "" : "s"} to \`${where}\` exactly as planned; nothing was generated again.`) + lines.push("") + for (const c of created) { + const id = oneLine(c.identifier) + const label = c.url && id ? `[${id}](${c.url})` : id + lines.push(`- ${label}${label ? " — " : ""}${oneLine(c.title)}`) + } + } + if (failed.length > 0) { + if (lines.length) lines.push("") + lines.push(`**${failed.length} could not be saved:**`) + lines.push("") + for (const f of failed) lines.push(`- ${oneLine(f.title)}: ${oneLine(f.error)}`) + } + if (skipped.length > 0) { + if (lines.length) lines.push("") + lines.push(`${skipped.length} were already saved earlier and were left alone.`) + } + if (lines.length === 0) lines.push("The plan had no test cases to save.") + return lines.join("\n") +} + +/** The counts the report table shows, taken from what tm add actually did. */ +export function exactSaveSummary(report, { project, folder } = {}) { + const created = report?.created?.length ?? 0 + const failed = report?.failed?.length ?? 0 + const skipped = report?.skipped?.length ?? 0 + return { + workflow: "generate", + event: "pr_comment", + cases_generated: String(created + failed + skipped), + cases_saved: String(created + skipped), + saved_to: `${folder} in ${project}`, + } +} + +/** + * How an exact save ends. `spend` is whether the plan is used up: once any case is saved, + * approving the same plan again would save those cases twice, so it is spent even when some + * failed. When none were saved it stays approvable. + */ +export function exactSaveOutcome(report, outcomes) { + const created = report?.created?.length ?? 0 + const failed = report?.failed?.length ?? 0 + const skipped = report?.skipped?.length ?? 0 + if (failed === 0) return { outcome: outcomes.SUCCEEDED, spend: true } + return { outcome: outcomes.RUN_BROKE, spend: created + skipped > 0 } +} + +/** A note for the reply when a go with edits had to generate again. */ +export const EDITED_GO_NOTE = + "Your go included changes, and saving the plan as it was can't apply them, so the cases were generated again with your changes as guidance. Check the list below; it may differ from the plan." diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 4534a84..58936bc 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -1,5 +1,5 @@ /** The Action's entry point: I/O only. Decisions live in the pure modules beside it. */ -import { readFileSync, appendFileSync, writeFileSync, existsSync, rmSync, readdirSync, accessSync, mkdtempSync, lstatSync, constants } from "node:fs" +import { readFileSync, appendFileSync, writeFileSync, existsSync, rmSync, readdirSync, accessSync, mkdtempSync, mkdirSync, lstatSync, constants } from "node:fs" import { spawnSync } from "node:child_process" import { join } from "node:path" import { tmpdir } from "node:os" @@ -22,6 +22,21 @@ import { decideGate, GATE, describeChangeSize, ANSWER_JOBS } from "./gate.mjs" import { mayAsk, describeTestRepoAccess, pickupReactionTarget, addPickupReaction } from "./access.mjs" import { findTicket } from "./ticket.mjs" import { childEnvFor } from "./child-env.mjs" +import { + approvedSavePlan, + pickCasesArtifact, + casesSubject, + checkCasesSubject, + parseTmAddReport, + renderCasesUnavailable, + renderExactSave, + exactSaveSummary, + exactSaveOutcome, + EDITED_GO_NOTE, + CASES_ARTIFACT, + CASES_DATA_FILE, + CASES_SUBJECT_FILE, +} from "./approved-cases.mjs" import { canIsolate, prepareSandbox, stopSandbox, handToAgent, writeAgentEnv, readAgentFile } from "./sandbox.mjs" import { agentEnvFrom, asAgentCommand, checkPrHandoff, readClaimResult } from "./sandbox-policy.mjs" import { startRelay } from "./relay.mjs" @@ -116,6 +131,8 @@ const RESULT_FILE = join(REPORT_DIR, "test-companion-result.md") // Must match the upload step in action.yml. const RESULT_ARTIFACT = "test-companion-result" +// A generate plan's cases as data, uploaded as the CASES_ARTIFACT so approving it can save them as they are. +const CASES_DIR = join(REPORT_DIR, "test-companion-cases") const setOutput = (k, v) => env.GITHUB_OUTPUT && appendFileSync(env.GITHUB_OUTPUT, `${k}=${v}\n`) const addToJobSummary = (md) => env.GITHUB_STEP_SUMMARY && appendFileSync(env.GITHUB_STEP_SUMMARY, md + "\n") @@ -679,6 +696,16 @@ async function main() { : ` Approved by @${plan.commenter}, who also asked for it.` notice(`Test Companion:${approvalNote}`) } + + // A plain go on a generate plan saves the plan's own cases; nothing is generated again. + const save = approvedSavePlan({ + workflow: plan.workflow, + replyText, + binding: pickBinding(bindings, plan.workflow), + runId: waiting?.runId, + }) + if (save.mode === "exact" || save.mode === "unavailable") saveApprovedCases(plan, save, approvalNote) + if (save.edited) approvalNote = `${approvalNote} ${EDITED_GO_NOTE}` } else { // A large change: the run covers what matters and lists what it skipped. const size = describeChangeSize(pull) @@ -841,9 +868,9 @@ async function main() { headBeforeRun = git(["rev-parse", "HEAD"]).out notice(`tc ${args.join(" ")}`) // Remove stale files so a run that produces nothing can't report an earlier run's work. - for (const f of [SUMMARY_FILE, RESULT_FILE]) { + for (const f of [SUMMARY_FILE, RESULT_FILE, `${RESULT_FILE}.json`, CASES_DIR]) { try { - rmSync(f, { force: true }) + rmSync(f, { force: true, recursive: true }) } catch { // Not worth failing over. } @@ -862,10 +889,13 @@ async function main() { const text = readAgentFile(from) if (text !== undefined) writeFileSync(to, text) } + keepCases(readAgentFile(`${runFiles.result}.json`), plan) if (claim.held && exitCode !== 0) releaseClaim(claim) if (!plan.preview) handoffNote = openHandedOffPr(runFiles.handoff, plan) } + if (!sandbox) keepCases(readOptional(`${RESULT_FILE}.json`), plan) + // Absent if the process was killed. let summaryText try { @@ -963,9 +993,15 @@ async function main() { plan.commenter, // Record the job, so approving can't run a different one. plan.workflow ?? summary.workflow, + // And the run, so approving can fetch the cases it kept. + env.GITHUB_RUN_ID, ), ) postCheck(outcome, view, sha, [], "in_progress") + // A plan's files are attached too: approving a generate plan saves these cases. + if (summaryText) setOutput("summary-file", SUMMARY_FILE) + if (hasResult) setOutput("result-file", RESULT_FILE) + if (existsSync(join(CASES_DIR, CASES_DATA_FILE))) setOutput("cases-dir", CASES_DIR) setOutput("outcome", "Proposed") setOutput("should-fail", "false") process.exit(0) @@ -1550,6 +1586,82 @@ function readBindings(configFile, event, branch) { } } +function readOptional(path) { + try { + return readFileSync(path, "utf8") + } catch { + return undefined + } +} + +/** Keep the run's cases file, with the repo and commit it was made for, for the cases artifact. */ +function keepCases(text, plan) { + if (text === undefined) return + try { + mkdirSync(CASES_DIR, { recursive: true }) + writeFileSync(join(CASES_DIR, CASES_DATA_FILE), text) + writeFileSync(join(CASES_DIR, CASES_SUBJECT_FILE), JSON.stringify(casesSubject({ repo: REPO, sha: plan.context?.sha }))) + } catch { + // Without it a plan can still be read; approving it says the cases are missing. + } +} + +/** + * Save an approved generate plan's own cases with `tc tm add` and finish the run. No AI runs, so + * what was reviewed is what is saved. Any problem is reported and nothing is generated instead. + */ +function saveApprovedCases(plan, save, approvalNote) { + const sha = plan.context.sha + const base = { + workflow: "generate", + planWorkflow: "generate", + approvedPlan, + approvalNote, + runUrl: RUN_URL, + cliVersion: env.TC_ACTION_CLI_VERSION, + askedBy: plan.commenter, + askedAt: plan.askedAt, + summary: { workflow: "generate", event: plan.event }, + } + const finish = (view, spend) => { + addToJobSummary(renderJobSummary(view)) + setOutput("outcome", view.outcome) + setOutput("should-fail", String(stepShouldFail(env.TC_ACTION_BUILD_EFFECT, view.outcome))) + postCheck(view.outcome, view, sha) + if (spend) markPlanSpent(approvedPlan?.id, consumedMarker) + if (plan.context.pr) postComment(plan.context.pr, { ...view, consumedMarker: spend ? consumedMarker : "" }) + process.exit(0) + } + const unavailable = (reason, detail) => + finish({ ...base, outcome: OUTCOMES.COULD_NOT_RUN, answer: renderCasesUnavailable(reason, detail) }, false) + + if (save.mode !== "exact") return unavailable(save.reason) + const picked = pickCasesArtifact(gh(["api", `repos/${REPO}/actions/runs/${save.runId}/artifacts?per_page=100`]), save.runId) + if (picked.error) return unavailable(picked.error) + + const dir = mkdtempSync(join(REPORT_DIR, "tc-approved-cases-")) + const got = runGh(["run", "download", save.runId, "-R", REPO, "-n", CASES_ARTIFACT, "-D", dir], { token: env.GH_TOKEN }) + if (got.status !== 0) return unavailable("download-failed") + const subject = checkCasesSubject(readOptional(join(dir, CASES_SUBJECT_FILE)), { repo: REPO, sha }) + if (!subject.ok) return unavailable(subject.why) + const casesFile = join(dir, CASES_DATA_FILE) + if (!existsSync(casesFile)) return unavailable("unreadable") + + notice(`Test Companion: saving the approved plan's cases from run ${save.runId} to ${save.folder} in ${save.project}.`) + const r = spawnSync( + TC_PATH, + ["tm", "add", "--from-file", casesFile, "--project", save.project, "--folder", save.folder, "--json"], + { encoding: "utf8", env: childEnvFor(env), timeout: 300_000 }, + ) + const report = parseTmAddReport(r.stdout) + if (!report) { + const why = String(r.stderr ?? "").trim().split("\n").filter(Boolean).at(-1) + return unavailable("save-failed", redactSecrets(why, secretsNow())) + } + const { outcome, spend } = exactSaveOutcome(report, OUTCOMES) + finish({ ...base, outcome, summary: { ...base.summary, ...exactSaveSummary(report, save) }, answer: renderExactSave(report, save) }, spend) +} + /** The binding a comment is asking for, or the one bound to the event. */ function pickBinding(doc, workflow) { const list = (doc?.bindings ?? []).filter((b) => b.enabled) diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index 147c3e6..04608a1 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -281,21 +281,23 @@ export function renderStopRemoved() { export const PROPOSAL_WINDOW_MS = 24 * 60 * 60 * 1000 // Keyed to the commit, so a new push makes an old proposal unapprovable. -export function proposalMarkerFor(sha, requester, workflow) { +// `run` is the run that made the plan, so approving it can fetch the cases that run kept. +export function proposalMarkerFor(sha, requester, workflow, runId) { const by = String(requester ?? "").trim() const job = String(workflow ?? "").trim() - return `` + const run = /^\d+$/.test(String(runId ?? "")) ? String(runId) : "" + return `` } -// Groups: 1 = sha, 2 = requester, 3 = workflow. The sha is captured and compared, +// Groups: 1 = sha, 2 = requester, 3 = workflow, 4 = run id. The sha is captured and compared, // never built into the regex. -const MARKER = // +const MARKER = // // The last marker, since ours ends the comment and anything earlier could be planted. const readMarker = (body) => { const all = [...String(body ?? "").matchAll(new RegExp(MARKER.source, "g"))] const found = all.at(-1) - return found ? { sha: found[1], requester: found[2], workflow: found[3] } : undefined + return found ? { sha: found[1], requester: found[2], workflow: found[3], runId: found[4] } : undefined } // The commit our in-progress check went on (the PR head), which on a comment run is not GITHUB_SHA. @@ -310,13 +312,13 @@ export function isApprovableJob(workflow, enabledWorkflows) { return job !== "" && !job.startsWith("-") && (enabledWorkflows ?? []).includes(job) } -export function renderProposal(body, sha, requester, workflow) { +export function renderProposal(body, sha, requester, workflow, runId) { return [ body, "", "Reply **go** to run this for real. Nothing is written until you do. To change the list first, reply **@browserstack go** with your edits. Leave it and it stops being approvable after a day.", "", - proposalMarkerFor(sha, requester, workflow), + proposalMarkerFor(sha, requester, workflow, runId), ].join("\n") } @@ -416,6 +418,7 @@ export function findProposal(comments, sha, now = Date.now(), us, opts = {}) { expired: !(age <= PROPOSAL_WINDOW_MS), requester: mark?.requester, workflow: mark?.workflow, + runId: mark?.runId, body: newest.body ?? "", } } diff --git a/test-companion/test/approved-cases.test.mjs b/test-companion/test/approved-cases.test.mjs new file mode 100644 index 0000000..03341ab --- /dev/null +++ b/test-companion/test/approved-cases.test.mjs @@ -0,0 +1,186 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { readFileSync } from "node:fs" +import { + approvedSavePlan, + pickCasesArtifact, + casesSubject, + checkCasesSubject, + parseTmAddReport, + renderCasesUnavailable, + renderExactSave, + exactSaveSummary, + exactSaveOutcome, + CASES_ARTIFACT, +} from "../src/approved-cases.mjs" +import { OUTCOMES } from "../src/outcome.mjs" +import { proposalMarkerFor, renderProposal, findProposal } from "../src/report.mjs" + +const BINDING = { workflow: "generate", input: { "save-to-tm": "true", tm_project: "Checkout", tm_folder: "Web" } } + +describe("approvedSavePlan: a plain go on a generate plan saves the plan's own cases", () => { + test("plain go with a destination and a recorded run: exact save", () => { + assert.deepEqual(approvedSavePlan({ workflow: "generate", replyText: "go", binding: BINDING, runId: "1234567890" }), { + mode: "exact", + project: "Checkout", + folder: "Web", + runId: "1234567890", + }) + }) + + test("go with edits regenerates, and says it is because of the edits", () => { + assert.deepEqual(approvedSavePlan({ workflow: "generate", replyText: "@browserstack go, drop T003", binding: BINDING, runId: "1" }), { + mode: "run", + edited: true, + }) + }) + + test("a plan with no recorded run is unavailable, never regenerated", () => { + assert.deepEqual(approvedSavePlan({ workflow: "generate", replyText: "go", binding: BINDING, runId: undefined }), { + mode: "unavailable", + reason: "no-run", + }) + assert.equal(approvedSavePlan({ workflow: "generate", replyText: "go", binding: BINDING, runId: "12; rm -rf /" }).mode, "unavailable") + }) + + test("other jobs, and generate with nowhere to save, run as before", () => { + assert.deepEqual(approvedSavePlan({ workflow: "test-pr", replyText: "go", binding: BINDING, runId: "1" }), { mode: "run" }) + assert.deepEqual(approvedSavePlan({ workflow: "generate", replyText: "go", binding: { input: {} }, runId: "1" }), { mode: "run" }) + assert.deepEqual( + approvedSavePlan({ workflow: "generate", replyText: "go", binding: { input: { ...BINDING.input, "save-to-tm": "false" } }, runId: "1" }), + { mode: "run" }, + ) + }) +}) + +describe("pickCasesArtifact", () => { + const art = (over) => ({ name: CASES_ARTIFACT, expired: false, workflow_run: { id: 7 }, id: 1, ...over }) + + test("the plan run's live cases artifact", () => { + assert.deepEqual(pickCasesArtifact({ artifacts: [art({ name: "test-companion-report" }), art()] }, "7"), { artifact: art() }) + }) + test("none from that run is missing; only expired ones is expired", () => { + assert.deepEqual(pickCasesArtifact({ artifacts: [art({ workflow_run: { id: 8 } })] }, "7"), { error: "missing" }) + assert.deepEqual(pickCasesArtifact({ artifacts: [art({ expired: true })] }, "7"), { error: "expired" }) + assert.deepEqual(pickCasesArtifact(null, "7"), { error: "missing" }) + }) +}) + +describe("checkCasesSubject: the saved cases must be for this repo and this plan's commit", () => { + const subject = JSON.stringify(casesSubject({ repo: "o/r", sha: "abc123" })) + test("same repo and commit", () => { + assert.deepEqual(checkCasesSubject(subject, { repo: "o/r", sha: "abc123" }), { ok: true }) + }) + test("another repo, another commit, or junk is refused", () => { + assert.deepEqual(checkCasesSubject(subject, { repo: "o/other", sha: "abc123" }), { ok: false, why: "other-repo" }) + assert.deepEqual(checkCasesSubject(subject, { repo: "o/r", sha: "def456" }), { ok: false, why: "other-commit" }) + assert.deepEqual(checkCasesSubject("not json", { repo: "o/r", sha: "abc123" }), { ok: false, why: "unreadable" }) + assert.deepEqual(checkCasesSubject(undefined, { repo: "o/r", sha: "abc123" }), { ok: false, why: "unreadable" }) + }) +}) + +describe("parseTmAddReport reads tc tm add --json", () => { + const report = { + project: "PR-1", + folderId: "4242", + created: [{ sourceId: "T001", title: "Base delay", identifier: "TC-1", url: "https://tm.example/TC-1" }], + failed: [], + } + test("the report object, with missing lists filled in", () => { + assert.deepEqual(parseTmAddReport(JSON.stringify(report, null, 2)), { ...report, skipped: [] }) + }) + test("the last object when something was printed before it", () => { + assert.equal(parseTmAddReport(`Saving 1 case…\n${JSON.stringify(report)}`)?.created.length, 1) + }) + test("no report means it stopped before saving", () => { + assert.equal(parseTmAddReport(""), undefined) + assert.equal(parseTmAddReport("Error: project not found"), undefined) + assert.equal(parseTmAddReport('{"error":"x"}'), undefined) + }) +}) + +describe("what the reply says", () => { + const where = { project: "Checkout", folder: "Web" } + const created = [ + { title: "Base delay", identifier: "TC-1", url: "https://tm.example/TC-1" }, + { title: "Cap", identifier: "TC-2" }, + ] + + test("every saved case is listed with its id, from tm add's own output", () => { + const text = renderExactSave({ created, failed: [], skipped: [] }, where) + assert.match(text, /Saved the 2 approved test cases to `Web in Checkout` exactly as planned; nothing was generated again\./) + assert.match(text, /- \[TC-1\]\(https:\/\/tm\.example\/TC-1\) — Base delay/) + assert.match(text, /- TC-2 — Cap/) + }) + + test("a partial save names what failed", () => { + const text = renderExactSave({ created: created.slice(0, 1), failed: [{ title: "Cap", error: "priority not allowed" }], skipped: [] }, where) + assert.match(text, /Saved the 1 approved test case to/) + assert.match(text, /\*\*1 could not be saved:\*\*\n\n- Cap: priority not allowed/) + }) + + test("the table counts agree with the list", () => { + assert.deepEqual(exactSaveSummary({ created, failed: [{ title: "x", error: "y" }], skipped: [] }, where), { + workflow: "generate", + event: "pr_comment", + cases_generated: "3", + cases_saved: "2", + saved_to: "Web in Checkout", + }) + }) + + test("unavailable says why, saves nothing and does not regenerate", () => { + const text = renderCasesUnavailable("expired") + assert.match(text, /I did not save anything: the plan's saved cases have expired\./) + assert.match(text, /I have not generated a new set/) + assert.match(renderCasesUnavailable("save-failed", "folder not found"), /before any were saved \(folder not found\)\./) + }) +}) + +describe("exactSaveOutcome: once anything is saved the plan is spent", () => { + test("all saved: succeeded and spent", () => { + assert.deepEqual(exactSaveOutcome({ created: [{}], failed: [], skipped: [] }, OUTCOMES), { outcome: OUTCOMES.SUCCEEDED, spend: true }) + }) + test("some saved, some failed: broke, but spent so approving again can't save them twice", () => { + assert.deepEqual(exactSaveOutcome({ created: [{}], failed: [{}], skipped: [] }, OUTCOMES), { outcome: OUTCOMES.RUN_BROKE, spend: true }) + }) + test("none saved: broke and still approvable", () => { + assert.deepEqual(exactSaveOutcome({ created: [], failed: [{}], skipped: [] }, OUTCOMES), { outcome: OUTCOMES.RUN_BROKE, spend: false }) + }) +}) + +describe("the plan records the run that made it", () => { + test("the marker carries the run id, and findProposal reads it back", () => { + const body = renderProposal("plan", "abc123", "octocat", "generate", "1234567890") + assert.match(body, //) + const found = findProposal([{ id: 5, user: { type: "Bot", login: "bot[bot]" }, body, created_at: new Date().toISOString() }], "abc123", Date.now(), "bot[bot]") + assert.equal(found?.runId, "1234567890") + assert.equal(found?.workflow, "generate") + }) + test("a non-numeric run id is left out; old markers still read", () => { + assert.equal(proposalMarkerFor("abc", "o", "generate", "x y"), "") + const old = "plan\n" + const found = findProposal([{ id: 1, user: { type: "Bot", login: "b" }, body: old, created_at: new Date().toISOString() }], "abc", Date.now(), "b") + assert.equal(found?.runId, undefined) + assert.equal(found?.workflow, "generate") + }) +}) + +describe("the wiring", () => { + const main = readFileSync(new URL("../src/main.mjs", import.meta.url), "utf8") + const action = readFileSync(new URL("../action.yml", import.meta.url), "utf8") + + test("a plan attaches its cases, and approving one saves them with tm add instead of running again", () => { + assert.match(main, /setOutput\("cases-dir", CASES_DIR\)/) + assert.match(main, /env\.GITHUB_RUN_ID,\n\s*\),\n\s*\)/) + assert.match(main, /"tm", "add", "--from-file", casesFile, "--project", save\.project, "--folder", save\.folder, "--json"/) + assert.match(main, /if \(save\.mode === "exact" \|\| save\.mode === "unavailable"\) saveApprovedCases\(plan, save, approvalNote\)/) + }) + + test("action.yml uploads the cases artifact, and a plan's report and result too", () => { + assert.match(action, /name: test-companion-cases\n\s+path: \$\{\{ steps\.run\.outputs\.cases-dir \}\}/) + const proposing = main.slice(main.indexOf("if (proposing && !runDidNotWork(outcome))"), main.indexOf('setOutput("outcome", "Proposed")')) + assert.match(proposing, /setOutput\("result-file", RESULT_FILE\)/) + assert.match(proposing, /setOutput\("summary-file", SUMMARY_FILE\)/) + }) +}) From 181fa6da58331f350a5c165d02673bd51e806a1e Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 13 Nov 2026 16:39:53 +0530 Subject: [PATCH 15/31] test-companion: take the save destination from the approved plan's own file A plain go on a generate plan still regenerated in a live run. The exact save was gated on the pipeline binding's tm_project and tm_folder, but the bindings the CLI reports carry no inputs, so the gate never opened. The tests used a binding shape the CLI never prints. Now a plain go on a generate plan that recorded its run always takes the exact-save path. The destination comes from the plan's cases file (its top-level `destination`), and `tc tm add --from-file` is called without --project/--folder so it uses that same destination. A plan whose file has no destination saves nothing, says why, and stays approvable. Whatever goes wrong on that path is reported; it never falls back to generating again. A go with edits, and a plan made before plans recorded their run, behave as before. Tests now use the bindings JSON exactly as the CLI prints it and a real-shaped cases file with and without a destination, including the case that failed live. Co-Authored-By: Claude Opus 5.5 --- test-companion/README.md | 12 ++- test-companion/src/approved-cases.mjs | 44 +++++--- test-companion/src/main.mjs | 41 +++++--- test-companion/test/approved-cases.test.mjs | 108 ++++++++++++++++---- 4 files changed, 152 insertions(+), 53 deletions(-) diff --git a/test-companion/README.md b/test-companion/README.md index ece1a55..fec0f65 100644 --- a/test-companion/README.md +++ b/test-companion/README.md @@ -70,11 +70,13 @@ someone with write access to reply `go`. Read the plan before you reply. Test Companion reacts 👀 to a request, or a `go`, once it picks it up. For `generate`, a plain `go` saves exactly the test cases in the plan to Test -Management: the plan's run keeps them (the `test-companion-cases` artifact, -for 7 days) and nothing is generated again. A `@browserstack go` with changes -generates the cases again with your changes as guidance, and says so. If the -plan's cases can't be found, nothing is saved and you are asked to generate a -fresh plan. +Management, in the project and folder the plan recorded: the plan's run keeps +them (the `test-companion-cases` artifact, for 7 days) and nothing is generated +again. A `@browserstack go` with changes generates the cases again with your +changes as guidance, and says so. If the plan's cases can't be found, or the +plan recorded no Test Management destination (set `tm_project` and `tm_folder` +for generate in the pipeline file), nothing is saved and you are asked to +generate a fresh plan. Without the GitHub App, Test Companion posts as `github-actions[bot]`, the account every workflow in your repository shares, and recognises its own plans diff --git a/test-companion/src/approved-cases.mjs b/test-companion/src/approved-cases.mjs index e696641..4f3bb5a 100644 --- a/test-companion/src/approved-cases.mjs +++ b/test-companion/src/approved-cases.mjs @@ -19,23 +19,38 @@ export const CASES_SUBJECT_FILE = "subject.json" /** * How a `go` on a plan is carried out: * - "exact": save the plan's own cases with tm add, no new run; - * - "unavailable": should be exact, but the plan's cases can't be found (say so, never regenerate); - * - "run": today's behaviour, a real run of the job. `edited` is set when that is because the - * approver changed the plan, which tm add cannot apply. - * Only generate with a Test Management destination is saved exactly; other jobs, and a generate - * with nowhere to save, run as before. + * - "unavailable": should be exact, but there is nothing to fetch (say so, never regenerate); + * - "run": a real run of the job. For generate that only happens when the approver changed the + * plan (`edited`), which tm add cannot apply. + * Where the cases go is read from the plan's own cases file, not from here. + * + * Safety rule: a plain go on a generate plan never becomes "run". Generating again would save + * cases nobody reviewed, so every problem after this point is reported instead. */ -export function approvedSavePlan({ workflow, replyText, binding, runId } = {}) { +export function approvedSavePlan({ workflow, replyText, runId } = {}) { if (workflow !== "generate") return { mode: "run" } - const input = binding?.input ?? {} - const project = String(input.tm_project ?? "").trim() - const folder = String(input.tm_folder ?? "").trim() - const saveSetting = String(input["save-to-tm"] ?? "").trim().toLowerCase() - if (!project || !folder || saveSetting === "false") return { mode: "run" } if (hasApproverEdits(replyText)) return { mode: "run", edited: true } // Plans made before the run id was recorded have nothing to fetch. if (!/^\d+$/.test(String(runId ?? ""))) return { mode: "unavailable", reason: "no-run" } - return { mode: "exact", project, folder, runId: String(runId) } + return { mode: "exact", runId: String(runId) } +} + +/** + * Where the approved cases go, from the cases file the plan's run kept (its top-level + * `destination`). The CLI writes it only when a Test Management destination is configured. + */ +export function readCasesDestination(casesText) { + let data + try { + data = JSON.parse(String(casesText ?? "")) + } catch { + return { error: "unreadable" } + } + if (!data || typeof data !== "object" || !Array.isArray(data.cases)) return { error: "unreadable" } + const project = String(data.destination?.project ?? "").trim() + const folder = String(data.destination?.folder ?? "").trim() + if (!project || !folder) return { error: "no-destination" } + return { project, folder } } /** The plan run's cases artifact from a `GET .../runs//artifacts` reply, or why there is none. */ @@ -103,6 +118,7 @@ const UNAVAILABLE_WHY = { unreadable: "the plan's saved cases could not be read", "other-repo": "the saved cases are not from this repository", "other-commit": "the saved cases are for a different commit than this plan", + "no-destination": "the plan recorded no Test Management destination, so there is nowhere to save its cases", "save-failed": "Test Management refused the cases before any were saved", } @@ -113,7 +129,9 @@ export function renderCasesUnavailable(reason, detail) { return [ `I did not save anything: ${why}${extra ? ` (${extra})` : ""}.`, "", - "I have not generated a new set in its place, because that would save cases nobody reviewed. Ask me to **generate** again and approve the fresh plan.", + reason === "no-destination" + ? "I have not generated a new set in its place, because that would save cases nobody reviewed. Set `tm_project` and `tm_folder` for generate in the pipeline file, then ask me to **generate** again and approve the fresh plan." + : "I have not generated a new set in its place, because that would save cases nobody reviewed. Ask me to **generate** again and approve the fresh plan.", ].join("\n") } diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 58936bc..b7926fd 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -24,6 +24,7 @@ import { findTicket } from "./ticket.mjs" import { childEnvFor } from "./child-env.mjs" import { approvedSavePlan, + readCasesDestination, pickCasesArtifact, casesSubject, checkCasesSubject, @@ -698,13 +699,12 @@ async function main() { } // A plain go on a generate plan saves the plan's own cases; nothing is generated again. - const save = approvedSavePlan({ - workflow: plan.workflow, - replyText, - binding: pickBinding(bindings, plan.workflow), - runId: waiting?.runId, - }) - if (save.mode === "exact" || save.mode === "unavailable") saveApprovedCases(plan, save, approvalNote) + const save = approvedSavePlan({ workflow: plan.workflow, replyText, runId: waiting?.runId }) + if (save.mode === "exact" || save.mode === "unavailable") { + saveApprovedCases(plan, save, approvalNote) + // saveApprovedCases always ends the run. Never fall through to generating again. + process.exit(0) + } if (save.edited) approvalNote = `${approvalNote} ${EDITED_GO_NOTE}` } else { // A large change: the run covers what matters and lists what it skipped. @@ -1636,6 +1636,15 @@ function saveApprovedCases(plan, save, approvalNote) { finish({ ...base, outcome: OUTCOMES.COULD_NOT_RUN, answer: renderCasesUnavailable(reason, detail) }, false) if (save.mode !== "exact") return unavailable(save.reason) + // Anything unexpected is reported too: a plain go must never turn into a new generate run. + try { + saveExactly(plan, save, sha, base, finish, unavailable) + } catch (e) { + unavailable("save-failed", redactSecrets(String(e?.message ?? e), secretsNow())) + } +} + +function saveExactly(plan, save, sha, base, finish, unavailable) { const picked = pickCasesArtifact(gh(["api", `repos/${REPO}/actions/runs/${save.runId}/artifacts?per_page=100`]), save.runId) if (picked.error) return unavailable(picked.error) @@ -1645,21 +1654,23 @@ function saveApprovedCases(plan, save, approvalNote) { const subject = checkCasesSubject(readOptional(join(dir, CASES_SUBJECT_FILE)), { repo: REPO, sha }) if (!subject.ok) return unavailable(subject.why) const casesFile = join(dir, CASES_DATA_FILE) - if (!existsSync(casesFile)) return unavailable("unreadable") + // The destination comes from the approved file itself, so the cases go where the plan said. + const where = readCasesDestination(readOptional(casesFile)) + if (where.error) return unavailable(where.error) - notice(`Test Companion: saving the approved plan's cases from run ${save.runId} to ${save.folder} in ${save.project}.`) - const r = spawnSync( - TC_PATH, - ["tm", "add", "--from-file", casesFile, "--project", save.project, "--folder", save.folder, "--json"], - { encoding: "utf8", env: childEnvFor(env), timeout: 300_000 }, - ) + notice(`Test Companion: saving the approved plan's cases from run ${save.runId} to ${where.folder} in ${where.project}.`) + const r = spawnSync(TC_PATH, ["tm", "add", "--from-file", casesFile, "--json"], { + encoding: "utf8", + env: childEnvFor(env), + timeout: 300_000, + }) const report = parseTmAddReport(r.stdout) if (!report) { const why = String(r.stderr ?? "").trim().split("\n").filter(Boolean).at(-1) return unavailable("save-failed", redactSecrets(why, secretsNow())) } const { outcome, spend } = exactSaveOutcome(report, OUTCOMES) - finish({ ...base, outcome, summary: { ...base.summary, ...exactSaveSummary(report, save) }, answer: renderExactSave(report, save) }, spend) + finish({ ...base, outcome, summary: { ...base.summary, ...exactSaveSummary(report, where) }, answer: renderExactSave(report, where) }, spend) } /** The binding a comment is asking for, or the one bound to the event. */ diff --git a/test-companion/test/approved-cases.test.mjs b/test-companion/test/approved-cases.test.mjs index 03341ab..70da75e 100644 --- a/test-companion/test/approved-cases.test.mjs +++ b/test-companion/test/approved-cases.test.mjs @@ -3,6 +3,7 @@ import assert from "node:assert/strict" import { readFileSync } from "node:fs" import { approvedSavePlan, + readCasesDestination, pickCasesArtifact, casesSubject, checkCasesSubject, @@ -14,42 +15,97 @@ import { CASES_ARTIFACT, } from "../src/approved-cases.mjs" import { OUTCOMES } from "../src/outcome.mjs" +import { isApprovalOnly } from "../src/intent.mjs" import { proposalMarkerFor, renderProposal, findProposal } from "../src/report.mjs" -const BINDING = { workflow: "generate", input: { "save-to-tm": "true", tm_project: "Checkout", tm_folder: "Web" } } +// `tc workflow bindings --json`, as the CLI really prints it: no `input`, so a binding can't say +// where cases go. The destination is in the plan's cases file instead. +const REAL_BINDINGS = JSON.parse( + '{"settings":{"ask_team":"","platform":"auto","tests_repo":"","tests_path":"","targets":[],"wcag":"","fail_on_new":false,"visual":false},' + + '"bindings":[{"name":"generate-on-comment","workflow":"generate","on":"pr_comment","mode":"live","approval":"review-before-run","enabled":true}]}', +) +const GENERATE_BINDING = REAL_BINDINGS.bindings[0] + +// A generate preview's result.md.json, shaped like the CLI's. +const CASES_FILE = { + version: 1, + workflow: "generate", + preview: true, + generated: 1, + savedToTestManagement: 0, + note: "None reached Test Management — this was a preview run.", + cases: [ + { + id: "T001", + title: "First retry uses the base delay", + description: "The first attempt waits exactly the base delay.", + preconditions: "The helper is importable.", + steps: [{ step: "Call the helper with attempt 1", result: "Returns the base delay" }], + priority: "high", + type: "functional", + }, + ], +} +const withDestination = { ...CASES_FILE, destination: { project: "Checkout", folder: "Web / Cart" } } describe("approvedSavePlan: a plain go on a generate plan saves the plan's own cases", () => { - test("plain go with a destination and a recorded run: exact save", () => { - assert.deepEqual(approvedSavePlan({ workflow: "generate", replyText: "go", binding: BINDING, runId: "1234567890" }), { + test("reproduces the live failure: real binding shape, plain go, run id present → exact, not run", () => { + assert.equal("input" in GENERATE_BINDING, false) + assert.deepEqual(approvedSavePlan({ workflow: GENERATE_BINDING.workflow, replyText: "go", binding: GENERATE_BINDING, runId: "1234567890" }), { mode: "exact", - project: "Checkout", - folder: "Web", runId: "1234567890", }) }) test("go with edits regenerates, and says it is because of the edits", () => { - assert.deepEqual(approvedSavePlan({ workflow: "generate", replyText: "@browserstack go, drop T003", binding: BINDING, runId: "1" }), { + assert.deepEqual(approvedSavePlan({ workflow: "generate", replyText: "@browserstack go, drop T003", runId: "1" }), { mode: "run", edited: true, }) }) test("a plan with no recorded run is unavailable, never regenerated", () => { - assert.deepEqual(approvedSavePlan({ workflow: "generate", replyText: "go", binding: BINDING, runId: undefined }), { - mode: "unavailable", - reason: "no-run", - }) - assert.equal(approvedSavePlan({ workflow: "generate", replyText: "go", binding: BINDING, runId: "12; rm -rf /" }).mode, "unavailable") + assert.deepEqual(approvedSavePlan({ workflow: "generate", replyText: "go", runId: undefined }), { mode: "unavailable", reason: "no-run" }) + assert.equal(approvedSavePlan({ workflow: "generate", replyText: "go", runId: "12; rm -rf /" }).mode, "unavailable") }) - test("other jobs, and generate with nowhere to save, run as before", () => { - assert.deepEqual(approvedSavePlan({ workflow: "test-pr", replyText: "go", binding: BINDING, runId: "1" }), { mode: "run" }) - assert.deepEqual(approvedSavePlan({ workflow: "generate", replyText: "go", binding: { input: {} }, runId: "1" }), { mode: "run" }) - assert.deepEqual( - approvedSavePlan({ workflow: "generate", replyText: "go", binding: { input: { ...BINDING.input, "save-to-tm": "false" } }, runId: "1" }), - { mode: "run" }, - ) + test("other jobs run as before", () => { + assert.deepEqual(approvedSavePlan({ workflow: "test-pr", replyText: "go", runId: "1" }), { mode: "run" }) + assert.deepEqual(approvedSavePlan({ workflow: "fix", replyText: "go", runId: "1" }), { mode: "run" }) + }) + + test("safety rule: a plain go on a generate plan with a run id never becomes a new generate run", () => { + // The reply as the Action passes it: the @browserstack mention is already stripped + // (plan-run.mjs), so "@browserstack go" arrives as "go". + const plainGoes = ["go", "Go", "GO", "go!", "go.", " go ", "go #123456"] + const bindings = [GENERATE_BINDING, undefined, {}, { input: {} }, { input: { "save-to-tm": "false" } }] + for (const replyText of plainGoes) { + assert.equal(isApprovalOnly(replyText), true, `"${replyText}" should be a plain approval`) + for (const binding of bindings) { + assert.equal(approvedSavePlan({ workflow: "generate", replyText, binding, runId: "77" }).mode, "exact", `"${replyText}" with ${JSON.stringify(binding)}`) + } + } + }) +}) + +describe("readCasesDestination: where the cases go comes from the approved file", () => { + test("the file's destination", () => { + assert.deepEqual(readCasesDestination(JSON.stringify(withDestination)), { project: "Checkout", folder: "Web / Cart" }) + }) + test("no destination (none configured, or an older CLI): nowhere to save", () => { + assert.deepEqual(readCasesDestination(JSON.stringify(CASES_FILE)), { error: "no-destination" }) + assert.deepEqual(readCasesDestination(JSON.stringify({ ...CASES_FILE, destination: { project: "Checkout" } })), { error: "no-destination" }) + }) + test("not a cases file: unreadable", () => { + assert.deepEqual(readCasesDestination("not json"), { error: "unreadable" }) + assert.deepEqual(readCasesDestination(JSON.stringify({ destination: withDestination.destination })), { error: "unreadable" }) + assert.deepEqual(readCasesDestination(undefined), { error: "unreadable" }) + }) + test("no destination says so, saves nothing and does not regenerate", () => { + const text = renderCasesUnavailable("no-destination") + assert.match(text, /I did not save anything: the plan recorded no Test Management destination/) + assert.match(text, /I have not generated a new set/) + assert.match(text, /`tm_project` and `tm_folder`/) }) }) @@ -173,8 +229,20 @@ describe("the wiring", () => { test("a plan attaches its cases, and approving one saves them with tm add instead of running again", () => { assert.match(main, /setOutput\("cases-dir", CASES_DIR\)/) assert.match(main, /env\.GITHUB_RUN_ID,\n\s*\),\n\s*\)/) - assert.match(main, /"tm", "add", "--from-file", casesFile, "--project", save\.project, "--folder", save\.folder, "--json"/) - assert.match(main, /if \(save\.mode === "exact" \|\| save\.mode === "unavailable"\) saveApprovedCases\(plan, save, approvalNote\)/) + // No --project/--folder: tm add takes the destination from the approved file. + assert.match(main, /\["tm", "add", "--from-file", casesFile, "--json"\]/) + assert.doesNotMatch(main, /binding\?\.input|\.input\?\.tm_/) + }) + + test("safety rule in the wiring: an exact or unavailable go always ends the run", () => { + assert.match( + main, + /if \(save\.mode === "exact" \|\| save\.mode === "unavailable"\) \{\n\s*saveApprovedCases\(plan, save, approvalNote\)\n[^\n]*\n\s*process\.exit\(0\)\n\s*\}/, + ) + const fn = main.slice(main.indexOf("function saveApprovedCases("), main.indexOf("function saveExactly(")) + assert.match(fn, /try \{\n\s*saveExactly\(/) + assert.match(fn, /catch \(e\) \{\n\s*unavailable\("save-failed"/) + assert.match(fn, /process\.exit\(0\)/) }) test("action.yml uploads the cases artifact, and a plan's report and result too", () => { From ac2478aea95aeca1a0ca4a49d6f6affa9ef11305 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 16 Nov 2026 12:49:30 +0530 Subject: [PATCH 16/31] test-companion: commit exactly the files the Action chose, and ignore replace refs and grafts The agent can write the checkout's index. A file it staged itself, without committing (a .github/actions file, CODEOWNERS, a symlink, .env, or a deletion), was filtered out of the list but still committed, because `git commit` takes the whole index. The commit is now built in a scratch index started from the commit the run began on, holding only the files that passed the filter, and the push sends that commit. The agent can also write .git/refs/replace and .git/info/grafts, which make git read a commit as a different tree or with different parents. Our git commands now ignore both (GIT_NO_REPLACE_OBJECTS, GIT_GRAFT_FILE=/dev/null). Co-Authored-By: Claude Opus 5.5 --- test-companion/src/main.mjs | 28 +++--- test-companion/src/push-work.mjs | 32 ++++++ test-companion/test/push-work.test.mjs | 132 ++++++++++++++++++++++++- 3 files changed, 176 insertions(+), 16 deletions(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index b7926fd..117ad49 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -93,6 +93,7 @@ import { savingFor, isRequestedByPerson, leaveOut, + commitPaths, leaveOutMessage, listFiles, renderPushNote, @@ -1084,10 +1085,6 @@ function saveWork(saving, plan) { }) if (push.length === 0) return undefined - // Named paths only, never `git add -A`. - const added = git(["add", "--", ...push]) - if (!added.ok) return renderPushNote({ branch, files: push, skipped: skip, error: added.err || "git add failed" }) - const subject = `test: ${plan.workflow ?? "Test Companion"} on ${String(plan.context.sha ?? "").slice(0, 7)}` const body = [ subject, @@ -1095,14 +1092,17 @@ function saveWork(saving, plan) { `Written by Test Companion for ${plan.context.pr ? `#${plan.context.pr}` : plan.context.sha}.`, `Run: ${RUN_URL}`, ].join("\n") - const committed = git(["-c", `user.name=${identity.name}`, "-c", `user.email=${identity.email}`, "commit", "-m", body]) - if (!committed.ok) { - return renderPushNote({ - branch, - files: push, - skipped: skip, - error: committed.err || committed.out || "nothing to commit", - }) + // Exactly the files above, on the commit the run started on. Never the real index: the + // agent can stage files there itself, and `git commit` would take them all. + const indexFile = join(REPORT_DIR, `tc-save-index-${process.pid}`) + let made + try { + made = commitPaths({ git, base: headBeforeRun, paths: push, identity, message: body, indexFile }) + } finally { + rmSync(indexFile, { force: true }) + } + if (!made.sha) { + return renderPushNote({ branch, files: push, skipped: skip, error: made.error || "nothing to commit" }) } // Push to a URL built from GITHUB_REPOSITORY, not `origin`: an insteadOf rewrite in @@ -1110,7 +1110,7 @@ function saveWork(saving, plan) { const target = pushUrlFor(REPO, env.GITHUB_SERVER_URL) if (!target) return renderPushNote({ branch, files: push, skipped: skip, error: `cannot push: bad repository name "${REPO}"` }) - const pushed = git(["push", target, `HEAD:refs/heads/${branch}`], pushCredentialEnv(env.GH_TOKEN, target)) + const pushed = git(["push", target, `${made.sha}:refs/heads/${branch}`], pushCredentialEnv(env.GH_TOKEN, target)) if (!pushed.ok) { // The workflow token is read-only by default. const hint = /403|permission|denied|protected/i.test(pushed.err) @@ -1118,7 +1118,7 @@ function saveWork(saving, plan) { : pushed.err || "git push failed" return renderPushNote({ branch, files: push, skipped: skip, error: hint }) } - return renderPushNote({ branch, sha: git(["rev-parse", "HEAD"]).out, files: push, skipped: skip }) + return renderPushNote({ branch, sha: made.sha, files: push, skipped: skip }) } /** Set up the locked-down user. Without it, stop unless allow-unisolated is set. */ diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index 8362204..4bf4d00 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -290,6 +290,10 @@ export function gitEnvFrom(env = {}) { out.GIT_ATTR_NOSYSTEM = "1" // File names come from the agent: `:!x` must name a file, not mean "everything but x". out.GIT_LITERAL_PATHSPECS = "1" + // The agent can write .git/refs/replace and .git/info/grafts. Either makes git read a + // commit as a different tree or with different parents, so ignore both. + out.GIT_NO_REPLACE_OBJECTS = "1" + out.GIT_GRAFT_FILE = "/dev/null" return out } @@ -395,6 +399,10 @@ export const GIT_SAFE_ARGS = Object.freeze([ "gpg.program=false", "-c", "core.attributesFile=/dev/null", + // gitEnvFrom points GIT_GRAFT_FILE at /dev/null; without this git prints a hint on stderr, + // which callers read as an error. + "-c", + "advice.graftFileDeprecated=false", ]) /** @@ -462,6 +470,30 @@ export function leaveOut({ git, base, tip, paths, identity, message, indexFile } return commit.ok && commit.out ? { sha: commit.out } : { error: "could not commit without the by-products" } } +/** + * One commit on `base` holding exactly `paths` as they are in the checkout, and nothing else. + * Built in a scratch index started from `base`: the agent can write the real index, so a + * file it staged itself (a .github file, a symlink, a deletion) would otherwise ride along. + * + * `git(args, extraEnv, input)` runs git; it must apply literal pathspecs. + * Returns { sha }, { empty } when the paths match `base`, or { error }. + */ +export function commitPaths({ git, base, paths, identity, message, indexFile }) { + const scratch = { GIT_INDEX_FILE: indexFile, GIT_LITERAL_PATHSPECS: "1" } + if (!git(["read-tree", base], scratch).ok) return { error: "could not read the commit the run started on" } + // A path that is gone from the checkout is recorded as deleted. + const added = git(["add", "--", ...paths], scratch) + if (!added.ok) return { error: added.err || "git add failed" } + const tree = git(["write-tree"], scratch) + if (!tree.ok || !tree.out) return { error: "could not write the run's files" } + if (tree.out === git(["rev-parse", `${base}^{tree}`]).out) return { empty: true } + const commit = git( + ["-c", `user.name=${identity.name}`, "-c", `user.email=${identity.email}`, "commit-tree", tree.out, "-p", base, "-m", message], + scratch, + ) + return commit.ok && commit.out ? { sha: commit.out } : { error: commit.err || "could not commit" } +} + // Agent-chosen text on one line, so it can't add a trailer such as Co-authored-by. const oneLine = (s) => String(s ?? "").replace(/[\x00-\x1f\x7f]+/g, " ").trim() diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index 296b657..fddbbb3 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -1,7 +1,7 @@ import { test, describe } from "node:test" import assert from "node:assert/strict" import { execFileSync, spawnSync } from "node:child_process" -import { mkdtempSync, mkdirSync, rmSync, readdirSync, readFileSync, writeFileSync } from "node:fs" +import { mkdtempSync, mkdirSync, rmSync, readdirSync, readFileSync, writeFileSync, symlinkSync, unlinkSync, existsSync } from "node:fs" import { tmpdir } from "node:os" import { dirname, join } from "node:path" import { @@ -28,6 +28,7 @@ import { commitIdentity, savingFor, leaveOut, + commitPaths, checkHandoffDescends, REWROTE_HISTORY, leaveOutMessage, @@ -769,7 +770,7 @@ describe("splitHandoffPaths — a handed-back branch", () => { function realGit(cwd) { const env = gitEnvFrom(process.env) return (args, extraEnv = {}, input = undefined) => { - const r = spawnSync("git", args, { cwd, encoding: "utf8", input, env: { ...env, ...extraEnv } }) + const r = spawnSync("git", [...GIT_SAFE_ARGS, ...args], { cwd, encoding: "utf8", input, env: { ...env, ...extraEnv } }) return { ok: r.status === 0, out: (r.stdout ?? "").trim(), err: (r.stderr ?? "").trim() } } } @@ -1101,3 +1102,130 @@ describe("a hand-off branch must build on the reviewed commit", () => { assert.deepEqual(checkHandoffDescends({ git: broken, reviewed: "zzz", tip: "t" }), { error: "fatal: Not a valid commit name zzz" }) }) }) + +describe("commitPaths — only the files the Action chose are committed", () => { + const identity = { name: "Test Companion", email: BOT_EMAIL } + + // The agent can write the real index: it stages files the push filter refuses, then stops. + function agentStagedRepo() { + const dir = mkdtempSync(join(tmpdir(), "tc-commitpaths-")) + const git = realGit(dir) + const id = ["-c", "user.name=t", "-c", "user.email=t@example.com"] + const must = (args) => { + const r = git(args) + assert.ok(r.ok, `${args.join(" ")}: ${r.err}`) + return r.out + } + must(["init", "-q"]) + writeFileSync(join(dir, "a.ts"), "old\n") + writeFileSync(join(dir, "keep.ts"), "keep\n") + must(["add", "a.ts", "keep.ts"]) + must([...id, "commit", "-qm", "base"]) + const base = must(["rev-parse", "HEAD"]) + // The run's real work: one changed file and one deleted file. + writeFileSync(join(dir, "a.ts"), "new\n") + writeFileSync(join(dir, "gone.ts"), "x\n") + // What a prompt-injected agent stages on its own, without committing. + mkdirSync(join(dir, ".github/actions/x"), { recursive: true }) + writeFileSync(join(dir, ".github/actions/x/action.yml"), "runs: evil\n") + writeFileSync(join(dir, "CODEOWNERS"), "* @mallory\n") + writeFileSync(join(dir, ".env"), "KEY=secret\n") + symlinkSync("/etc/passwd", join(dir, "link")) + must(["add", "--", ".github/actions/x/action.yml", "CODEOWNERS", ".env", "link"]) + must(["rm", "-q", "--cached", "keep.ts"]) + return { dir, git, base } + } + + test("files the agent staged itself are not in the commit", () => { + const { dir, git, base } = agentStagedRepo() + const out = commitPaths({ git, base, paths: ["a.ts"], identity, message: "m", indexFile: join(dir, "scratch") }) + assert.ok(out.sha, JSON.stringify(out)) + assert.equal(git(["diff", "--name-only", base, out.sha]).out, "a.ts") + assert.equal(git(["show", `${out.sha}:a.ts`]).out, "new") + for (const p of [".github/actions/x/action.yml", "CODEOWNERS", ".env", "link"]) { + assert.equal(git(["cat-file", "-e", `${out.sha}:${p}`]).ok, false, `${p} was committed`) + } + // The agent's staged deletion of keep.ts did not ride along either. + assert.equal(git(["show", `${out.sha}:keep.ts`]).out, "keep") + assert.equal(git(["rev-list", "--parents", "-n", "1", out.sha]).out, `${out.sha} ${base}`) + rmSync(dir, { recursive: true, force: true }) + }) + + test("a file the run deleted is committed as deleted", () => { + const { dir, git, base } = agentStagedRepo() + unlinkSync(join(dir, "keep.ts")) + const out = commitPaths({ git, base, paths: ["a.ts", "keep.ts"], identity, message: "m", indexFile: join(dir, "scratch") }) + assert.ok(out.sha, JSON.stringify(out)) + assert.equal(git(["diff", "--name-only", base, out.sha]).out.split("\n").sort().join(","), "a.ts,keep.ts") + assert.equal(git(["cat-file", "-e", `${out.sha}:keep.ts`]).ok, false) + rmSync(dir, { recursive: true, force: true }) + }) + + test("nothing changed in the chosen files: no commit", () => { + const { dir, git, base } = agentStagedRepo() + writeFileSync(join(dir, "a.ts"), "old\n") + assert.deepEqual(commitPaths({ git, base, paths: ["a.ts"], identity, message: "m", indexFile: join(dir, "scratch") }), { empty: true }) + rmSync(dir, { recursive: true, force: true }) + }) + + test("the real index is left alone", () => { + const { dir, git, base } = agentStagedRepo() + const before = readFileSync(join(dir, ".git/index")) + commitPaths({ git, base, paths: ["a.ts"], identity, message: "m", indexFile: join(dir, "scratch") }) + assert.ok(readFileSync(join(dir, ".git/index")).equals(before)) + assert.equal(existsSync(join(dir, "scratch")), true, "the caller removes the scratch index") + rmSync(dir, { recursive: true, force: true }) + }) +}) + +describe("git ignores replace refs and grafts the agent could plant", () => { + test("the environment turns both off", () => { + const got = gitEnvFrom({}) + assert.equal(got.GIT_NO_REPLACE_OBJECTS, "1") + assert.equal(got.GIT_GRAFT_FILE, "/dev/null") + }) + + test("a replace ref can't swap the base's files, and a graft can't fake ancestry", () => { + const { dir, git, base, tip } = repoWithBranch({ "a.ts": "new\n" }) + const plain = (args) => spawnSync("git", args, { cwd: dir, encoding: "utf8" }) + // A commit whose tree carries a .github file, swapped in for base by a replace ref. + mkdirSync(join(dir, ".github/actions/x"), { recursive: true }) + writeFileSync(join(dir, ".github/actions/x/action.yml"), "runs: evil\n") + const idx = { ...process.env, GIT_INDEX_FILE: join(dir, "evil-index") } + spawnSync("git", ["read-tree", base], { cwd: dir, env: idx }) + spawnSync("git", ["add", ".github/actions/x/action.yml"], { cwd: dir, env: idx }) + const evilTree = spawnSync("git", ["write-tree"], { cwd: dir, env: idx, encoding: "utf8" }).stdout.trim() + const evil = plain(["-c", "user.name=t", "-c", "user.email=t@e", "commit-tree", evilTree, "-m", "evil"]).stdout.trim() + assert.equal(plain(["replace", base, evil]).status, 0) + assert.match(plain(["ls-tree", "-r", "--name-only", base]).stdout, /\.github/, "plain git does follow the replace ref") + assert.doesNotMatch(git(["ls-tree", "-r", "--name-only", base]).out, /\.github/) + plain(["replace", "-d", base]) + // A graft that claims the reviewed commit is an ancestor of an unrelated one. + const lone = plain(["-c", "user.name=t", "-c", "user.email=t@e", "commit-tree", evilTree, "-m", "lone"]).stdout.trim() + mkdirSync(join(dir, ".git/info"), { recursive: true }) + writeFileSync(join(dir, ".git/info/grafts"), `${lone} ${tip}\n`) + assert.equal(plain(["merge-base", "--is-ancestor", tip, lone]).status, 0, "plain git does follow the graft") + assert.equal(checkHandoffDescends({ git, reviewed: tip, tip: lone }).refused, true) + rmSync(dir, { recursive: true, force: true }) + }) + + test("a planted hook does not run, and a URL rewrite stops the commit", () => { + const { dir, git } = repoWithBranch({ "a.ts": "new\n" }) + const marker = join(dir, "hook-ran") + mkdirSync(join(dir, ".git/hooks"), { recursive: true }) + writeFileSync(join(dir, ".git/hooks/pre-commit"), `#!/bin/sh\ntouch "${marker}"\n`, { mode: 0o755 }) + writeFileSync(join(dir, ".git/hooks/post-commit"), `#!/bin/sh\ntouch "${marker}"\n`, { mode: 0o755 }) + const safe = git + writeFileSync(join(dir, "b.ts"), "b\n") + assert.ok(safe(["add", "b.ts"]).ok) + assert.ok(safe(["-c", "user.name=t", "-c", "user.email=t@e", "commit", "-qm", "c"]).ok) + assert.equal(existsSync(marker), false, "a hook ran") + assert.equal(plain(dir, ["config", "url.https://evil.test/.insteadOf", "https://github.com/"]).status, 0) + assert.deepEqual(unexpectedConfigKeys(git(CONFIG_READ_ARGS).out), ["url.https://evil.test/.insteadof"]) + rmSync(dir, { recursive: true, force: true }) + }) +}) + +function plain(cwd, args) { + return spawnSync("git", args, { cwd, encoding: "utf8" }) +} From 425eafe66b879a50d499fa63fc1e6bde8b079732 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 16 Nov 2026 12:51:26 +0530 Subject: [PATCH 17/31] test-companion: approving a fix plan reads the failed run that plan looked at On a `go`, the failed run was chosen again from the reply, which names no run, so it fell back to the newest red run. With Lint (newer) and E2E both red, `@browserstack fix e2e` planned against E2E and `go` then fixed Lint; `fix run ` lost the named run the same way. The plan now records the run it read (`build=` in its marker). Approving reads that run and only that run: if it can no longer be used (no longer red, not on this pull request's commits) the reply says so, and a plan that didn't record a run is refused. Neither ever falls back to another run. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/main.mjs | 35 ++++++++++++-- test-companion/src/report.mjs | 25 ++++++---- test-companion/src/runs.mjs | 14 ++++++ test-companion/test/run-selection.test.mjs | 55 ++++++++++++++++++++++ 4 files changed, 118 insertions(+), 11 deletions(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 117ad49..ecd4c06 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -9,7 +9,7 @@ import { chooseRunsToCancel, runIdsFromChecks, ciOnCommit, - parseRunReference, + runRequestFor, matchFailedByName, checkNamedRun, renderCiStillRunning, @@ -234,6 +234,8 @@ let approvedPlan /** Which failed run a build-reading job looked at, and what it left out. */ let buildNote = "" +/** That run's id, recorded in a plan so approving it reads the same run. */ +let buildRunId = "" // No --depth: it would make the full clone shallow. The URL carries its own credential // because the starter sets persist-credentials: false. @@ -724,7 +726,24 @@ async function main() { // rca, fix and verdict need a failed build: one the comment names, else the newest red run on this commit. if (NEEDS_A_BUILD.has(plan.workflow) && !plan.context.build) { const short = String(plan.context.sha).slice(0, 7) - const named = parseRunReference(requestText, { repo: REPO, serverUrl: env.GITHUB_SERVER_URL }) + // Approving reads the run the plan looked at. "go" names no run, so choosing again could + // pick a different one (a newer red run, or the newest instead of the one named). + const approving = Boolean(gate.approvedProposal) + const asked = runRequestFor({ + approving, + plannedRunId: waiting?.buildRunId, + requestText, + repo: REPO, + serverUrl: env.GITHUB_SERVER_URL, + }) + if (asked.refuse) { + replyOnPr( + plan.context.pr, + "That plan doesn't record which failed run it looked at (an older version of Test Companion made it), so I have not run it. Ask me again and I will put up a fresh one.", + ) + finishSkipped("proposal-without-run", plan.workflow) + } + const named = asked.named if (named?.error) { replyOnPr(plan.context.pr, named.error) finishSkipped("run-not-usable", "link") @@ -746,9 +765,16 @@ async function main() { selfWorkflowName: env.TC_ACTION_SELF_WORKFLOW, }) if (!check.ok) { - replyOnPr(plan.context.pr, `I can't use run \`${named.id}\`: ${check.why}`) + // Never a different run in its place: that is not what was approved. + replyOnPr( + plan.context.pr, + approving + ? `The run this plan looked at, \`${named.id}\`, can't be used any more: ${check.why} Ask me again and I will put up a fresh plan.` + : `I can't use run \`${named.id}\`: ${check.why}`, + ) finishSkipped("run-not-usable", named.id) } + buildRunId = String(named.id) plan.context.build = check.url buildNote = renderBuildNote(check, { ask: ASK_WORD[plan.workflow] }) } else { @@ -773,6 +799,7 @@ async function main() { ) finishSkipped(ci.running.length ? "ci-still-running" : "nothing-failed", plan.workflow) } + buildRunId = String(chosen.id) plan.context.build = chosen.url buildNote = renderBuildNote(chosen, { otherFailed: ci.failed.filter((r) => r !== chosen), @@ -996,6 +1023,8 @@ async function main() { plan.workflow ?? summary.workflow, // And the run, so approving can fetch the cases it kept. env.GITHUB_RUN_ID, + // And the failed run it read, so approving reads that run and no other. + { build: buildRunId }, ), ) postCheck(outcome, view, sha, [], "in_progress") diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index 04608a1..04d328c 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -282,22 +282,29 @@ export const PROPOSAL_WINDOW_MS = 24 * 60 * 60 * 1000 // Keyed to the commit, so a new push makes an old proposal unapprovable. // `run` is the run that made the plan, so approving it can fetch the cases that run kept. -export function proposalMarkerFor(sha, requester, workflow, runId) { +// `build` is the failed run the plan looked at, so approving it fixes that run and no other. +// `cases` is the sha256 of the cases file the plan showed, so approving it saves exactly that. +export function proposalMarkerFor(sha, requester, workflow, runId, { build, cases } = {}) { const by = String(requester ?? "").trim() const job = String(workflow ?? "").trim() const run = /^\d+$/.test(String(runId ?? "")) ? String(runId) : "" - return `` + const failed = /^\d+$/.test(String(build ?? "")) ? String(build) : "" + const digest = /^[0-9a-f]{64}$/.test(String(cases ?? "")) ? String(cases) : "" + return `` } -// Groups: 1 = sha, 2 = requester, 3 = workflow, 4 = run id. The sha is captured and compared, -// never built into the regex. -const MARKER = // +// Groups: 1 = sha, 2 = requester, 3 = workflow, 4 = run id, 5 = failed run id, 6 = cases sha256. +// The sha is captured and compared, never built into the regex. Older markers lack 4 to 6. +const MARKER = + // // The last marker, since ours ends the comment and anything earlier could be planted. const readMarker = (body) => { const all = [...String(body ?? "").matchAll(new RegExp(MARKER.source, "g"))] const found = all.at(-1) - return found ? { sha: found[1], requester: found[2], workflow: found[3], runId: found[4] } : undefined + return found + ? { sha: found[1], requester: found[2], workflow: found[3], runId: found[4], buildRunId: found[5], casesSha256: found[6] } + : undefined } // The commit our in-progress check went on (the PR head), which on a comment run is not GITHUB_SHA. @@ -312,13 +319,13 @@ export function isApprovableJob(workflow, enabledWorkflows) { return job !== "" && !job.startsWith("-") && (enabledWorkflows ?? []).includes(job) } -export function renderProposal(body, sha, requester, workflow, runId) { +export function renderProposal(body, sha, requester, workflow, runId, pinned = {}) { return [ body, "", "Reply **go** to run this for real. Nothing is written until you do. To change the list first, reply **@browserstack go** with your edits. Leave it and it stops being approvable after a day.", "", - proposalMarkerFor(sha, requester, workflow, runId), + proposalMarkerFor(sha, requester, workflow, runId, pinned), ].join("\n") } @@ -419,6 +426,8 @@ export function findProposal(comments, sha, now = Date.now(), us, opts = {}) { requester: mark?.requester, workflow: mark?.workflow, runId: mark?.runId, + buildRunId: mark?.buildRunId, + casesSha256: mark?.casesSha256, body: newest.body ?? "", } } diff --git a/test-companion/src/runs.mjs b/test-companion/src/runs.mjs index 36eb309..4512f1f 100644 --- a/test-companion/src/runs.mjs +++ b/test-companion/src/runs.mjs @@ -79,6 +79,20 @@ export function parseRunReference(text, { repo, serverUrl } = {}) { return bare ? { id: bare[1] } : undefined } +/** + * Which run a build-reading request names. Approving a plan reads the run the plan looked at + * (`plannedRunId`, from its marker), never one chosen again: "go" names no run, so choosing + * again could pick a newer red run, or the newest instead of the one named. + * Returns { named } as parseRunReference would (undefined = choose from the commit's red runs), + * or { refuse: "no-planned-run" } when an approved plan didn't record its run. + */ +export function runRequestFor({ approving, plannedRunId, requestText, repo, serverUrl } = {}) { + if (approving) { + return /^\d+$/.test(String(plannedRunId ?? "")) ? { named: { id: String(plannedRunId) } } : { refuse: "no-planned-run" } + } + return { named: parseRunReference(requestText, { repo, serverUrl }) } +} + // Words that say what to do, not which workflow to look at. const NOT_A_NAME = new Set( "browserstack explain fix find bad pr prs pull request the a an why what did does do failed failing fail failure build builds run runs this that these on in for please of to and go verdict rca me it is test tests ci check checks job jobs with my our".split( diff --git a/test-companion/test/run-selection.test.mjs b/test-companion/test/run-selection.test.mjs index 233a837..6c2dc90 100644 --- a/test-companion/test/run-selection.test.mjs +++ b/test-companion/test/run-selection.test.mjs @@ -7,7 +7,12 @@ import { checkNamedRun, renderCiStillRunning, renderBuildNote, + runRequestFor, } from "../src/runs.mjs" +import { findProposal, renderProposal, proposalMarkerFor } from "../src/report.mjs" + +const SHA = "a".repeat(40) +const US = "browserstack[bot]" const run = (id, name, status, conclusion = null, extra = {}) => ({ id, @@ -135,3 +140,53 @@ describe("the replies", () => { assert.equal(renderBuildNote(undefined), "") }) }) + +describe("approving a plan reads the run that plan looked at", () => { + const opts = { repo: "org/repo", serverUrl: "https://github.com" } + // Newest first, as ciOnCommit returns them: Lint failed after E2E. + const lint = { id: 300, name: "Lint", url: "https://github.com/org/repo/actions/runs/300" } + const e2e = { id: 200, name: "E2E Tests (linux)", url: "https://github.com/org/repo/actions/runs/200" } + const failed = [lint, e2e] + const plannedFrom = (requestText) => { + const asked = runRequestFor({ approving: false, requestText, ...opts }) + return asked.named?.id ?? (matchFailedByName(requestText, failed) ?? failed[0]).id + } + const approvedRun = (body) => { + const waiting = findProposal([{ id: 9, user: { login: US, type: "Bot" }, created_at: new Date().toISOString(), body }], SHA, Date.now(), US) + return runRequestFor({ approving: true, plannedRunId: waiting?.buildRunId, requestText: "go", ...opts }) + } + + test("`fix e2e` then `go` fixes E2E, though Lint is newer and also red", () => { + const build = plannedFrom("@browserstack fix e2e") + assert.equal(String(build), "200") + const asked = approvedRun(renderProposal("plan", SHA, "octocat", "fix", "111", { build })) + assert.deepEqual(asked, { named: { id: "200" } }) + }) + + test("`fix run ` then `go` fixes that run", () => { + const build = plannedFrom("@browserstack fix run 200000") + assert.equal(build, "200000") + assert.deepEqual(approvedRun(renderProposal("plan", SHA, "octocat", "fix", "111", { build })), { named: { id: "200000" } }) + }) + + test("`go` itself never names a run, so it can't choose one", () => { + assert.deepEqual(runRequestFor({ approving: false, requestText: "go", ...opts }), { named: undefined }) + }) + + test("a plan that didn't record its run is refused, never re-chosen", () => { + assert.deepEqual(approvedRun(renderProposal("plan", SHA, "octocat", "fix", "111")), { refuse: "no-planned-run" }) + }) + + test("the recorded run is still checked: one that is no longer red can't be used", () => { + const run = { id: 200, html_url: e2e.url, head_sha: SHA, status: "completed", conclusion: "success", name: e2e.name, head_repository: { full_name: "org/repo" } } + assert.match(checkNamedRun(run, { prShas: [SHA] }).why, /did not fail/) + }) + + test("the marker keeps the run and still reads markers without it", () => { + assert.match(proposalMarkerFor(SHA, "o", "fix", "111", { build: "200" }), / run=111 build=200 -->$/) + assert.equal(proposalMarkerFor(SHA, "o", "fix", "111", { build: "2 00" }).includes("build="), false) + const old = findProposal([{ id: 9, user: { login: US, type: "Bot" }, created_at: new Date().toISOString(), body: `p\n\n` }], SHA, Date.now(), US) + assert.equal(old.runId, "111") + assert.equal(old.buildRunId, undefined) + }) +}) From e76f3f9ac1b2e57543fdeafed75f37a3baadcc76 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 16 Nov 2026 12:53:59 +0530 Subject: [PATCH 18/31] test-companion: approving a generate plan saves exactly the cases the plan showed The plan comment showed the CLI's Markdown result, while approving saved the JSON cases file. Both are written in the agent's folder, and nothing checked they described the same cases, so an agent steered by the pull request's content could make `go` save other cases, or send them elsewhere, without the reviewer seeing it. A generate plan is now shown from the same cases file approving saves, including where it will be saved ("Will be saved to: ..."), and that file's sha256 goes in our own plan comment's marker. On `go`, the downloaded file must have that sha256 before anything is saved; otherwise nothing is saved and nothing is generated in its place. Agent-written text in the plan is shown as plain text, so it can't mention anyone or add links. This also fixes the source-text test the previous commit broke. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/approved-cases.mjs | 74 +++++++++++++- test-companion/src/main.mjs | 45 +++++++-- test-companion/test/approved-cases.test.mjs | 103 ++++++++++++++++++-- 3 files changed, 205 insertions(+), 17 deletions(-) diff --git a/test-companion/src/approved-cases.mjs b/test-companion/src/approved-cases.mjs index 4f3bb5a..4b4f9bd 100644 --- a/test-companion/src/approved-cases.mjs +++ b/test-companion/src/approved-cases.mjs @@ -9,6 +9,7 @@ * * Everything here is pure; main.mjs does the API calls and runs tc. */ +import { createHash } from "node:crypto" import { hasApproverEdits } from "./report.mjs" /** The artifact a generate preview leaves: the cases file plus subject.json naming its repo and commit. */ @@ -27,12 +28,79 @@ export const CASES_SUBJECT_FILE = "subject.json" * Safety rule: a plain go on a generate plan never becomes "run". Generating again would save * cases nobody reviewed, so every problem after this point is reported instead. */ -export function approvedSavePlan({ workflow, replyText, runId } = {}) { +export function approvedSavePlan({ workflow, replyText, runId, casesSha256 } = {}) { if (workflow !== "generate") return { mode: "run" } if (hasApproverEdits(replyText)) return { mode: "run", edited: true } // Plans made before the run id was recorded have nothing to fetch. if (!/^\d+$/.test(String(runId ?? ""))) return { mode: "unavailable", reason: "no-run" } - return { mode: "exact", runId: String(runId) } + // Without the fingerprint of the cases the plan showed, nothing ties the saved file to them. + if (!/^[0-9a-f]{64}$/.test(String(casesSha256 ?? ""))) return { mode: "unavailable", reason: "no-digest" } + return { mode: "exact", runId: String(runId), casesSha256: String(casesSha256) } +} + +/** The sha256 of a cases file, as recorded in the plan and checked before saving. */ +export function casesDigest(bytes) { + return createHash("sha256").update(bytes ?? "").digest("hex") +} + +// Agent-written text shown in the plan: one line, no Markdown, no @mentions or autolinks. +const plain = (s) => + String(s ?? "") + .replace(/\s+/g, " ") + .trim() + .replace(/[\\`*_{}[\]()<>#+!|~]/g, "\\$&") + .replace(/@/g, "@\u200b") + .replace(/:\/\//g, ":\u200b//") + +// A GitHub comment holds 65,536 characters; past this, the plan lists titles only. +const PLAN_DETAIL_LIMIT = 45_000 + +/** + * The plan the reviewer reads, from the same cases file that approving saves, so what is + * shown and what is saved can't differ. Undefined when the file can't be read as cases. + * Returns { body, digest }; the digest goes in the plan's marker. + */ +export function renderPlannedCases(bytes) { + let data + try { + data = JSON.parse(Buffer.from(bytes ?? "").toString("utf8")) + } catch { + return undefined + } + if (!data || typeof data !== "object" || !Array.isArray(data.cases)) return undefined + const cases = data.cases.filter((c) => c && typeof c === "object") + const where = readCasesDestination(JSON.stringify(data)) + const ids = (a, b) => [a, b].filter(Boolean).map(plain).join(", ") + const head = where.error + ? "**Will be saved to:** nowhere. This plan records no Test Management destination, so approving it can't save these cases." + : `**Will be saved to:** ${plain(where.folder)} in ${plain(where.project)}${ + data.destination?.folderId || data.destination?.projectId + ? ` (${ids(data.destination?.projectId, data.destination?.folderId)})` + : "" + }, once approved.` + const count = `**${cases.length} test case${cases.length === 1 ? "" : "s"}:**` + const title = (c, i) => `${plain(c.id) || `#${i + 1}`} — ${plain(c.title) || "(untitled)"}` + const detailed = cases.map((c, i) => { + const lines = [`#### ${title(c, i)}`] + if (String(c.priority ?? "").trim()) lines.push(`Priority: ${plain(c.priority)}`) + if (String(c.preconditions ?? "").trim()) lines.push(`Before you start: ${plain(c.preconditions)}`) + const steps = Array.isArray(c.steps) ? c.steps : [] + steps.forEach((st, n) => { + lines.push(`${n + 1}. ${plain(st?.step ?? st?.description ?? st?.action)}`) + const expected = st?.result ?? st?.expected ?? st?.expectedResult + if (String(expected ?? "").trim()) lines.push(` - Expected: ${plain(expected)}`) + }) + return lines.join("\n") + }) + let list = detailed.join("\n\n") + if (list.length > PLAN_DETAIL_LIMIT) { + list = [ + "_Too long to show every step here; the titles are below and the full cases are in this run's result file._", + "", + ...cases.map((c, i) => `- ${title(c, i)}`), + ].join("\n") + } + return { body: [head, "", count, "", list].join("\n"), digest: casesDigest(bytes) } } /** @@ -118,6 +186,8 @@ const UNAVAILABLE_WHY = { unreadable: "the plan's saved cases could not be read", "other-repo": "the saved cases are not from this repository", "other-commit": "the saved cases are for a different commit than this plan", + "no-digest": "this plan didn't record which cases it showed (an older version of Test Companion made it)", + changed: "the saved cases are not the ones this plan showed", "no-destination": "the plan recorded no Test Management destination, so there is nowhere to save its cases", "save-failed": "Test Management refused the cases before any were saved", } diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index ecd4c06..61aaa13 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -25,6 +25,8 @@ import { childEnvFor } from "./child-env.mjs" import { approvedSavePlan, readCasesDestination, + renderPlannedCases, + casesDigest, pickCasesArtifact, casesSubject, checkCasesSubject, @@ -702,7 +704,12 @@ async function main() { } // A plain go on a generate plan saves the plan's own cases; nothing is generated again. - const save = approvedSavePlan({ workflow: plan.workflow, replyText, runId: waiting?.runId }) + const save = approvedSavePlan({ + workflow: plan.workflow, + replyText, + runId: waiting?.runId, + casesSha256: waiting?.casesSha256, + }) if (save.mode === "exact" || save.mode === "unavailable") { saveApprovedCases(plan, save, approvalNote) // saveApprovedCases always ends the run. Never fall through to generating again. @@ -1005,12 +1012,21 @@ async function main() { const sizeNote = size.large ? `> **This is a large change** — ${size.files} files and ${size.lines} changed lines. Trim the list below before replying go if you only want part of it covered.\n\n` : "" - // The plan itself, from the result file. + // The plan itself. A generate plan is shown from the cases file approving it saves, and that + // file's fingerprint goes in the marker, so what the reviewer reads is exactly what is saved. let planBody = "" - try { - planBody = readFileSync(RESULT_FILE, "utf8") - } catch { - // No plan; renderProposalBody says so. + let casesSha256 = "" + const planned = + (plan.workflow ?? summary.workflow) === "generate" ? renderPlannedCases(readBytes(join(CASES_DIR, CASES_DATA_FILE))) : undefined + if (planned) { + planBody = planned.body + casesSha256 = planned.digest + } else { + try { + planBody = readFileSync(RESULT_FILE, "utf8") + } catch { + // No plan; renderProposalBody says so. + } } replyOnPr( plan.context.pr, @@ -1023,8 +1039,8 @@ async function main() { plan.workflow ?? summary.workflow, // And the run, so approving can fetch the cases it kept. env.GITHUB_RUN_ID, - // And the failed run it read, so approving reads that run and no other. - { build: buildRunId }, + // And the failed run it read, and the cases it showed, so approving uses exactly those. + { build: buildRunId, cases: casesSha256 }, ), ) postCheck(outcome, view, sha, [], "in_progress") @@ -1623,6 +1639,14 @@ function readOptional(path) { } } +function readBytes(path) { + try { + return readFileSync(path) + } catch { + return undefined + } +} + /** Keep the run's cases file, with the repo and commit it was made for, for the cases artifact. */ function keepCases(text, plan) { if (text === undefined) return @@ -1683,8 +1707,11 @@ function saveExactly(plan, save, sha, base, finish, unavailable) { const subject = checkCasesSubject(readOptional(join(dir, CASES_SUBJECT_FILE)), { repo: REPO, sha }) if (!subject.ok) return unavailable(subject.why) const casesFile = join(dir, CASES_DATA_FILE) + // Only the exact file the plan was shown from: its fingerprint is in our own plan comment. + const bytes = readBytes(casesFile) + if (bytes === undefined || casesDigest(bytes) !== save.casesSha256) return unavailable("changed") // The destination comes from the approved file itself, so the cases go where the plan said. - const where = readCasesDestination(readOptional(casesFile)) + const where = readCasesDestination(bytes.toString("utf8")) if (where.error) return unavailable(where.error) notice(`Test Companion: saving the approved plan's cases from run ${save.runId} to ${where.folder} in ${where.project}.`) diff --git a/test-companion/test/approved-cases.test.mjs b/test-companion/test/approved-cases.test.mjs index 70da75e..124a73a 100644 --- a/test-companion/test/approved-cases.test.mjs +++ b/test-companion/test/approved-cases.test.mjs @@ -12,6 +12,8 @@ import { renderExactSave, exactSaveSummary, exactSaveOutcome, + renderPlannedCases, + casesDigest, CASES_ARTIFACT, } from "../src/approved-cases.mjs" import { OUTCOMES } from "../src/outcome.mjs" @@ -25,6 +27,8 @@ const REAL_BINDINGS = JSON.parse( '"bindings":[{"name":"generate-on-comment","workflow":"generate","on":"pr_comment","mode":"live","approval":"review-before-run","enabled":true}]}', ) const GENERATE_BINDING = REAL_BINDINGS.bindings[0] +// The sha256 of the cases file a plan showed, as its marker records it. +const DIGEST = "d".repeat(64) // A generate preview's result.md.json, shaped like the CLI's. const CASES_FILE = { @@ -51,10 +55,10 @@ const withDestination = { ...CASES_FILE, destination: { project: "Checkout", fol describe("approvedSavePlan: a plain go on a generate plan saves the plan's own cases", () => { test("reproduces the live failure: real binding shape, plain go, run id present → exact, not run", () => { assert.equal("input" in GENERATE_BINDING, false) - assert.deepEqual(approvedSavePlan({ workflow: GENERATE_BINDING.workflow, replyText: "go", binding: GENERATE_BINDING, runId: "1234567890" }), { - mode: "exact", - runId: "1234567890", - }) + assert.deepEqual( + approvedSavePlan({ workflow: GENERATE_BINDING.workflow, replyText: "go", binding: GENERATE_BINDING, runId: "1234567890", casesSha256: DIGEST }), + { mode: "exact", runId: "1234567890", casesSha256: DIGEST }, + ) }) test("go with edits regenerates, and says it is because of the edits", () => { @@ -82,7 +86,13 @@ describe("approvedSavePlan: a plain go on a generate plan saves the plan's own c for (const replyText of plainGoes) { assert.equal(isApprovalOnly(replyText), true, `"${replyText}" should be a plain approval`) for (const binding of bindings) { - assert.equal(approvedSavePlan({ workflow: "generate", replyText, binding, runId: "77" }).mode, "exact", `"${replyText}" with ${JSON.stringify(binding)}`) + assert.equal( + approvedSavePlan({ workflow: "generate", replyText, binding, runId: "77", casesSha256: DIGEST }).mode, + "exact", + `"${replyText}" with ${JSON.stringify(binding)}`, + ) + // Without the cases' fingerprint it is refused, still never a new generate run. + assert.equal(approvedSavePlan({ workflow: "generate", replyText, binding, runId: "77" }).mode, "unavailable") } } }) @@ -228,7 +238,7 @@ describe("the wiring", () => { test("a plan attaches its cases, and approving one saves them with tm add instead of running again", () => { assert.match(main, /setOutput\("cases-dir", CASES_DIR\)/) - assert.match(main, /env\.GITHUB_RUN_ID,\n\s*\),\n\s*\)/) + assert.match(main, /env\.GITHUB_RUN_ID,\n[^\n]*\n\s*\{ build: buildRunId, cases: casesSha256 \},\n\s*\),\n\s*\)/) // No --project/--folder: tm add takes the destination from the approved file. assert.match(main, /\["tm", "add", "--from-file", casesFile, "--json"\]/) assert.doesNotMatch(main, /binding\?\.input|\.input\?\.tm_/) @@ -252,3 +262,84 @@ describe("the wiring", () => { assert.match(proposing, /setOutput\("summary-file", SUMMARY_FILE\)/) }) }) + +describe("approving saves exactly the cases the plan showed", () => { + const file = Buffer.from( + JSON.stringify({ + destination: { project: "rag_assignment", folder: "github-app", projectId: "PR-42", folderId: "7" }, + cases: [ + { id: "T001", title: "Base delay", priority: "high", preconditions: "Module importable", steps: [{ step: "Call f(1)", result: "500" }] }, + { id: "T002", title: "Ping @octocat [here](https://evil.test)", steps: [] }, + ], + }), + ) + + test("the plan is shown from the cases file, with where it will be saved", () => { + const shown = renderPlannedCases(file) + assert.match(shown.body, /\*\*Will be saved to:\*\* github-app in rag\\_assignment \(PR-42, 7\), once approved\./) + assert.match(shown.body, /\*\*2 test cases:\*\*/) + assert.match(shown.body, /#### T001 — Base delay\nPriority: high\nBefore you start: Module importable\n1\. Call f\\\(1\\\)\n {3}- Expected: 500/) + assert.equal(shown.digest, casesDigest(file)) + }) + + test("agent-written text can't mention anyone or add a link", () => { + const shown = renderPlannedCases(file).body + assert.doesNotMatch(shown, /@octocat/) + assert.doesNotMatch(shown, /\]\(https:\/\//) + assert.doesNotMatch(shown, /https:\/\/evil/) + }) + + test("one changed byte changes the fingerprint", () => { + const changed = Buffer.from(file.toString("utf8").replace("Base delay", "Base delaz")) + assert.notEqual(casesDigest(changed), renderPlannedCases(file).digest) + }) + + test("a file with no destination says approving can't save it", () => { + const none = Buffer.from(JSON.stringify({ cases: [{ id: "T001", title: "x" }] })) + assert.match(renderPlannedCases(none).body, /Will be saved to:\*\* nowhere/) + }) + + test("a plan too long for one comment still lists every title", () => { + const big = Buffer.from( + JSON.stringify({ + destination: { project: "p", folder: "f" }, + cases: Array.from({ length: 300 }, (_, i) => ({ id: `T${i}`, title: `case ${i}`, steps: [{ step: "s".repeat(200), result: "r" }] })), + }), + ) + const shown = renderPlannedCases(big).body + assert.ok(shown.length < 60_000, String(shown.length)) + assert.match(shown, /- T299 — case 299/) + }) + + test("an unreadable file shows no generated plan", () => { + assert.equal(renderPlannedCases(Buffer.from("not json")), undefined) + assert.equal(renderPlannedCases(Buffer.from("{}")), undefined) + }) + + test("the fingerprint travels in our plan comment", () => { + const digest = casesDigest(file) + const body = renderProposal("plan", "abc123", "octocat", "generate", "1234567890", { cases: digest }) + const waiting = findProposal([{ id: 5, user: { login: "browserstack[bot]", type: "Bot" }, created_at: new Date().toISOString(), body }], "abc123", Date.now(), "browserstack[bot]") + assert.equal(waiting.casesSha256, digest) + assert.deepEqual(approvedSavePlan({ workflow: "generate", replyText: "go", runId: waiting.runId, casesSha256: waiting.casesSha256 }), { + mode: "exact", + runId: "1234567890", + casesSha256: digest, + }) + }) + + test("a plan without the fingerprint is refused, with a reason", () => { + assert.deepEqual(approvedSavePlan({ workflow: "generate", replyText: "go", runId: "1" }), { mode: "unavailable", reason: "no-digest" }) + assert.match(renderCasesUnavailable("no-digest"), /didn't record which cases it showed/) + assert.match(renderCasesUnavailable("changed"), /not the ones this plan showed/) + }) + + test("the wiring checks the fingerprint before tm add, and shows a generate plan from the file", () => { + const main = readFileSync(new URL("../src/main.mjs", import.meta.url), "utf8") + const fn = main.slice(main.indexOf("function saveExactly("), main.indexOf("function pickBinding(")) + const check = fn.indexOf('casesDigest(bytes) !== save.casesSha256) return unavailable("changed")') + assert.ok(check > 0, "no fingerprint check") + assert.ok(check < fn.indexOf('"tm", "add"'), "the check must come before tm add") + assert.match(main, /renderPlannedCases\(readBytes\(join\(CASES_DIR, CASES_DATA_FILE\)\)\)/) + }) +}) From 40ef7c1474b70b45cb5f3fe16e54f67654c03096 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 16 Nov 2026 12:55:02 +0530 Subject: [PATCH 19/31] test-companion: never read a failed run that was started from a fork A fork's pull request can carry the same commit as a branch here, and its runs show up in this repository's run list. Neither the automatic choice of failed run nor a run named in a comment checked where the run came from, so `explain`, `fix` or `find the bad PR` could read a log a fork's author wrote and act on it. Both now keep only runs whose head repository is this repository (compared without case). A run that doesn't say, or a caller that doesn't say which repository it is, gets nothing: the check fails closed. Naming a fork's run is refused with a reason. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/main.mjs | 2 + test-companion/src/runs.mjs | 21 +++++++-- test-companion/test/run-selection.test.mjs | 50 ++++++++++++++++++---- test-companion/test/runs.test.mjs | 3 +- 4 files changed, 63 insertions(+), 13 deletions(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 61aaa13..8738fd0 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -770,6 +770,7 @@ async function main() { shasComplete: listed.complete, selfRunId: env.GITHUB_RUN_ID, selfWorkflowName: env.TC_ACTION_SELF_WORKFLOW, + repo: REPO, }) if (!check.ok) { // Never a different run in its place: that is not what was approved. @@ -794,6 +795,7 @@ async function main() { const ci = ciOnCommit(runs?.workflow_runs, { selfRunId: env.GITHUB_RUN_ID, selfWorkflowName: env.TC_ACTION_SELF_WORKFLOW, + repo: REPO, }) const chosen = matchFailedByName(requestText, ci.failed) ?? ci.failed[0] if (!chosen) { diff --git a/test-companion/src/runs.mjs b/test-companion/src/runs.mjs index 4512f1f..2e00e42 100644 --- a/test-companion/src/runs.mjs +++ b/test-companion/src/runs.mjs @@ -39,13 +39,25 @@ const nameOf = (r) => .replace(/[[\]`<>|\r\n]/g, " ") .trim() || "workflow" +/** + * Whether a run was started from this repository's own branches, not a fork's. A fork's pull + * request can carry the same commit, and its run's log is the fork author's to write. Fails + * closed: a run, or a caller, that doesn't say which repository is never ours. + */ +export function isOwnRepoRun(run, repo) { + const want = String(repo ?? "").toLowerCase() + const from = String(run?.head_repository?.full_name ?? "").toLowerCase() + return want !== "" && from === want +} + /** * The other runs on a commit, newest first: finished red ones and ones still going. - * Excludes our own runs. + * Excludes our own runs, and runs started from a fork. */ -export function ciOnCommit(runs, { selfRunId, selfWorkflowName } = {}) { +export function ciOnCommit(runs, { selfRunId, selfWorkflowName, repo } = {}) { const others = (runs ?? []) .filter((r) => r && String(r.id) !== String(selfRunId) && (!selfWorkflowName || r.name !== selfWorkflowName)) + .filter((r) => isOwnRepoRun(r, repo)) .sort((a, b) => (b.id ?? 0) - (a.id ?? 0)) const pick = (r) => ({ id: r.id, name: nameOf(r), url: r.html_url }) return { @@ -182,7 +194,7 @@ export function prCommitShas(slurped, headSha) { } /** Why a run named in a comment can't be used, or { ok, url, name }. */ -export function checkNamedRun(run, { prShas = [], shasComplete = true, selfRunId, selfWorkflowName } = {}) { +export function checkNamedRun(run, { prShas = [], shasComplete = true, selfRunId, selfWorkflowName, repo } = {}) { if (!run?.id || !run?.html_url) return { why: "I could not find that run in this repository." } if (String(run.id) === String(selfRunId) || (selfWorkflowName && run.name === selfWorkflowName)) { return { why: "That is one of Test Companion's own runs." } @@ -195,6 +207,9 @@ export function checkNamedRun(run, { prShas = [], shasComplete = true, selfRunId : "I couldn't list all of this pull request's commits, so I can only use runs on its latest commit. Name a run from the latest commit, or ask without naming one.", } } + if (!isOwnRepoRun(run, repo)) { + return { why: "That run was started from a fork, not from this repository's own branches, so I won't read its log." } + } if (IN_FLIGHT.has(run.status)) return { why: "That run is still going. Ask me again when it finishes." } if (run.status !== "completed" || !RED.has(run.conclusion)) { return { why: `That run did not fail (it finished as ${run.conclusion ?? "unknown"}).` } diff --git a/test-companion/test/run-selection.test.mjs b/test-companion/test/run-selection.test.mjs index 6c2dc90..97506bf 100644 --- a/test-companion/test/run-selection.test.mjs +++ b/test-companion/test/run-selection.test.mjs @@ -21,6 +21,7 @@ const run = (id, name, status, conclusion = null, extra = {}) => ({ conclusion, html_url: `https://github.com/o/r/actions/runs/${id}`, head_sha: "abc1234", + head_repository: { full_name: "o/r" }, ...extra, }) @@ -35,7 +36,7 @@ describe("ciOnCommit", () => { run(5, "Test Companion", "in_progress"), run(6, "Other", "completed", "failure"), ], - { selfRunId: 6, selfWorkflowName: "Test Companion" }, + { selfRunId: 6, selfWorkflowName: "Test Companion", repo: "o/r" }, ) assert.deepEqual( ci.failed.map((r) => r.name), @@ -48,7 +49,7 @@ describe("ciOnCommit", () => { }) test("a workflow name can't break the comment's formatting", () => { - const ci = ciOnCommit([run(1, "x`] (evil)|\nname", "completed", "failure")]) + const ci = ciOnCommit([run(1, "x`] (evil)|\nname", "completed", "failure")], { repo: "o/r" }) assert.doesNotMatch(ci.failed[0].name, /[`[\]|\n]/) }) }) @@ -104,15 +105,15 @@ describe("matchFailedByName", () => { describe("checkNamedRun", () => { const prShas = ["abc1234", "def5678"] test("a finished red run on one of the PR's commits is used", () => { - const r = checkNamedRun(run(9, "E2E", "completed", "failure"), { prShas }) + const r = checkNamedRun(run(9, "E2E", "completed", "failure"), { prShas, repo: "o/r" }) assert.deepEqual(r, { ok: true, url: "https://github.com/o/r/actions/runs/9", name: "E2E" }) }) test("each reason a run can't be used", () => { - assert.match(checkNamedRun(null, { prShas }).why, /could not find/) - assert.match(checkNamedRun(run(9, "E2E", "completed", "failure", { head_sha: "zzz" }), { prShas }).why, /not for a commit/) - assert.match(checkNamedRun(run(9, "E2E", "in_progress"), { prShas }).why, /still going/) - assert.match(checkNamedRun(run(9, "E2E", "completed", "success"), { prShas }).why, /did not fail/) - assert.match(checkNamedRun(run(9, "TC", "completed", "failure"), { prShas, selfWorkflowName: "TC" }).why, /own runs/) + assert.match(checkNamedRun(null, { prShas, repo: "o/r" }).why, /could not find/) + assert.match(checkNamedRun(run(9, "E2E", "completed", "failure", { head_sha: "zzz" }), { prShas, repo: "o/r" }).why, /not for a commit/) + assert.match(checkNamedRun(run(9, "E2E", "in_progress"), { prShas, repo: "o/r" }).why, /still going/) + assert.match(checkNamedRun(run(9, "E2E", "completed", "success"), { prShas, repo: "o/r" }).why, /did not fail/) + assert.match(checkNamedRun(run(9, "TC", "completed", "failure"), { prShas, selfWorkflowName: "TC", repo: "o/r" }).why, /own runs/) }) }) @@ -179,7 +180,7 @@ describe("approving a plan reads the run that plan looked at", () => { test("the recorded run is still checked: one that is no longer red can't be used", () => { const run = { id: 200, html_url: e2e.url, head_sha: SHA, status: "completed", conclusion: "success", name: e2e.name, head_repository: { full_name: "org/repo" } } - assert.match(checkNamedRun(run, { prShas: [SHA] }).why, /did not fail/) + assert.match(checkNamedRun(run, { prShas: [SHA], repo: "org/repo" }).why, /did not fail/) }) test("the marker keeps the run and still reads markers without it", () => { @@ -190,3 +191,34 @@ describe("approving a plan reads the run that plan looked at", () => { assert.equal(old.buildRunId, undefined) }) }) + +describe("a fork's run is never the one read", () => { + const fork = (id, extra = {}) => run(id, "CI", "completed", "failure", { head_repository: { full_name: "mallory/r" }, ...extra }) + + test("a fork's red run on the same commit is left out", () => { + const ci = ciOnCommit([fork(9), run(3, "E2E", "completed", "failure")], { repo: "o/r" }) + assert.deepEqual( + ci.failed.map((r) => r.id), + [3], + ) + }) + + test("the repository is compared without case", () => { + assert.equal(ciOnCommit([run(3, "E2E", "completed", "failure")], { repo: "O/R" }).failed.length, 1) + }) + + test("a run that doesn't say where it came from is left out", () => { + const unknown = run(3, "E2E", "completed", "failure", { head_repository: undefined }) + assert.equal(ciOnCommit([unknown], { repo: "o/r" }).failed.length, 0) + assert.match(checkNamedRun(unknown, { prShas: ["abc1234"], repo: "o/r" }).why, /fork/) + }) + + test("a caller that doesn't say which repository gets nothing", () => { + assert.equal(ciOnCommit([run(3, "E2E", "completed", "failure")], {}).failed.length, 0) + assert.match(checkNamedRun(run(3, "E2E", "completed", "failure"), { prShas: ["abc1234"] }).why, /fork/) + }) + + test("naming a fork's run is refused", () => { + assert.match(checkNamedRun(fork(9), { prShas: ["abc1234"], repo: "o/r" }).why, /started from a fork/) + }) +}) diff --git a/test-companion/test/runs.test.mjs b/test-companion/test/runs.test.mjs index b5eba12..f2eaaf6 100644 --- a/test-companion/test/runs.test.mjs +++ b/test-companion/test/runs.test.mjs @@ -92,7 +92,7 @@ describe("runIdsFromChecks — finding OUR in-flight runs on a commit", () => { describe("ciOnCommit — which build 'explain the failure' means", () => { // The newest failed run is failed[0]. - const latestFailedRun = (runs, opts) => ciOnCommit(runs, opts).failed[0]?.url + const latestFailedRun = (runs, opts) => ciOnCommit(runs, { repo: "acme/shop", ...opts }).failed[0]?.url // `rca`, `fix` and `verdict` need to know which build failed. const run = (id, conclusion, name = "CI", status = "completed") => ({ id, @@ -100,6 +100,7 @@ describe("ciOnCommit — which build 'explain the failure' means", () => { status, conclusion, html_url: `https://github.com/acme/shop/actions/runs/${id}`, + head_repository: { full_name: "acme/shop" }, }) test("picks the newest failed run", () => { From d3e9c7937951f91b60a2af9d7f6e7d258205318e Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 16 Nov 2026 12:56:38 +0530 Subject: [PATCH 20/31] test-companion: say when an older CLI is why a plan's cases can't be saved Saving a plan's own cases needs CLI 1.1.16 or later. With an older CLI on the approving run, `tm add --from-file` doesn't exist and the reply blamed Test Management; with a plan made by CLI 1.1.15, the cases file has no destination and the reply told users to set tm_project and tm_folder even when they had, so they could loop. The approving run now checks its CLI has `tm add --from-file` before doing anything, and says it needs 1.1.16 or later (the plan stays approvable). The plan's run records which CLI made it, so a missing destination from an older CLI is named as that; a missing destination from a new enough CLI still points at the pipeline file, and mentions the CLI as the other cause. The step that runs the Action now receives the CLI version line; it was only passed to the sign-in step, so the Action never had it. Co-Authored-By: Claude Opus 5.5 --- test-companion/action.yml | 2 + test-companion/src/approved-cases.mjs | 51 +++++++++++++++--- test-companion/src/main.mjs | 19 ++++++- test-companion/test/approved-cases.test.mjs | 58 ++++++++++++++++++++- 4 files changed, 121 insertions(+), 9 deletions(-) diff --git a/test-companion/action.yml b/test-companion/action.yml index 764bf6a..498eb97 100644 --- a/test-companion/action.yml +++ b/test-companion/action.yml @@ -213,6 +213,8 @@ runs: TC_ACTION_APP_ID: ${{ inputs.app-id }} TC_ACTION_APP_PRIVATE_KEY: ${{ inputs.app-private-key }} TC_ACTION_ALLOW_UNISOLATED: ${{ inputs.allow-unisolated }} + # The `tc --version` line, so a plan records which CLI made it. + TC_ACTION_CLI_VERSION: ${{ steps.install.outputs.cli-version }} run: node "${{ github.action_path }}/src/main.mjs" - name: Attach the report diff --git a/test-companion/src/approved-cases.mjs b/test-companion/src/approved-cases.mjs index 4b4f9bd..624b8ad 100644 --- a/test-companion/src/approved-cases.mjs +++ b/test-companion/src/approved-cases.mjs @@ -130,9 +130,39 @@ export function pickCasesArtifact(list, runId) { return live ? { artifact: live } : { error: "expired" } } -/** What the plan run records about itself, so a go run can't save cases made for another repo or commit. */ -export function casesSubject({ repo, sha }) { - return { repo: String(repo ?? ""), sha: String(sha ?? "") } +/** + * What the plan run records about itself, so a go run can't save cases made for another repo or + * commit. `cli` is the `tc --version` line that made the plan, to tell an older CLI apart from a + * missing destination. + */ +export function casesSubject({ repo, sha, cli }) { + return { repo: String(repo ?? ""), sha: String(sha ?? ""), cli: String(cli ?? "") } +} + +// The first CLI whose cases file records its Test Management destination. +export const CASES_DESTINATION_CLI = "1.1.16" + +/** The x.y.z in a `tc --version` line, as numbers, or undefined. */ +export function cliVersionOf(text) { + const m = /(\d+)\.(\d+)\.(\d+)/.exec(String(text ?? "")) + return m ? [Number(m[1]), Number(m[2]), Number(m[3])] : undefined +} + +/** Whether that CLI writes a destination into its cases file: true, false, or undefined when unknown. */ +export function cliRecordsDestination(text) { + const v = cliVersionOf(text) + if (!v) return undefined + const min = cliVersionOf(CASES_DESTINATION_CLI) + for (let i = 0; i < 3; i++) if (v[i] !== min[i]) return v[i] > min[i] + return true +} + +/** + * Why a cases file with no destination can't be saved: "old-cli-plan" when the plan's CLI is + * known to predate destinations, else "no-destination" (none configured, or the CLI unknown). + */ +export function missingDestinationReason(planCli) { + return cliRecordsDestination(planCli) === false ? "old-cli-plan" : "no-destination" } /** Whether a downloaded subject.json matches the repo and commit being approved; `why` when not. */ @@ -145,7 +175,7 @@ export function checkCasesSubject(subjectText, { repo, sha }) { } if (!subject || subject.repo !== repo) return { ok: false, why: "other-repo" } if (!sha || subject.sha !== sha) return { ok: false, why: "other-commit" } - return { ok: true } + return { ok: true, cli: String(subject.cli ?? "") } } /** @@ -189,6 +219,8 @@ const UNAVAILABLE_WHY = { "no-digest": "this plan didn't record which cases it showed (an older version of Test Companion made it)", changed: "the saved cases are not the ones this plan showed", "no-destination": "the plan recorded no Test Management destination, so there is nowhere to save its cases", + "old-cli": `saving a plan's own cases needs Test Companion CLI ${CASES_DESTINATION_CLI} or later, and this run has an older one`, + "old-cli-plan": `this plan was made with a Test Companion CLI older than ${CASES_DESTINATION_CLI}, which doesn't record where to save its cases`, "save-failed": "Test Management refused the cases before any were saved", } @@ -196,12 +228,17 @@ const UNAVAILABLE_WHY = { export function renderCasesUnavailable(reason, detail) { const why = UNAVAILABLE_WHY[reason] ?? "the plan's cases could not be used" const extra = String(detail ?? "").trim() + const notInstead = "I have not generated a new set in its place, because that would save cases nobody reviewed." + const update = `Update the CLI: remove the action's \`cli-version\` pin, or pin ${CASES_DESTINATION_CLI} or later.` + const next = { + "no-destination": `${notInstead} Set \`tm_project\` and \`tm_folder\` for generate in the pipeline file. If they are already set, the plan was probably made with a CLI older than ${CASES_DESTINATION_CLI}: ${update} Then ask me to **generate** again and approve the fresh plan.`, + "old-cli": `${notInstead} ${update} Then reply **go** to this plan again; it is still waiting.`, + "old-cli-plan": `${notInstead} ${update} Then ask me to **generate** again and approve the fresh plan.`, + } return [ `I did not save anything: ${why}${extra ? ` (${extra})` : ""}.`, "", - reason === "no-destination" - ? "I have not generated a new set in its place, because that would save cases nobody reviewed. Set `tm_project` and `tm_folder` for generate in the pipeline file, then ask me to **generate** again and approve the fresh plan." - : "I have not generated a new set in its place, because that would save cases nobody reviewed. Ask me to **generate** again and approve the fresh plan.", + next[reason] ?? `${notInstead} Ask me to **generate** again and approve the fresh plan.`, ].join("\n") } diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 8738fd0..fbf503b 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -25,6 +25,7 @@ import { childEnvFor } from "./child-env.mjs" import { approvedSavePlan, readCasesDestination, + missingDestinationReason, renderPlannedCases, casesDigest, pickCasesArtifact, @@ -1198,6 +1199,16 @@ function isolateAgent(plan) { process.exit(1) } +/** Whether the installed CLI can save a plan's own cases (`tc tm add --from-file`), asked once. */ +let tmAddFromFile +function cliHasTmAddFromFile() { + if (tmAddFromFile !== undefined) return tmAddFromFile + // An unknown subcommand prints its parent's help, which never names this flag. + const r = spawnSync(TC_PATH, ["tm", "add", "--help"], { encoding: "utf8", env: childEnvFor(env), timeout: 60_000 }) + tmAddFromFile = /--from-file/.test(`${r.stdout ?? ""}${r.stderr ?? ""}`) + return tmAddFromFile +} + /** What the installed CLI supports, asked once. */ let cliAbilities function cliSupports() { @@ -1655,7 +1666,10 @@ function keepCases(text, plan) { try { mkdirSync(CASES_DIR, { recursive: true }) writeFileSync(join(CASES_DIR, CASES_DATA_FILE), text) - writeFileSync(join(CASES_DIR, CASES_SUBJECT_FILE), JSON.stringify(casesSubject({ repo: REPO, sha: plan.context?.sha }))) + writeFileSync( + join(CASES_DIR, CASES_SUBJECT_FILE), + JSON.stringify(casesSubject({ repo: REPO, sha: plan.context?.sha, cli: env.TC_ACTION_CLI_VERSION })), + ) } catch { // Without it a plan can still be read; approving it says the cases are missing. } @@ -1700,6 +1714,8 @@ function saveApprovedCases(plan, save, approvalNote) { } function saveExactly(plan, save, sha, base, finish, unavailable) { + // An older CLI has no `tm add --from-file`; say so rather than report a refusal. + if (!cliHasTmAddFromFile()) return unavailable("old-cli", String(env.TC_ACTION_CLI_VERSION ?? "").trim()) const picked = pickCasesArtifact(gh(["api", `repos/${REPO}/actions/runs/${save.runId}/artifacts?per_page=100`]), save.runId) if (picked.error) return unavailable(picked.error) @@ -1714,6 +1730,7 @@ function saveExactly(plan, save, sha, base, finish, unavailable) { if (bytes === undefined || casesDigest(bytes) !== save.casesSha256) return unavailable("changed") // The destination comes from the approved file itself, so the cases go where the plan said. const where = readCasesDestination(bytes.toString("utf8")) + if (where.error === "no-destination") return unavailable(missingDestinationReason(subject.cli), subject.cli || undefined) if (where.error) return unavailable(where.error) notice(`Test Companion: saving the approved plan's cases from run ${save.runId} to ${where.folder} in ${where.project}.`) diff --git a/test-companion/test/approved-cases.test.mjs b/test-companion/test/approved-cases.test.mjs index 124a73a..5d60943 100644 --- a/test-companion/test/approved-cases.test.mjs +++ b/test-companion/test/approved-cases.test.mjs @@ -12,6 +12,10 @@ import { renderExactSave, exactSaveSummary, exactSaveOutcome, + cliVersionOf, + cliRecordsDestination, + missingDestinationReason, + CASES_DESTINATION_CLI, renderPlannedCases, casesDigest, CASES_ARTIFACT, @@ -135,7 +139,7 @@ describe("pickCasesArtifact", () => { describe("checkCasesSubject: the saved cases must be for this repo and this plan's commit", () => { const subject = JSON.stringify(casesSubject({ repo: "o/r", sha: "abc123" })) test("same repo and commit", () => { - assert.deepEqual(checkCasesSubject(subject, { repo: "o/r", sha: "abc123" }), { ok: true }) + assert.deepEqual(checkCasesSubject(subject, { repo: "o/r", sha: "abc123" }), { ok: true, cli: "" }) }) test("another repo, another commit, or junk is refused", () => { assert.deepEqual(checkCasesSubject(subject, { repo: "o/other", sha: "abc123" }), { ok: false, why: "other-repo" }) @@ -343,3 +347,55 @@ describe("approving saves exactly the cases the plan showed", () => { assert.match(main, /renderPlannedCases\(readBytes\(join\(CASES_DIR, CASES_DATA_FILE\)\)\)/) }) }) + +describe("an older CLI is named as the problem, not the pipeline file", () => { + test("reads the version from the `tc --version` line", () => { + assert.deepEqual(cliVersionOf("test-companion CLI v1.1.16 (beta)"), [1, 1, 16]) + assert.equal(cliVersionOf("unknown"), undefined) + assert.equal(CASES_DESTINATION_CLI, "1.1.16") + }) + + test("which CLIs record a destination", () => { + assert.equal(cliRecordsDestination("test-companion CLI v1.1.15 (beta)"), false) + assert.equal(cliRecordsDestination("test-companion CLI v1.0.99"), false) + assert.equal(cliRecordsDestination("test-companion CLI v1.1.16 (beta)"), true) + assert.equal(cliRecordsDestination("test-companion CLI v1.2.0"), true) + assert.equal(cliRecordsDestination("test-companion CLI v2.0.0"), true) + assert.equal(cliRecordsDestination(""), undefined) + }) + + test("a plan made by an older CLI isn't blamed on the pipeline file", () => { + assert.equal(missingDestinationReason("test-companion CLI v1.1.15 (beta)"), "old-cli-plan") + const text = renderCasesUnavailable("old-cli-plan", "test-companion CLI v1.1.15 (beta)") + assert.match(text, /older than 1\.1\.16/) + assert.doesNotMatch(text, /Set `tm_project`/) + assert.match(text, /generate\*\* again/) + }) + + test("with a new enough CLI, a missing destination points at the pipeline file, and mentions the CLI only as a fallback", () => { + assert.equal(missingDestinationReason("test-companion CLI v1.1.16 (beta)"), "no-destination") + assert.equal(missingDestinationReason(""), "no-destination") + const text = renderCasesUnavailable("no-destination") + assert.match(text, /Set `tm_project` and `tm_folder`/) + assert.match(text, /If they are already set, the plan was probably made with a CLI older than 1\.1\.16/) + }) + + test("a go run whose CLI can't save from a file says so, and the plan stays approvable", () => { + const text = renderCasesUnavailable("old-cli", "test-companion CLI v1.1.14") + assert.match(text, /needs Test Companion CLI 1\.1\.16 or later/) + assert.match(text, /reply \*\*go\*\* to this plan again; it is still waiting/) + assert.doesNotMatch(text, /refused/) + }) + + test("the plan records its CLI, and the go run checks its own before anything else", () => { + const subject = JSON.stringify(casesSubject({ repo: "o/r", sha: "abc", cli: "test-companion CLI v1.1.15 (beta)" })) + assert.equal(checkCasesSubject(subject, { repo: "o/r", sha: "abc" }).cli, "test-companion CLI v1.1.15 (beta)") + const main = readFileSync(new URL("../src/main.mjs", import.meta.url), "utf8") + const fn = main.slice(main.indexOf("function saveExactly("), main.indexOf("function pickBinding(")) + assert.ok(fn.indexOf("cliHasTmAddFromFile()") < fn.indexOf("pickCasesArtifact("), "the CLI check comes first") + assert.match(fn, /missingDestinationReason\(subject\.cli\)/) + const action = readFileSync(new URL("../action.yml", import.meta.url), "utf8") + const runStep = action.slice(action.indexOf("- name: Run\n"), action.indexOf("- name: Attach the report")) + assert.match(runStep, /TC_ACTION_CLI_VERSION: \$\{\{ steps\.install\.outputs\.cli-version \}\}/) + }) +}) From 3fc6bced20c1013ec1afceca6c29b0efee87e827 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 16 Nov 2026 12:58:33 +0530 Subject: [PATCH 21/31] test-companion: bring the README up to date - Each request gets its own reply; automatic runs keep one status comment. - Hand-back pull requests: a draft, based on the pull request asked on, one squashed bot commit, by-products left out, history rewrites refused. - The GitHub App's repository permissions, and why Actions needs read and write. - Saving a generate plan's own cases needs CLI 1.1.16 or later; the pin example now uses a version that exists. - Existing users should copy the starter again, so bot comments stop starting runs. - github-token should be the workflow token or the App's, not a person's. Co-Authored-By: Claude Opus 5.5 --- test-companion/README.md | 53 ++++++++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/test-companion/README.md b/test-companion/README.md index fec0f65..6fe4750 100644 --- a/test-companion/README.md +++ b/test-companion/README.md @@ -3,8 +3,10 @@ Ask BrowserStack Test Companion for test work from a pull request. Comment `@browserstack test this PR` and it reads the change, proposes the -tests it would write, and writes them once someone replies `go`. The result -comes back as one comment and one check. +tests it would write, and writes them once someone replies `go`. Each request +(an `@browserstack` comment, or a `go`) gets its own reply with that run's +report, plus a "Test Companion" check. Automatic runs, which answer no +request, keep one status comment per pull request up to date. ## Quick start @@ -34,6 +36,10 @@ are easy to miss: - concurrency rules that keep one run per pull request and let closing a pull request cancel its run. +**Already using an older copy?** Copy the starter file again. Older copies let +comments written by bots, Test Companion's own included, start a run, which +costs a runner each time and can push a waiting `go` out of the queue. + ## Pin the CLI version Without a pin the CLI upgrades itself on a release day, so two runs a day apart @@ -45,7 +51,7 @@ from the release you tested; a version without a checksum is refused. with: username: ${{ secrets.BROWSERSTACK_USERNAME }} access-key: ${{ secrets.BROWSERSTACK_ACCESS_KEY }} - cli-version: "1.2.0" + cli-version: "1.1.16" cli-sha256: "" ``` @@ -62,6 +68,7 @@ from the release you tested; a version without a checksum is refused. | `preview` | no | Report what would happen and change nothing | | `allow-unisolated` | no | `"true"` runs the agent as the job's own user when it cannot be locked down, where it can reach this job's tokens and secrets. Off by default | | `app-id` / `app-private-key` | no | Post as the GitHub App. Keep the workflow's `permissions:` block: the workflow token is the fallback | +| `github-token` | no | The token for the Action's own GitHub calls. Leave it as the default (the workflow token) or use the GitHub App's token, never a personal access token: Test Companion recognises its own comments by their bot account, and a person's token would make them look like that person's | ## Approving a plan @@ -70,13 +77,39 @@ someone with write access to reply `go`. Read the plan before you reply. Test Companion reacts 👀 to a request, or a `go`, once it picks it up. For `generate`, a plain `go` saves exactly the test cases in the plan to Test -Management, in the project and folder the plan recorded: the plan's run keeps -them (the `test-companion-cases` artifact, for 7 days) and nothing is generated -again. A `@browserstack go` with changes generates the cases again with your -changes as guidance, and says so. If the plan's cases can't be found, or the -plan recorded no Test Management destination (set `tm_project` and `tm_folder` -for generate in the pipeline file), nothing is saved and you are asked to -generate a fresh plan. +Management, in the project and folder the plan shows under "Will be saved to": +the plan's run keeps them (the `test-companion-cases` artifact, for 7 days) and +nothing is generated again. This needs **Test Companion CLI 1.1.16 or later**, +for both the plan and the `go`; with an older CLI nothing is saved and the reply +says to update it. A `@browserstack go` with changes generates the cases again +with your changes as guidance, and says so. If the plan's cases can't be found +or have changed, or the plan recorded no Test Management destination (set +`tm_project` and `tm_folder` for generate in the pipeline file), nothing is +saved and you are asked to generate a fresh plan. + +For `fix` and `automate`, the work is handed back as a pull request: + +- opened as a draft (a normal pull request where drafts aren't available); +- based on the pull request it was asked on, so it carries only the fix (if the + run didn't start on that pull request's commit, it's based on the default + branch instead, and the reply says so); +- one squashed commit by Test Companion's bot account, so the agent's own + commits never reach GitHub; +- tool by-products (logs, `.env`, `browserstack.err` and the like) left out; +- refused, with nothing pushed, if the agent's branch rewrote history, for + example by resetting past the commit that was reviewed. + +### The GitHub App's permissions + +Installing the App asks for these repository permissions, and each run's token +is limited to the one repository it runs in: + +| Permission | Why | +|---|---| +| Contents: read and write | Commits approved tests to the pull request's branch, and pushes the branch for a hand-back pull request | +| Pull requests: read and write | Posts its replies, and opens the hand-back pull request | +| Checks: read and write | Posts the "Test Companion" check | +| Actions: read and write | Read: the failed run's logs for `explain`, `fix` and `find the bad PR`, and the plan's saved cases when a `generate` plan is approved. Write: cancelling its own runs when a pull request is closed without merging | Without the GitHub App, Test Companion posts as `github-actions[bot]`, the account every workflow in your repository shares, and recognises its own plans From 2dbf7a4bd6786cb7421bba7f71e08a89f9e1376c Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 16 Nov 2026 14:00:49 +0530 Subject: [PATCH 22/31] test-companion: safer exact saves (interrupted saves, lookups, reply text, artifact origin) - A `tc tm add` that stopped without a report (killed, timed out, or no output) may have saved some cases. The plan is now used up and the reply says to check Test Management before asking again, so a second `go` can't save them twice. Only the CLI's own "nothing was saved" leaves it approvable. - A failed lookup of the plan's artifacts is reported as a lookup failure (the plan stays approvable), not as "the cases are missing". The error output of `gh run download` and `tm add` goes to the job log with secrets hidden, never to the comment. - The exact-save reply shows titles, ids and errors as plain text, and links a case only when its address is https on browserstack.com. - The cases artifact must come from a run in this repository that started from it (repository_id and head_repository_id). Its commit is not compared: a plan asked for in a comment runs on the default branch's commit; the plan's commit is checked through subject.json and the cases' fingerprint. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/approved-cases.mjs | 72 +++++++++++--- test-companion/src/main.mjs | 32 +++++- test-companion/test/approved-cases.test.mjs | 104 ++++++++++++++++++-- 3 files changed, 184 insertions(+), 24 deletions(-) diff --git a/test-companion/src/approved-cases.mjs b/test-companion/src/approved-cases.mjs index 624b8ad..926fd89 100644 --- a/test-companion/src/approved-cases.mjs +++ b/test-companion/src/approved-cases.mjs @@ -121,11 +121,22 @@ export function readCasesDestination(casesText) { return { project, folder } } -/** The plan run's cases artifact from a `GET .../runs//artifacts` reply, or why there is none. */ -export function pickCasesArtifact(list, runId) { +/** + * The plan run's cases artifact from a `GET .../runs//artifacts` reply, or why there is none. + * The run must belong to this repository and have started from it (`repoId`, GitHub's numeric + * id): fails closed when either is missing. Its commit is not compared: a plan asked for in a + * comment runs on the default branch's commit, and the plan's own commit is checked through + * subject.json and the cases' fingerprint instead. + */ +export function pickCasesArtifact(list, runId, { repoId } = {}) { const want = String(runId ?? "") - const mine = (list?.artifacts ?? []).filter((a) => a?.name === CASES_ARTIFACT && String(a?.workflow_run?.id ?? "") === want) - if (mine.length === 0) return { error: "missing" } + const named = (list?.artifacts ?? []).filter((a) => a?.name === CASES_ARTIFACT && String(a?.workflow_run?.id ?? "") === want) + if (named.length === 0) return { error: "missing" } + const repo = String(repoId ?? "") + const ours = (a) => + repo !== "" && String(a?.workflow_run?.repository_id ?? "") === repo && String(a?.workflow_run?.head_repository_id ?? "") === repo + const mine = named.filter(ours) + if (mine.length === 0) return { error: "other-repo" } const live = mine.find((a) => !a.expired) return live ? { artifact: live } : { error: "expired" } } @@ -222,6 +233,7 @@ const UNAVAILABLE_WHY = { "old-cli": `saving a plan's own cases needs Test Companion CLI ${CASES_DESTINATION_CLI} or later, and this run has an older one`, "old-cli-plan": `this plan was made with a Test Companion CLI older than ${CASES_DESTINATION_CLI}, which doesn't record where to save its cases`, "save-failed": "Test Management refused the cases before any were saved", + "lookup-failed": "I couldn't look up the plan's saved cases on GitHub just now", } /** The reply when the approved cases can't be saved as they are. Never regenerates. */ @@ -231,6 +243,7 @@ export function renderCasesUnavailable(reason, detail) { const notInstead = "I have not generated a new set in its place, because that would save cases nobody reviewed." const update = `Update the CLI: remove the action's \`cli-version\` pin, or pin ${CASES_DESTINATION_CLI} or later.` const next = { + "lookup-failed": `${notInstead} Reply **go** to this plan again in a minute; it is still waiting.`, "no-destination": `${notInstead} Set \`tm_project\` and \`tm_folder\` for generate in the pipeline file. If they are already set, the plan was probably made with a CLI older than ${CASES_DESTINATION_CLI}: ${update} Then ask me to **generate** again and approve the fresh plan.`, "old-cli": `${notInstead} ${update} Then reply **go** to this plan again; it is still waiting.`, "old-cli-plan": `${notInstead} ${update} Then ask me to **generate** again and approve the fresh plan.`, @@ -242,7 +255,20 @@ export function renderCasesUnavailable(reason, detail) { ].join("\n") } -const oneLine = (s) => String(s ?? "").replace(/\s+/g, " ").trim() +/** A link to a saved case, only to BrowserStack over https; anything else is not linked. */ +export function browserstackLink(url) { + let u + try { + u = new URL(String(url ?? "")) + } catch { + return undefined + } + const host = u.hostname.toLowerCase() + if (u.protocol !== "https:" || u.username || u.password) return undefined + if (host !== "browserstack.com" && !host.endsWith(".browserstack.com")) return undefined + // href percent-encodes spaces and angle brackets, so it is safe inside <...>. + return u.href +} /** The answer text for an exact save: what was created, with ids, and anything that failed. */ export function renderExactSave(report, { project, folder } = {}) { @@ -250,21 +276,22 @@ export function renderExactSave(report, { project, folder } = {}) { const failed = report?.failed ?? [] const skipped = report?.skipped ?? [] const lines = [] - const where = `${folder} in ${project}` + const where = `${plain(folder)} in ${plain(project)}` if (created.length > 0) { - lines.push(`Saved the ${created.length} approved test case${created.length === 1 ? "" : "s"} to \`${where}\` exactly as planned; nothing was generated again.`) + lines.push(`Saved the ${created.length} approved test case${created.length === 1 ? "" : "s"} to ${where} exactly as planned; nothing was generated again.`) lines.push("") for (const c of created) { - const id = oneLine(c.identifier) - const label = c.url && id ? `[${id}](${c.url})` : id - lines.push(`- ${label}${label ? " — " : ""}${oneLine(c.title)}`) + const id = plain(c.identifier) + const link = browserstackLink(c.url) + const label = link && id ? `[${id}](<${link}>)` : id + lines.push(`- ${label}${label ? " — " : ""}${plain(c.title)}`) } } if (failed.length > 0) { if (lines.length) lines.push("") lines.push(`**${failed.length} could not be saved:**`) lines.push("") - for (const f of failed) lines.push(`- ${oneLine(f.title)}: ${oneLine(f.error)}`) + for (const f of failed) lines.push(`- ${plain(f.title)}: ${plain(f.error)}`) } if (skipped.length > 0) { if (lines.length) lines.push("") @@ -301,6 +328,29 @@ export function exactSaveOutcome(report, outcomes) { return { outcome: outcomes.RUN_BROKE, spend: created + skipped > 0 } } +/** + * The reply when `tc tm add` stopped without saying what it saved (killed, timed out, or no + * report). Some cases may be in Test Management, so the plan is used up: approving it again + * could save them twice. + */ +export function renderSaveInterrupted({ project, folder } = {}, why) { + const where = project && folder ? `${plain(folder)} in ${plain(project)}` : "Test Management" + const extra = String(why ?? "").trim() + return [ + `The save stopped before it reported what it saved${extra ? ` (${plain(extra)})` : ""}, so some of the approved cases may already be in ${where}.`, + "", + `This plan is now used up, so approving it again can't save them twice. Check ${where} before asking me to **generate** again.`, + ].join("\n") +} + +/** + * Whether `tc tm add` said, in its own words, that it saved nothing. Only then is the plan + * left approvable; any other stop may have saved some cases. + */ +export function tmAddSavedNothing(stderr) { + return /\bnothing was saved\b/i.test(String(stderr ?? "")) +} + /** A note for the reply when a go with edits had to generate again. */ export const EDITED_GO_NOTE = "Your go included changes, and saving the plan as it was can't apply them, so the cases were generated again with your changes as guidance. Check the list below; it may differ from the plan." diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index fbf503b..627258a 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -25,6 +25,8 @@ import { childEnvFor } from "./child-env.mjs" import { approvedSavePlan, readCasesDestination, + renderSaveInterrupted, + tmAddSavedNothing, missingDestinationReason, renderPlannedCases, casesDigest, @@ -1652,6 +1654,12 @@ function readOptional(path) { } } +/** Write a command's error output to the job log, with secrets hidden. Never posted on the PR. */ +function logRedacted(what, text) { + const shown = redactSecrets(String(text ?? "").trim(), secretsNow()) + if (shown) console.log(`Test Companion: ${what} said:\n${shown.slice(0, 4000)}`) +} + function readBytes(path) { try { return readFileSync(path) @@ -1716,12 +1724,18 @@ function saveApprovedCases(plan, save, approvalNote) { function saveExactly(plan, save, sha, base, finish, unavailable) { // An older CLI has no `tm add --from-file`; say so rather than report a refusal. if (!cliHasTmAddFromFile()) return unavailable("old-cli", String(env.TC_ACTION_CLI_VERSION ?? "").trim()) - const picked = pickCasesArtifact(gh(["api", `repos/${REPO}/actions/runs/${save.runId}/artifacts?per_page=100`]), save.runId) + // A lookup that failed is not "the cases are missing": the plan stays approvable. + const lookup = classifyLookup(ghRaw(["api", `repos/${REPO}/actions/runs/${save.runId}/artifacts?per_page=100`])) + if (lookup.failed) return unavailable("lookup-failed") + const picked = pickCasesArtifact(lookup.found, save.runId, { repoId: env.GITHUB_REPOSITORY_ID }) if (picked.error) return unavailable(picked.error) const dir = mkdtempSync(join(REPORT_DIR, "tc-approved-cases-")) const got = runGh(["run", "download", save.runId, "-R", REPO, "-n", CASES_ARTIFACT, "-D", dir], { token: env.GH_TOKEN }) - if (got.status !== 0) return unavailable("download-failed") + if (got.status !== 0) { + logRedacted("gh run download", got.stderr) + return unavailable("download-failed") + } const subject = checkCasesSubject(readOptional(join(dir, CASES_SUBJECT_FILE)), { repo: REPO, sha }) if (!subject.ok) return unavailable(subject.why) const casesFile = join(dir, CASES_DATA_FILE) @@ -1739,11 +1753,19 @@ function saveExactly(plan, save, sha, base, finish, unavailable) { env: childEnvFor(env), timeout: 300_000, }) + const stderr = String(r.stderr ?? "") const report = parseTmAddReport(r.stdout) if (!report) { - const why = String(r.stderr ?? "").trim().split("\n").filter(Boolean).at(-1) - return unavailable("save-failed", redactSecrets(why, secretsNow())) - } + logRedacted("tc tm add", stderr || r.error?.message) + const said = stderr.trim().split("\n").filter(Boolean).at(-1) + const why = redactSecrets(said || (r.signal ? `stopped by ${r.signal}` : r.error?.message) || "", secretsNow()) + // Only the CLI's own "nothing was saved" leaves the plan approvable. A save that was + // killed, timed out or printed no report may have saved some, so the plan is used up: + // a second go could save them twice. + if (!r.error && !r.signal && tmAddSavedNothing(stderr)) return unavailable("save-failed", why) + return finish({ ...base, outcome: OUTCOMES.RUN_BROKE, answer: renderSaveInterrupted(where, why) }, true) + } + if (report.failed.length > 0) logRedacted("tc tm add", stderr) const { outcome, spend } = exactSaveOutcome(report, OUTCOMES) finish({ ...base, outcome, summary: { ...base.summary, ...exactSaveSummary(report, where) }, answer: renderExactSave(report, where) }, spend) } diff --git a/test-companion/test/approved-cases.test.mjs b/test-companion/test/approved-cases.test.mjs index 5d60943..47ac593 100644 --- a/test-companion/test/approved-cases.test.mjs +++ b/test-companion/test/approved-cases.test.mjs @@ -19,6 +19,9 @@ import { renderPlannedCases, casesDigest, CASES_ARTIFACT, + browserstackLink, + renderSaveInterrupted, + tmAddSavedNothing, } from "../src/approved-cases.mjs" import { OUTCOMES } from "../src/outcome.mjs" import { isApprovalOnly } from "../src/intent.mjs" @@ -124,15 +127,28 @@ describe("readCasesDestination: where the cases go comes from the approved file" }) describe("pickCasesArtifact", () => { - const art = (over) => ({ name: CASES_ARTIFACT, expired: false, workflow_run: { id: 7 }, id: 1, ...over }) + const run = { id: 7, repository_id: 42, head_repository_id: 42, head_sha: "def" } + const art = (over) => ({ name: CASES_ARTIFACT, expired: false, workflow_run: run, id: 1, ...over }) + const opts = { repoId: "42" } test("the plan run's live cases artifact", () => { - assert.deepEqual(pickCasesArtifact({ artifacts: [art({ name: "test-companion-report" }), art()] }, "7"), { artifact: art() }) + assert.deepEqual(pickCasesArtifact({ artifacts: [art({ name: "test-companion-report" }), art()] }, "7", opts), { artifact: art() }) }) test("none from that run is missing; only expired ones is expired", () => { - assert.deepEqual(pickCasesArtifact({ artifacts: [art({ workflow_run: { id: 8 } })] }, "7"), { error: "missing" }) - assert.deepEqual(pickCasesArtifact({ artifacts: [art({ expired: true })] }, "7"), { error: "expired" }) - assert.deepEqual(pickCasesArtifact(null, "7"), { error: "missing" }) + assert.deepEqual(pickCasesArtifact({ artifacts: [art({ workflow_run: { ...run, id: 8 } })] }, "7", opts), { error: "missing" }) + assert.deepEqual(pickCasesArtifact({ artifacts: [art({ expired: true })] }, "7", opts), { error: "expired" }) + assert.deepEqual(pickCasesArtifact(null, "7", opts), { error: "missing" }) + }) + test("a run from another repository, or started from a fork, is refused", () => { + assert.deepEqual(pickCasesArtifact({ artifacts: [art({ workflow_run: { ...run, repository_id: 9 } })] }, "7", opts), { error: "other-repo" }) + assert.deepEqual(pickCasesArtifact({ artifacts: [art({ workflow_run: { ...run, head_repository_id: 9 } })] }, "7", opts), { error: "other-repo" }) + }) + test("fails closed when either side doesn't say which repository", () => { + assert.deepEqual(pickCasesArtifact({ artifacts: [art()] }, "7", {}), { error: "other-repo" }) + assert.deepEqual(pickCasesArtifact({ artifacts: [art({ workflow_run: { id: 7 } })] }, "7", opts), { error: "other-repo" }) + }) + test("the run's commit is not compared: a plan asked for in a comment runs on the default branch", () => { + assert.ok(pickCasesArtifact({ artifacts: [art({ workflow_run: { ...run, head_sha: "anything" } })] }, "7", opts).artifact) }) }) @@ -172,14 +188,14 @@ describe("parseTmAddReport reads tc tm add --json", () => { describe("what the reply says", () => { const where = { project: "Checkout", folder: "Web" } const created = [ - { title: "Base delay", identifier: "TC-1", url: "https://tm.example/TC-1" }, + { title: "Base delay", identifier: "TC-1", url: "https://test-management.browserstack.com/projects/1/test-cases/TC-1" }, { title: "Cap", identifier: "TC-2" }, ] test("every saved case is listed with its id, from tm add's own output", () => { const text = renderExactSave({ created, failed: [], skipped: [] }, where) - assert.match(text, /Saved the 2 approved test cases to `Web in Checkout` exactly as planned; nothing was generated again\./) - assert.match(text, /- \[TC-1\]\(https:\/\/tm\.example\/TC-1\) — Base delay/) + assert.match(text, /Saved the 2 approved test cases to Web in Checkout exactly as planned; nothing was generated again\./) + assert.match(text, /- \[TC-1\]\(\) — Base delay/) assert.match(text, /- TC-2 — Cap/) }) @@ -399,3 +415,75 @@ describe("an older CLI is named as the problem, not the pipeline file", () => { assert.match(runStep, /TC_ACTION_CLI_VERSION: \$\{\{ steps\.install\.outputs\.cli-version \}\}/) }) }) + +describe("the exact-save reply can't be used to mention people or plant links", () => { + test("only https BrowserStack addresses are linked", () => { + assert.ok(browserstackLink("https://test-management.browserstack.com/projects/1")) + assert.ok(browserstackLink("https://browserstack.com/x")) + for (const u of ["https://evil.test/x", "http://browserstack.com/x", "https://browserstack.com.evil.test/", "https://u@browserstack.com/", "javascript:alert(1)", ""]) { + assert.equal(browserstackLink(u), undefined, u) + } + }) + + test("titles, ids and errors are plain text", () => { + const text = renderExactSave( + { + created: [{ identifier: "TC-1", title: "Ping @octocat [here](https://evil.test)", url: "https://evil.test/TC-1" }], + failed: [{ title: "**bold**", error: "see " }], + skipped: [], + }, + { project: "P", folder: "F" }, + ) + assert.doesNotMatch(text, /@octocat/) + assert.doesNotMatch(text, /\]\(https:\/\/evil/) + assert.doesNotMatch(text, /\( { + test("only the CLI's own 'nothing was saved' keeps the plan approvable", () => { + assert.equal(tmAddSavedNothing("Error: nothing was saved: case T003 has no title"), true) + assert.equal(tmAddSavedNothing("signal: killed"), false) + assert.equal(tmAddSavedNothing(""), false) + }) + + test("the reply says some may be saved, and to check before asking again", () => { + const text = renderSaveInterrupted({ project: "rag", folder: "github-app" }, "stopped by SIGTERM") + assert.match(text, /some of the approved cases may already be in github-app in rag/) + assert.match(text, /used up/) + assert.match(text, /Check github-app in rag before asking me to \*\*generate\*\* again/) + }) + + test("the wiring: killed, timed out or no report is a used-up Run broke; only 'nothing was saved' stays approvable", () => { + const main = readFileSync(new URL("../src/main.mjs", import.meta.url), "utf8") + const fn = main.slice(main.indexOf("function saveExactly("), main.indexOf("function pickBinding(")) + assert.match(fn, /if \(!r\.error && !r\.signal && tmAddSavedNothing\(stderr\)\) return unavailable\("save-failed", why\)/) + assert.match(fn, /return finish\(\{ \.\.\.base, outcome: OUTCOMES\.RUN_BROKE, answer: renderSaveInterrupted\(where, why\) \}, true\)/) + }) +}) + +describe("failure detail is kept, and kept out of the comment", () => { + test("a failed artifact lookup is reported as a lookup failure, and the plan stays approvable", () => { + const text = renderCasesUnavailable("lookup-failed") + assert.match(text, /couldn't look up the plan's saved cases/) + assert.match(text, /it is still waiting/) + const main = readFileSync(new URL("../src/main.mjs", import.meta.url), "utf8") + const fn = main.slice(main.indexOf("function saveExactly("), main.indexOf("function pickBinding(")) + assert.match(fn, /if \(lookup\.failed\) return unavailable\("lookup-failed"\)/) + assert.match(fn, /pickCasesArtifact\(lookup\.found, save\.runId, \{ repoId: env\.GITHUB_REPOSITORY_ID \}\)/) + }) + + test("download and tm add errors go to the job log with secrets hidden, not the comment", () => { + const main = readFileSync(new URL("../src/main.mjs", import.meta.url), "utf8") + const log = main.slice(main.indexOf("function logRedacted("), main.indexOf("function readBytes(")) + assert.match(log, /redactSecrets\(/) + assert.match(log, /console\.log\(/) + assert.doesNotMatch(log, /replyOnPr|postComment/) + const fn = main.slice(main.indexOf("function saveExactly("), main.indexOf("function pickBinding(")) + assert.match(fn, /logRedacted\("gh run download", got\.stderr\)/) + assert.match(fn, /logRedacted\("tc tm add", stderr \|\| r\.error\?\.message\)/) + }) +}) From 045938ca59aa1870831e51bcd1c8dbb5c44c7b51 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 16 Nov 2026 14:01:49 +0530 Subject: [PATCH 23/31] test-companion: never answer our own comments when github-token is a person's Our comments quote "@browserstack go". The bot filters skip them by account type, which works for the workflow token and the App, but a personal access token posts as a person, so our own replies could start runs. The Action now asks GitHub who github-token belongs to (only a person's token can read GET /user) and, if it's a person, skips that account's comments and warns that the workflow token or the App should be used. The account we post as is skipped whatever its type. The README already says github-token should be the workflow token or the App's. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/main.mjs | 20 +++++++++++++++ test-companion/src/plan-run.mjs | 14 +++++++---- test-companion/test/plan-run.test.mjs | 36 +++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 5 deletions(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 627258a..2020c53 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -252,6 +252,23 @@ function fetchAndCheckout(sha, branch) { return branch ? git(["checkout", "-B", branch, sha]).ok : git(["checkout", "--detach", sha]).ok } +/** + * The person `github-token` belongs to, when it is a personal token, else undefined. Only a + * person's token can read `GET /user`; the workflow token and an App token can't. Our comments + * then appear under that person's account, so they are skipped like a bot's would be. + */ +function personTokenLogin() { + const r = runGh(["api", "user", "--jq", ".login"], { token: env.GH_TOKEN }) + const login = r.status === 0 ? String(r.stdout ?? "").trim() : "" + if (!login) return undefined + warn( + `Test Companion: github-token belongs to ${login}, a person, not the workflow or the GitHub App. ` + + `Its comments will appear as ${login}'s, so Test Companion ignores comments by ${login} and can't recognise its own plans. ` + + "Use the default workflow token or the GitHub App instead.", + ) + return login +} + /** A repo-scoped App installation token, or undefined to fall back to the workflow token. */ async function appInstallationToken() { const creds = appCredentials(env) @@ -478,6 +495,8 @@ async function main() { env.GH_TOKEN = appToken notice("Test Companion: acting as the GitHub App.") } + // The account our comments appear under, so we never answer ourselves. + const selfLogin = appToken ? botLogin : personTokenLogin() ?? botLogin // No credentials is the normal case for a fork pull request: skip, don't fail. if (!env.BROWSERSTACK_USERNAME || !env.BROWSERSTACK_ACCESS_KEY) { @@ -494,6 +513,7 @@ async function main() { let plan = planRun({ eventName: env.GITHUB_EVENT_NAME, + selfLogin, payload: readEvent(), repo: REPO, sha: env.GITHUB_SHA, diff --git a/test-companion/src/plan-run.mjs b/test-companion/src/plan-run.mjs index 673d54d..888ec15 100644 --- a/test-companion/src/plan-run.mjs +++ b/test-companion/src/plan-run.mjs @@ -95,8 +95,8 @@ function cleanContext(pairs) { * Decide what this event should run: `{skip}` or * `{event?, workflow?, preview?, context}`. No event means ignore the bindings. */ -export function planRun({ eventName, payload = {}, inputs = {}, repo, sha, ref } = {}) { - const built = route({ eventName, payload, inputs, repo, sha, ref }) +export function planRun({ eventName, payload = {}, inputs = {}, repo, sha, ref, selfLogin } = {}) { + const built = route({ eventName, payload, inputs, repo, sha, ref, selfLogin }) if (built.skip) return built // A named job wins on any trigger and bypasses the bindings. @@ -116,10 +116,10 @@ export function planRun({ eventName, payload = {}, inputs = {}, repo, sha, ref } return { ...built, context } } -function route({ eventName, payload, inputs, repo, sha, ref }) { +function route({ eventName, payload, inputs, repo, sha, ref, selfLogin }) { switch (eventName) { case "issue_comment": - return fromComment(payload, inputs) + return fromComment(payload, inputs, selfLogin) case "pull_request": case "pull_request_target": return fromPullRequest(payload, repo) @@ -139,12 +139,16 @@ function route({ eventName, payload, inputs, repo, sha, ref }) { } } -function fromComment(payload, inputs) { +function fromComment(payload, inputs, selfLogin) { if (payload.action !== "created") return skip("comment-edited", payload.action) if (!payload.issue?.pull_request) return skip("not-a-pull-request") // Our own comments quote "@browserstack go"; reacting to them loops and can push a real request out of the queue. const author = payload.comment?.user if (author?.type === "Bot" || /\[bot\]$/i.test(author?.login ?? "")) return skip("bot-comment", author?.login) + // The account we post as, whatever its type: with a person's token our comments are that + // person's, and they quote "@browserstack go" too. + const self = String(selfLogin ?? "").trim().toLowerCase() + if (self && String(author?.login ?? "").toLowerCase() === self) return skip("own-comment", author?.login) const body = payload.comment?.body ?? "" const addressed = MENTION.test(body) diff --git a/test-companion/test/plan-run.test.mjs b/test-companion/test/plan-run.test.mjs index 63ce06b..db6f3db 100644 --- a/test-companion/test/plan-run.test.mjs +++ b/test-companion/test/plan-run.test.mjs @@ -1,5 +1,6 @@ import { test, describe } from "node:test" import assert from "node:assert/strict" +import { readFileSync } from "node:fs" import { planRun, TC_EVENTS, contextArgs, withPullRequestHead } from "../src/plan-run.mjs" const pr = (over = {}) => ({ @@ -657,3 +658,38 @@ describe("comments from bots, our own included, are ignored", () => { assert.equal(run({ login: "an-sdet", type: "User" }).skip, undefined) }) }) + +describe("comments by the account we post as never start a run", () => { + const event = (login, type = "User", body = "Reply **@browserstack go** to run this for real.") => ({ + action: "created", + issue: { number: 7, pull_request: {} }, + comment: { id: 1, body, user: { login, type } }, + }) + + test("with a person's token, that person's comments are ours and skipped", () => { + const out = planRun({ eventName: "issue_comment", payload: event("jane"), repo: "acme/shop", selfLogin: "jane" }) + assert.equal(out.skip?.reason, "own-comment") + }) + + test("the login is compared without case", () => { + assert.equal(planRun({ eventName: "issue_comment", payload: event("Jane"), repo: "acme/shop", selfLogin: "jane" }).skip?.reason, "own-comment") + }) + + test("someone else's request still runs", () => { + const out = planRun({ eventName: "issue_comment", payload: event("bob", "User", "@browserstack test this PR"), repo: "acme/shop", selfLogin: "jane" }) + assert.equal(out.skip, undefined) + }) + + test("bots are still skipped with no login given", () => { + assert.equal(planRun({ eventName: "issue_comment", payload: event("browserstack[bot]", "Bot"), repo: "acme/shop" }).skip?.reason, "bot-comment") + }) + + test("the wiring: a person's token is detected and warned about, and its login is passed", () => { + const main = readFileSync(new URL("../src/main.mjs", import.meta.url), "utf8") + assert.match(main, /const selfLogin = appToken \? botLogin : personTokenLogin\(\) \?\? botLogin/) + assert.match(main, /planRun\(\{\n\s*eventName: env\.GITHUB_EVENT_NAME,\n\s*selfLogin,/) + const fn = main.slice(main.indexOf("function personTokenLogin("), main.indexOf("async function appInstallationToken(")) + assert.match(fn, /\["api", "user", "--jq", "\.login"\]/) + assert.match(fn, /warn\(/) + }) +}) From 3c1fea820b522a774198fa752399ecd80aa78ef0 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 16 Nov 2026 14:02:23 +0530 Subject: [PATCH 24/31] test-companion: read every run on a commit, not just the first 100 Choosing which failed run to read, and cancelling runs when a pull request closes, both listed one page of 100 runs. A busy commit could have its failure past the first page. Both now read every page, up to 1,000 runs. If any page can't be read, the list counts as unreadable, so a partial list can't make "nothing failed" look true. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/main.mjs | 10 +++++-- test-companion/src/runs.mjs | 16 +++++++++++ test-companion/test/run-selection.test.mjs | 33 ++++++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 2020c53..4681e5f 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -9,6 +9,7 @@ import { chooseRunsToCancel, runIdsFromChecks, ciOnCommit, + collectRunPages, runRequestFor, matchFailedByName, checkNamedRun, @@ -809,7 +810,7 @@ async function main() { plan.context.build = check.url buildNote = renderBuildNote(check, { ask: ASK_WORD[plan.workflow] }) } else { - const runs = gh(["api", `repos/${REPO}/actions/runs?head_sha=${plan.context.sha}&per_page=100`]) + const runs = runsOnCommit(plan.context.sha) if (!Array.isArray(runs?.workflow_runs)) { // A failed lookup is not "nothing failed". replyOnPr(plan.context.pr, LOOKUP_FAILED_REPLY) @@ -1503,7 +1504,7 @@ function cancelRunsOn(sha) { const checks = gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`]) const fromChecks = runIdsFromChecks(checks?.check_runs, { selfRunId: env.GITHUB_RUN_ID }) // Plus head_sha matches, for pull_request runs whose check isn't posted yet. - const list = gh(["api", `repos/${REPO}/actions/runs?head_sha=${sha}&per_page=100`]) + const list = runsOnCommit(sha) const fromRuns = chooseRunsToCancel(list?.workflow_runs, { selfRunId: env.GITHUB_RUN_ID, headSha: sha, @@ -1796,6 +1797,11 @@ function pickBinding(doc, workflow) { return workflow ? list.find((b) => b.workflow === workflow) : list[0] } +/** Every workflow run on a commit (all pages, up to 1,000), or undefined if any page failed. */ +function runsOnCommit(sha) { + return collectRunPages((page) => gh(["api", `repos/${REPO}/actions/runs?head_sha=${sha}&per_page=100&page=${page}`])) +} + /** A plain reply. Carries no COMMENT_MARKER, so later runs never overwrite it. */ function replyOnPr(pr, body) { if (!pr) return diff --git a/test-companion/src/runs.mjs b/test-companion/src/runs.mjs index 2e00e42..e773e0a 100644 --- a/test-companion/src/runs.mjs +++ b/test-companion/src/runs.mjs @@ -5,6 +5,22 @@ const IN_FLIGHT = new Set(["queued", "in_progress", "waiting", "requested", "pending"]) +/** + * Every run on a commit, page by page, up to `maxPages` pages (1,000 runs by default). + * `fetchPage(page)` returns GitHub's reply for one page. Any page that can't be read makes the + * whole list unreadable (undefined): a partial list could make "nothing failed" look true. + */ +export function collectRunPages(fetchPage, { perPage = 100, maxPages = 10 } = {}) { + const all = [] + for (let page = 1; page <= maxPages; page++) { + const got = fetchPage(page) + if (!Array.isArray(got?.workflow_runs)) return undefined + all.push(...got.workflow_runs) + if (got.workflow_runs.length < perPage) break + } + return { workflow_runs: all } +} + /** Ids are compared as strings: GITHUB_RUN_ID is a string, the API's a number. */ export function chooseRunsToCancel(runs, { selfRunId, headSha, workflowName } = {}) { return (runs ?? []) diff --git a/test-companion/test/run-selection.test.mjs b/test-companion/test/run-selection.test.mjs index 97506bf..4324bca 100644 --- a/test-companion/test/run-selection.test.mjs +++ b/test-companion/test/run-selection.test.mjs @@ -1,5 +1,6 @@ import { test, describe } from "node:test" import assert from "node:assert/strict" +import { readFileSync } from "node:fs" import { ciOnCommit, parseRunReference, @@ -8,6 +9,7 @@ import { renderCiStillRunning, renderBuildNote, runRequestFor, + collectRunPages, } from "../src/runs.mjs" import { findProposal, renderProposal, proposalMarkerFor } from "../src/report.mjs" @@ -222,3 +224,34 @@ describe("a fork's run is never the one read", () => { assert.match(checkNamedRun(fork(9), { prShas: ["abc1234"], repo: "o/r" }).why, /started from a fork/) }) }) + +describe("every run on a commit is read, not just the first 100", () => { + const page = (n, size) => ({ workflow_runs: Array.from({ length: size }, (_, i) => ({ id: n * 1000 + i })) }) + + test("reads pages until one comes back short", () => { + const asked = [] + const got = collectRunPages((p) => (asked.push(p), page(p, p < 3 ? 100 : 7))) + assert.deepEqual(asked, [1, 2, 3]) + assert.equal(got.workflow_runs.length, 207) + }) + + test("stops at 1,000 runs", () => { + const asked = [] + const got = collectRunPages((p) => (asked.push(p), page(p, 100))) + assert.equal(asked.length, 10) + assert.equal(got.workflow_runs.length, 1000) + }) + + test("a page that can't be read makes the list unreadable, never a partial 'nothing failed'", () => { + assert.equal(collectRunPages((p) => (p === 2 ? undefined : page(p, 100))), undefined) + assert.equal(collectRunPages(() => ({ message: "Bad credentials" })), undefined) + }) + + test("both listings in the Action use it", () => { + const main = readFileSync(new URL("../src/main.mjs", import.meta.url), "utf8") + assert.doesNotMatch(main, /actions\/runs\?head_sha=\$\{[^}]+\}&per_page=100`\]\)/) + assert.match(main, /const runs = runsOnCommit\(plan\.context\.sha\)/) + assert.match(main, /const list = runsOnCommit\(sha\)/) + assert.match(main, /&per_page=100&page=\$\{page\}/) + }) +}) From 55b36cb4ec328c400cb618c4a68227b5fb8044f9 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 16 Nov 2026 14:03:13 +0530 Subject: [PATCH 25/31] test-companion: cap the whole App sign-in at about 30 seconds Each of the two App calls could take three 10-second tries plus waits, so a GitHub outage delayed the fallback to the workflow token by 70 to 100 seconds. The sign-in now has one 30-second budget for every call, try and wait: each try's time limit shrinks to what's left, no try starts after the deadline, and no wait runs that would end past it. The retry rules are otherwise unchanged. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/app-auth.mjs | 28 +++++++++-- test-companion/src/main.mjs | 5 +- test-companion/test/app-auth.test.mjs | 71 +++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 4 deletions(-) diff --git a/test-companion/src/app-auth.mjs b/test-companion/src/app-auth.mjs index 9ec723e..f967f46 100644 --- a/test-companion/src/app-auth.mjs +++ b/test-companion/src/app-auth.mjs @@ -30,17 +30,39 @@ export function jwtClaims(appId, nowSeconds) { */ export async function appApi( path, - { jwt, method = "GET", body, apiUrl, fetchImpl = fetch, attempts = 3, sleep = defaultSleep, timeoutMs = APP_CALL_TIMEOUT_MS } = {}, + { + jwt, + method = "GET", + body, + apiUrl, + fetchImpl = fetch, + attempts = 3, + sleep = defaultSleep, + timeoutMs = APP_CALL_TIMEOUT_MS, + // When set, no try starts, and no wait runs, past this time (ms since the epoch). + deadline, + now = Date.now, + } = {}, ) { + const left = () => (deadline === undefined ? Infinity : deadline - now()) let result for (let attempt = 1; attempt <= attempts; attempt++) { - result = await appApiOnce(path, { jwt, method, body, apiUrl, fetchImpl, timeoutMs }) + if (left() <= 0) return { ...(result ?? OUT_OF_TIME), attempts: attempt - 1 } + result = await appApiOnce(path, { jwt, method, body, apiUrl, fetchImpl, timeoutMs: Math.min(timeoutMs, left()) }) if (!isRetryable(result) || attempt === attempts) return { ...result, attempts: attempt } - await sleep(retryDelayMs(result, attempt)) + const wait = retryDelayMs(result, attempt) + // A wait that would end past the deadline can't lead to another try. + if (wait >= left()) return { ...result, attempts: attempt } + await sleep(wait) } return { ...result, attempts } } +const OUT_OF_TIME = { ok: false, status: 0, data: undefined, reason: "ran out of time to reach GitHub" } + +/** The most the whole App sign-in may take, every call, try and wait included, before falling back. */ +export const APP_SIGN_IN_BUDGET_MS = 30_000 + const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) /** One App call gives up after this long; a timeout counts as a network error, so it is retried. */ diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 4681e5f..eeb0801 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -53,6 +53,7 @@ import { signJwt, installationTokenRequest, appApi, + APP_SIGN_IN_BUDGET_MS, installationLookupMessage, } from "./app-auth.mjs" import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail, approvalWasSpent, runDidNotWork } from "./outcome.mjs" @@ -286,8 +287,10 @@ async function appInstallationToken() { } // The JWT authenticates the App; only the installation token can touch a repository. + // One time budget for the whole sign-in, so a GitHub outage delays the fallback by at most that. + const deadline = Date.now() + APP_SIGN_IN_BUDGET_MS const asApp = async (path, opts) => { - const r = await appApi(path, { jwt, apiUrl: env.GITHUB_API_URL, ...opts }) + const r = await appApi(path, { jwt, apiUrl: env.GITHUB_API_URL, deadline, ...opts }) if (r.ok && r.attempts > 1) notice(`Test Companion: reaching GitHub as the App took ${r.attempts} tries.`) return r } diff --git a/test-companion/test/app-auth.test.mjs b/test-companion/test/app-auth.test.mjs index 369e007..fdd0b5b 100644 --- a/test-companion/test/app-auth.test.mjs +++ b/test-companion/test/app-auth.test.mjs @@ -1,5 +1,6 @@ import { test, describe } from "node:test" import assert from "node:assert/strict" +import { readFileSync } from "node:fs" import { generateKeyPairSync, createVerify } from "node:crypto" import { appCredentials, @@ -7,6 +8,7 @@ import { signJwt, installationTokenRequest, appApi, + APP_SIGN_IN_BUDGET_MS, isRetryable, retryDelayMs, installationLookupMessage, @@ -250,3 +252,72 @@ describe("appApi gives each call a time limit", () => { assert.equal(calls, 3) }) }) + +describe("the whole App sign-in fits in about 30 seconds", () => { + // A fake clock: each GitHub call "takes" `callMs`, each wait takes its length. + function clock(callMs) { + let t = 0 + const calls = [] + return { + now: () => t, + sleep: async (ms) => { + t += ms + }, + fetchImpl: async () => { + calls.push(t) + t += callMs + return { ok: false, status: 503, statusText: "Service Unavailable", json: async () => ({}), headers: { get: () => null } } + }, + calls, + get t() { + return t + }, + } + } + + test("the budget is 30 seconds", () => { + assert.equal(APP_SIGN_IN_BUDGET_MS, 30_000) + }) + + test("two slow calls share one budget and stop by the deadline", async () => { + const c = clock(10_000) + const deadline = c.now() + APP_SIGN_IN_BUDGET_MS + const opts = { jwt: "J", fetchImpl: c.fetchImpl, sleep: c.sleep, now: c.now, deadline } + const first = await appApi("repos/o/r/installation", opts) + const second = await appApi("app/installations/1/access_tokens", { ...opts, method: "POST", body: {} }) + assert.equal(first.ok, false) + assert.equal(second.ok, false) + assert.ok(c.t <= 30_000 + 10_000, `took ${c.t}ms`) + assert.ok(c.calls.every((started) => started < deadline), "no call starts after the deadline") + }) + + test("no wait that would end past the deadline", async () => { + const c = clock(0) + const r = await appApi("x", { jwt: "J", fetchImpl: c.fetchImpl, sleep: c.sleep, now: c.now, deadline: 2_000 }) + // 1s wait after the first try fits; the 3s wait after the second would not. + assert.equal(r.attempts, 2) + assert.equal(c.t, 1_000) + }) + + test("past the deadline, nothing is tried", async () => { + const c = clock(0) + const r = await appApi("x", { jwt: "J", fetchImpl: c.fetchImpl, sleep: c.sleep, now: () => 5, deadline: 5 }) + assert.equal(c.calls.length, 0) + assert.equal(r.ok, false) + assert.equal(r.status, 0) + }) + + test("without a deadline the retry rules are unchanged", async () => { + const c = clock(0) + const r = await appApi("x", { jwt: "J", fetchImpl: c.fetchImpl, sleep: c.sleep, now: c.now }) + assert.equal(r.attempts, 3) + assert.equal(c.t, 4_000) + }) + + test("the sign-in passes one deadline to every call", () => { + const main = readFileSync(new URL("../src/main.mjs", import.meta.url), "utf8") + const fn = main.slice(main.indexOf("async function appInstallationToken("), main.indexOf("return token\n}")) + assert.match(fn, /const deadline = Date\.now\(\) \+ APP_SIGN_IN_BUDGET_MS/) + assert.match(fn, /appApi\(path, \{ jwt, apiUrl: env\.GITHUB_API_URL, deadline, \.\.\.opts \}\)/) + }) +}) From e718ed08f7edd2a7374519be5a73aec48aff42b2 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 16 Nov 2026 14:04:09 +0530 Subject: [PATCH 26/31] test-companion: don't hand back a pull request built on GitHub's merge commit On a pull_request event the job starts on GitHub's merge of the pull request into its base. If moving to the pull request's own commit failed, the run worked on the merge, the squash was parented on it, and the hand-back pull request (based on the default branch) carried the whole original pull request plus the fix. That case is now refused before anything is squashed or pushed, and the reply says why. Parenting the squash on the pull request's commit instead would not be right: the run's files are the merge's, so it would pull in the base branch's newer changes. A comment run that stayed on the default branch keeps its documented fallback. The README says both. Co-Authored-By: Claude Opus 5.5 --- test-companion/README.md | 6 +++-- test-companion/src/main.mjs | 7 ++++++ test-companion/src/push-work.mjs | 14 ++++++++++++ test-companion/test/push-work.test.mjs | 31 ++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 2 deletions(-) diff --git a/test-companion/README.md b/test-companion/README.md index 6fe4750..37db10a 100644 --- a/test-companion/README.md +++ b/test-companion/README.md @@ -91,8 +91,10 @@ For `fix` and `automate`, the work is handed back as a pull request: - opened as a draft (a normal pull request where drafts aren't available); - based on the pull request it was asked on, so it carries only the fix (if the - run didn't start on that pull request's commit, it's based on the default - branch instead, and the reply says so); + run started on the default branch instead of that pull request's commit, it's + based on the default branch, and the reply says so; if it started on GitHub's + merge of the pull request, nothing is opened, because it would carry the + whole pull request); - one squashed commit by Test Companion's bot account, so the agent's own commits never reach GitHub; - tool by-products (logs, `.env`, `browserstack.err` and the like) left out; diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index eeb0801..edf5bbb 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -96,6 +96,7 @@ import { splitPushable, splitHandoffPaths, checkHandoffDescends, + startedOnMergeCommit, REWROTE_HISTORY, savingFor, isRequestedByPerson, @@ -1369,6 +1370,12 @@ function openHandedOffPr(path, plan) { if (!headBeforeRun) return "**Could not open the pull request:** could not read the commit the run started on." const reviewed = git(["rev-parse", "--verify", "--quiet", `${headBeforeRun}^{commit}`]) if (!reviewed.ok || !reviewed.out) return "**Could not open the pull request:** could not read the commit the run started on." + if ( + plan.context.pr && + startedOnMergeCommit({ eventName: env.GITHUB_EVENT_NAME, startedOn: headBeforeRun, eventSha: env.GITHUB_SHA, prHead: plan.context.sha }) + ) { + return `**Did not open a pull request:** the run couldn't move to #${plan.context.pr}'s commit, so it ran on GitHub's merge of #${plan.context.pr} into its base. A pull request built from that would carry all of #${plan.context.pr}, not just the fix. Nothing was pushed; ask me again.` + } const descends = checkHandoffDescends({ git, reviewed: reviewed.out, tip: tip.out }) if (descends.error) return `**Could not open the pull request:** ${descends.error}.` if (!descends.ok) return `**Did not open a pull request:** ${REWROTE_HISTORY}.` diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index 4bf4d00..e7d852f 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -422,6 +422,20 @@ export function handedOffPrRequest({ req, pr, prBranch, defaultBranch, onReviewe } } +/** + * Did the run start on GitHub's merge commit for a pull request, rather than the pull request's + * own commit? On a `pull_request` event the job checks out that merge (the event's sha); if + * moving to the pull request's commit failed, the run's files are the merge's. A hand-back + * built from it would carry the whole pull request, so it is refused. (Parenting it on the pull + * request's commit instead would pull in the base branch's newer changes.) + */ +export function startedOnMergeCommit({ eventName, startedOn, eventSha, prHead } = {}) { + const at = String(startedOn ?? "").toLowerCase() + return ( + eventName === "pull_request" && at !== "" && at === String(eventSha ?? "").toLowerCase() && at !== String(prHead ?? "").toLowerCase() + ) +} + export const REWROTE_HISTORY = "the run's branch doesn't build on the commit that was reviewed (it rewrote history), so its changes can't be trusted as a fix for this pull request" diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index fddbbb3..db66d6e 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -29,6 +29,7 @@ import { savingFor, leaveOut, commitPaths, + startedOnMergeCommit, checkHandoffDescends, REWROTE_HISTORY, leaveOutMessage, @@ -1229,3 +1230,33 @@ describe("git ignores replace refs and grafts the agent could plant", () => { function plain(cwd, args) { return spawnSync("git", args, { cwd, encoding: "utf8" }) } + +describe("a hand-back is never built on the pull_request merge commit", () => { + const merge = "m".repeat(40) + const head = "h".repeat(40) + + test("a pull_request run still on the merge commit is caught", () => { + assert.equal(startedOnMergeCommit({ eventName: "pull_request", startedOn: merge, eventSha: merge, prHead: head }), true) + }) + + test("a run that moved to the pull request's commit is fine", () => { + assert.equal(startedOnMergeCommit({ eventName: "pull_request", startedOn: head, eventSha: merge, prHead: head }), false) + }) + + test("a comment run on the default branch keeps its documented fallback", () => { + assert.equal(startedOnMergeCommit({ eventName: "issue_comment", startedOn: merge, eventSha: merge, prHead: head }), false) + }) + + test("no starting commit is not read as the merge commit", () => { + assert.equal(startedOnMergeCommit({ eventName: "pull_request", startedOn: "", eventSha: "", prHead: head }), false) + }) + + test("the wiring refuses before anything is squashed or pushed", () => { + const main = readFileSync(new URL("../src/main.mjs", import.meta.url), "utf8") + const fn = main.slice(main.indexOf("function openHandedOffPr("), main.indexOf("function openHandedOffPr(") + 6000) + const refuse = fn.indexOf("startedOnMergeCommit(") + assert.ok(refuse > 0) + assert.ok(refuse < fn.indexOf("leaveOut("), "refused before the squash") + assert.ok(refuse < fn.indexOf('git(["push"'), "refused before any push") + }) +}) From fa37db7a76fa3033814a7f372cc45e1d60b551cc Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 16 Nov 2026 15:32:01 +0530 Subject: [PATCH 27/31] test-companion: a generate plan shows every field approving saves The plan showed a case's id, title, priority, preconditions and steps, but `tc tm add --from-file` also saves its description, type, tags, references and custom fields, and for a BDD case its feature, background and scenario. A reviewer could approve text and links they never saw. - The plan now shows every one of those fields, each case in a collapsible block, all as plain text: multi-line text in a code block, references and custom fields unlinked, no mentions. A case already in Test Management is labelled as skipped. - One list of shown fields drives both the plan and a check run just before saving: if any case carries a non-empty field the plan doesn't show (an unknown or newer field, or one of the wrong shape), nothing is saved and the reply says why. The plan warns about it too. - A plan too large for one comment shows the cases that fit, says it can't be approved to save exactly as shown, and records that in its marker, so approving it is refused and asks for fewer cases. The run's own report above the cases is kept short so the cases get most of the room. - A generate plan that had to fall back to the agent's text records that too, and approving it says the plan couldn't be checked against its cases file, rather than blaming an older version. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/approved-cases.mjs | 192 +++++++++++++++++--- test-companion/src/main.mjs | 35 +++- test-companion/src/report.mjs | 30 ++- test-companion/test/approved-cases.test.mjs | 165 +++++++++++++++-- 4 files changed, 369 insertions(+), 53 deletions(-) diff --git a/test-companion/src/approved-cases.mjs b/test-companion/src/approved-cases.mjs index 926fd89..2eff128 100644 --- a/test-companion/src/approved-cases.mjs +++ b/test-companion/src/approved-cases.mjs @@ -28,9 +28,12 @@ export const CASES_SUBJECT_FILE = "subject.json" * Safety rule: a plain go on a generate plan never becomes "run". Generating again would save * cases nobody reviewed, so every problem after this point is reported instead. */ -export function approvedSavePlan({ workflow, replyText, runId, casesSha256 } = {}) { +export function approvedSavePlan({ workflow, replyText, runId, casesSha256, casesShown } = {}) { if (workflow !== "generate") return { mode: "run" } if (hasApproverEdits(replyText)) return { mode: "run", edited: true } + // A plan that didn't show all its cases, or showed the agent's text instead, wasn't reviewed in full. + if (casesShown === "partial") return { mode: "unavailable", reason: "partial" } + if (casesShown === "markdown") return { mode: "unavailable", reason: "unchecked" } // Plans made before the run id was recorded have nothing to fetch. if (!/^\d+$/.test(String(runId ?? ""))) return { mode: "unavailable", reason: "no-run" } // Without the fingerprint of the cases the plan showed, nothing ties the saved file to them. @@ -52,15 +55,136 @@ const plain = (s) => .replace(/@/g, "@\u200b") .replace(/:\/\//g, ":\u200b//") -// A GitHub comment holds 65,536 characters; past this, the plan lists titles only. -const PLAN_DETAIL_LIMIT = 45_000 +// Agent-written text that may span lines, shown in a code block: nothing in it is Markdown, +// a mention or a link. The fence is longer than any run of backticks inside. +const block = (s) => { + const text = String(s ?? "").replace(/\r\n?/g, "\n").replace(/^\n+|\s+$/g, "") + const longest = Math.max(2, ...(text.match(/`+/g) ?? []).map((run) => run.length)) + const fence = "`".repeat(longest + 1) + return `${fence}text\n${text}\n${fence}` +} +// Text inside , which is HTML, not Markdown. +const summaryText = (s) => + String(s ?? "") + .replace(/\s+/g, " ") + .trim() + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/@/g, "@\u200b") + .replace(/:\/\//g, ":\u200b//") + +const isText = (v) => typeof v === "string" +const isTextList = (v) => Array.isArray(v) && v.every(isText) +const isTextMap = (v) => Boolean(v) && typeof v === "object" && !Array.isArray(v) && Object.values(v).every(isText) +const isStepList = (v) => + Array.isArray(v) && + v.every((st) => st && typeof st === "object" && !Array.isArray(st) && Object.entries(st).every(([k, x]) => STEP_FIELDS.has(k) && (x === undefined || x === null || isText(x)))) +const textLine = (label) => (v) => `**${label}:** ${plain(v)}` +const textBlock = (label) => (v) => (/\n/.test(v) ? `**${label}:**\n\n${block(v)}\n` : `**${label}:** ${plain(v)}`) + +// A step's fields; any other key in a step is not shown, so a case carrying one is refused. +const STEP_FIELDS = new Set(["step", "result"]) + +/** + * Every field of a case in the cases file that the plan shows, in the order shown: the one list + * both the plan and the pre-save check use, so they can't drift. `id`, `title` and `folder` are + * shown in the case's heading; `render` is undefined for those. These are all the fields + * `tc tm add --from-file` sends (a BDD case's feature, scenario and background included). + */ +const CASE_FIELDS = { + id: { valid: isText }, + title: { valid: isText }, + folder: { valid: isText }, + format: { valid: isText, render: (v) => `**Format:** ${String(v).toLowerCase() === "bdd" ? "BDD" : plain(v)}` }, + type: { valid: isText, render: textLine("Type") }, + priority: { valid: isText, render: textLine("Priority") }, + tags: { valid: isTextList, render: (v) => `**Tags:** ${v.map(plain).join(", ")}` }, + description: { valid: isText, render: textBlock("Description") }, + preconditions: { valid: isText, render: textBlock("Before you start") }, + feature: { valid: isText, render: textBlock("Feature") }, + background: { valid: isText, render: textBlock("Background") }, + bddScenario: { valid: isText, render: textBlock("Scenario") }, + steps: { + valid: isStepList, + render: (v) => + [ + "**Steps:**", + "", + ...v.flatMap((st, n) => [ + `${n + 1}. ${plain(st?.step)}`, + ...(String(st?.result ?? "").trim() ? [` - Expected: ${plain(st.result)}`] : []), + ]), + ].join("\n"), + }, + references: { valid: isTextList, render: (v) => ["**References:**", "", ...v.map((r) => `- ${plain(r)}`)].join("\n") }, + customFields: { + valid: isTextMap, + render: (v) => ["**Custom fields:**", "", ...Object.entries(v).map(([k, x]) => `- ${plain(k)}: ${plain(x)}`)].join("\n"), + }, +} + +/** The case fields the plan shows. */ +export const RENDERED_CASE_FIELDS = Object.freeze(Object.keys(CASE_FIELDS)) + +const isEmpty = (v) => + v === undefined || + v === null || + (typeof v === "string" && v.trim() === "") || + (Array.isArray(v) && v.length === 0) || + (typeof v === "object" && !Array.isArray(v) && Object.keys(v).length === 0) + +/** + * The fields the plan can't show, as "T003: links" labels: a non-empty field not in + * RENDERED_CASE_FIELDS (an unknown or newer one), or one whose value isn't the shape the plan + * shows. Any of these means approving can't save the cases exactly as shown. + */ +export function unshownFields(data) { + const out = [] + const cases = Array.isArray(data?.cases) ? data.cases : [] + cases.forEach((c, i) => { + const name = isText(c?.id) && c.id.trim() ? c.id.trim() : `#${i + 1}` + if (!c || typeof c !== "object" || Array.isArray(c)) { + out.push(`${name}: not a test case`) + return + } + for (const [key, value] of Object.entries(c)) { + if (isEmpty(value)) continue + const field = CASE_FIELDS[key] + if (!field || !field.valid(value)) out.push(`${name}: ${key}`) + } + }) + return out +} + +/** One case, every field the cases file holds for it, in a collapsible block. */ +function renderCase(c, i) { + const id = isText(c.id) && c.id.trim() ? c.id : `#${i + 1}` + const heading = `${summaryText(id)} — ${summaryText(c.title) || "(untitled)"}` + const lines = [] + if (isText(c.folder) && c.folder.trim()) { + lines.push(`_Already in Test Management (${plain(c.folder)}): saving skips this case._`, "") + } + for (const [key, field] of Object.entries(CASE_FIELDS)) { + const value = c[key] + if (!field.render || isEmpty(value) || !field.valid(value)) continue + lines.push(field.render(value), "") + } + return [`
${heading}`, "", ...lines, "
"].join("\n") +} /** * The plan the reviewer reads, from the same cases file that approving saves, so what is - * shown and what is saved can't differ. Undefined when the file can't be read as cases. - * Returns { body, digest }; the digest goes in the plan's marker. + * shown and what is saved can't differ. Every field `tc tm add` sends is shown; fields it + * can't show are listed, and approving such a plan is refused before saving. + * + * `maxChars` is the room left in the comment. A plan that doesn't fit shows the cases that + * do, says so, and returns `complete: false`: it can't be approved for an exact save. + * + * Returns { body, digest, complete, unshown } or undefined when the file can't be read as cases. */ -export function renderPlannedCases(bytes) { +export function renderPlannedCases(bytes, { maxChars = 55_000 } = {}) { let data try { data = JSON.parse(Buffer.from(bytes ?? "").toString("utf8")) @@ -68,7 +192,7 @@ export function renderPlannedCases(bytes) { return undefined } if (!data || typeof data !== "object" || !Array.isArray(data.cases)) return undefined - const cases = data.cases.filter((c) => c && typeof c === "object") + const cases = data.cases const where = readCasesDestination(JSON.stringify(data)) const ids = (a, b) => [a, b].filter(Boolean).map(plain).join(", ") const head = where.error @@ -79,28 +203,36 @@ export function renderPlannedCases(bytes) { : "" }, once approved.` const count = `**${cases.length} test case${cases.length === 1 ? "" : "s"}:**` - const title = (c, i) => `${plain(c.id) || `#${i + 1}`} — ${plain(c.title) || "(untitled)"}` - const detailed = cases.map((c, i) => { - const lines = [`#### ${title(c, i)}`] - if (String(c.priority ?? "").trim()) lines.push(`Priority: ${plain(c.priority)}`) - if (String(c.preconditions ?? "").trim()) lines.push(`Before you start: ${plain(c.preconditions)}`) - const steps = Array.isArray(c.steps) ? c.steps : [] - steps.forEach((st, n) => { - lines.push(`${n + 1}. ${plain(st?.step ?? st?.description ?? st?.action)}`) - const expected = st?.result ?? st?.expected ?? st?.expectedResult - if (String(expected ?? "").trim()) lines.push(` - Expected: ${plain(expected)}`) - }) - return lines.join("\n") - }) - let list = detailed.join("\n\n") - if (list.length > PLAN_DETAIL_LIMIT) { - list = [ - "_Too long to show every step here; the titles are below and the full cases are in this run's result file._", - "", - ...cases.map((c, i) => `- ${title(c, i)}`), - ].join("\n") + const unshown = unshownFields(data) + const warning = unshown.length + ? `**Approving this plan can't save its cases exactly as shown:** some cases carry fields this plan can't show (${unshown + .slice(0, 20) + .map(plain) + .join(", ")}${unshown.length > 20 ? ", …" : ""}). Ask me to **generate** again.` + : "" + const top = [head, ...(warning ? ["", warning] : []), "", count, ""].join("\n") + const tooLarge = (shown) => + `**This plan is too large to show in full** (${shown} of ${cases.length} cases shown), so it can't be approved to save exactly as shown. Ask me to **generate** fewer cases.` + const rendered = cases.map((c, i) => (c && typeof c === "object" && !Array.isArray(c) ? renderCase(c, i) : `- #${i + 1}: not a test case`)) + const all = rendered.join("\n\n") + if (top.length + all.length <= maxChars) { + return { body: top + all, digest: casesDigest(bytes), complete: true, unshown } + } + // Room for the note, then as many whole cases as fit. + const room = maxChars - top.length - tooLarge(cases.length).length - 4 + const kept = [] + let used = 0 + for (const r of rendered) { + if (used + r.length + 2 > room) break + kept.push(r) + used += r.length + 2 + } + return { + body: [top + tooLarge(kept.length), "", kept.join("\n\n")].join("\n"), + digest: casesDigest(bytes), + complete: false, + unshown, } - return { body: [head, "", count, "", list].join("\n"), digest: casesDigest(bytes) } } /** @@ -229,6 +361,9 @@ const UNAVAILABLE_WHY = { "other-commit": "the saved cases are for a different commit than this plan", "no-digest": "this plan didn't record which cases it showed (an older version of Test Companion made it)", changed: "the saved cases are not the ones this plan showed", + partial: "this plan was too large to show every case, so not all of it was reviewed", + unchecked: "this plan couldn't be shown from its cases file, so what it showed can't be checked against what would be saved", + unshown: "some cases carry fields the plan couldn't show, so they weren't reviewed", "no-destination": "the plan recorded no Test Management destination, so there is nowhere to save its cases", "old-cli": `saving a plan's own cases needs Test Companion CLI ${CASES_DESTINATION_CLI} or later, and this run has an older one`, "old-cli-plan": `this plan was made with a Test Companion CLI older than ${CASES_DESTINATION_CLI}, which doesn't record where to save its cases`, @@ -244,6 +379,7 @@ export function renderCasesUnavailable(reason, detail) { const update = `Update the CLI: remove the action's \`cli-version\` pin, or pin ${CASES_DESTINATION_CLI} or later.` const next = { "lookup-failed": `${notInstead} Reply **go** to this plan again in a minute; it is still waiting.`, + partial: `${notInstead} Ask me to **generate** fewer cases, then approve the fresh plan.`, "no-destination": `${notInstead} Set \`tm_project\` and \`tm_folder\` for generate in the pipeline file. If they are already set, the plan was probably made with a CLI older than ${CASES_DESTINATION_CLI}: ${update} Then ask me to **generate** again and approve the fresh plan.`, "old-cli": `${notInstead} ${update} Then reply **go** to this plan again; it is still waiting.`, "old-cli-plan": `${notInstead} ${update} Then ask me to **generate** again and approve the fresh plan.`, diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index edf5bbb..d22fe0f 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -26,6 +26,7 @@ import { childEnvFor } from "./child-env.mjs" import { approvedSavePlan, readCasesDestination, + unshownFields, renderSaveInterrupted, tmAddSavedNothing, missingDestinationReason, @@ -60,6 +61,8 @@ import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail import { renderJobSummary, renderCheckSummary, + renderSummaryWithin, + COMMENT_LIMIT, renderComment, requestMarkerFor, chooseCommentAction, @@ -242,6 +245,11 @@ let approvedPlan /** Which failed run a build-reading job looked at, and what it left out. */ let buildNote = "" +/** A generate plan's room: the run's report above the cases is kept to this. */ +const PLAN_SUMMARY_LIMIT = 12_000 +/** The proposal's own text and marker, around the plan. */ +const PROPOSAL_OVERHEAD = 2_000 + /** That run's id, recorded in a plan so approving it reads the same run. */ let buildRunId = "" @@ -737,6 +745,7 @@ async function main() { replyText, runId: waiting?.runId, casesSha256: waiting?.casesSha256, + casesShown: waiting?.casesShown, }) if (save.mode === "exact" || save.mode === "unavailable") { saveApprovedCases(plan, save, approvalNote) @@ -1046,12 +1055,21 @@ async function main() { // file's fingerprint goes in the marker, so what the reviewer reads is exactly what is saved. let planBody = "" let casesSha256 = "" - const planned = - (plan.workflow ?? summary.workflow) === "generate" ? renderPlannedCases(readBytes(join(CASES_DIR, CASES_DATA_FILE))) : undefined + let casesShown = "" + const isGenerate = (plan.workflow ?? summary.workflow) === "generate" + const casesBytes = isGenerate ? readBytes(join(CASES_DIR, CASES_DATA_FILE)) : undefined + // The cases get most of the comment; the run's own report above them is kept short. + const summaryShown = casesBytes ? renderSummaryWithin(view, PLAN_SUMMARY_LIMIT) : renderCheckSummary(view) + const planned = casesBytes + ? renderPlannedCases(casesBytes, { maxChars: COMMENT_LIMIT - summaryShown.length - sizeNote.length - PROPOSAL_OVERHEAD }) + : undefined if (planned) { planBody = planned.body casesSha256 = planned.digest + if (!planned.complete) casesShown = "partial" } else { + // A generate plan shown from the agent's text can't be tied to the cases it would save. + if (isGenerate) casesShown = "markdown" try { planBody = readFileSync(RESULT_FILE, "utf8") } catch { @@ -1062,7 +1080,7 @@ async function main() { plan.context.pr, renderProposal( // Capped: an over-long comment fails to post. - renderProposalBody({ summary: renderCheckSummary(view), plan: planBody, sizeNote }), + renderProposalBody({ summary: summaryShown, plan: planBody, sizeNote }), sha, plan.commenter, // Record the job, so approving can't run a different one. @@ -1070,7 +1088,7 @@ async function main() { // And the run, so approving can fetch the cases it kept. env.GITHUB_RUN_ID, // And the failed run it read, and the cases it showed, so approving uses exactly those. - { build: buildRunId, cases: casesSha256 }, + { build: buildRunId, cases: casesSha256, shown: casesShown }, ), ) postCheck(outcome, view, sha, [], "in_progress") @@ -1773,6 +1791,15 @@ function saveExactly(plan, save, sha, base, finish, unavailable) { // Only the exact file the plan was shown from: its fingerprint is in our own plan comment. const bytes = readBytes(casesFile) if (bytes === undefined || casesDigest(bytes) !== save.casesSha256) return unavailable("changed") + // Every field that would be saved must be one the plan showed. + let data + try { + data = JSON.parse(bytes.toString("utf8")) + } catch { + return unavailable("unreadable") + } + const unshown = unshownFields(data) + if (unshown.length) return unavailable("unshown", unshown.slice(0, 10).join(", ")) // The destination comes from the approved file itself, so the cases go where the plan said. const where = readCasesDestination(bytes.toString("utf8")) if (where.error === "no-destination") return unavailable(missingDestinationReason(subject.cli), subject.cli || undefined) diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index 04d328c..8551355 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -216,6 +216,11 @@ export function renderCheckSummary(args) { return renderWithin(args, CHECK_SUMMARY_LIMIT, "") } +/** The same summary within a smaller limit, so a plan's cases get most of a comment's room. */ +export function renderSummaryWithin(args, limit) { + return renderWithin(args, limit, "") +} + /** Render within `limit`, cutting the agent's answer first so the table, links and marker survive. */ function renderWithin(args, limit, tail) { const body = renderJobSummary(args) @@ -284,26 +289,38 @@ export const PROPOSAL_WINDOW_MS = 24 * 60 * 60 * 1000 // `run` is the run that made the plan, so approving it can fetch the cases that run kept. // `build` is the failed run the plan looked at, so approving it fixes that run and no other. // `cases` is the sha256 of the cases file the plan showed, so approving it saves exactly that. -export function proposalMarkerFor(sha, requester, workflow, runId, { build, cases } = {}) { +// `shown` is "partial" when the plan was too large to show every case, or "markdown" when a +// generate plan couldn't be shown from its cases file: either way it can't be saved as approved. +export function proposalMarkerFor(sha, requester, workflow, runId, { build, cases, shown } = {}) { const by = String(requester ?? "").trim() const job = String(workflow ?? "").trim() const run = /^\d+$/.test(String(runId ?? "")) ? String(runId) : "" const failed = /^\d+$/.test(String(build ?? "")) ? String(build) : "" const digest = /^[0-9a-f]{64}$/.test(String(cases ?? "")) ? String(cases) : "" - return `` + const how = shown === "partial" || shown === "markdown" ? shown : "" + return `` } -// Groups: 1 = sha, 2 = requester, 3 = workflow, 4 = run id, 5 = failed run id, 6 = cases sha256. -// The sha is captured and compared, never built into the regex. Older markers lack 4 to 6. +// Groups: 1 = sha, 2 = requester, 3 = workflow, 4 = run id, 5 = failed run id, 6 = cases sha256, +// 7 = how the cases were shown. The sha is captured and compared, never built into the regex. +// Older markers lack 4 to 7. const MARKER = - // + // // The last marker, since ours ends the comment and anything earlier could be planted. const readMarker = (body) => { const all = [...String(body ?? "").matchAll(new RegExp(MARKER.source, "g"))] const found = all.at(-1) return found - ? { sha: found[1], requester: found[2], workflow: found[3], runId: found[4], buildRunId: found[5], casesSha256: found[6] } + ? { + sha: found[1], + requester: found[2], + workflow: found[3], + runId: found[4], + buildRunId: found[5], + casesSha256: found[6], + casesShown: found[7], + } : undefined } @@ -428,6 +445,7 @@ export function findProposal(comments, sha, now = Date.now(), us, opts = {}) { runId: mark?.runId, buildRunId: mark?.buildRunId, casesSha256: mark?.casesSha256, + casesShown: mark?.casesShown, body: newest.body ?? "", } } diff --git a/test-companion/test/approved-cases.test.mjs b/test-companion/test/approved-cases.test.mjs index 47ac593..2c4373b 100644 --- a/test-companion/test/approved-cases.test.mjs +++ b/test-companion/test/approved-cases.test.mjs @@ -18,6 +18,8 @@ import { CASES_DESTINATION_CLI, renderPlannedCases, casesDigest, + unshownFields, + RENDERED_CASE_FIELDS, CASES_ARTIFACT, browserstackLink, renderSaveInterrupted, @@ -258,7 +260,7 @@ describe("the wiring", () => { test("a plan attaches its cases, and approving one saves them with tm add instead of running again", () => { assert.match(main, /setOutput\("cases-dir", CASES_DIR\)/) - assert.match(main, /env\.GITHUB_RUN_ID,\n[^\n]*\n\s*\{ build: buildRunId, cases: casesSha256 \},\n\s*\),\n\s*\)/) + assert.match(main, /env\.GITHUB_RUN_ID,\n[^\n]*\n\s*\{ build: buildRunId, cases: casesSha256, shown: casesShown \},\n\s*\),\n\s*\)/) // No --project/--folder: tm add takes the destination from the approved file. assert.match(main, /\["tm", "add", "--from-file", casesFile, "--json"\]/) assert.doesNotMatch(main, /binding\?\.input|\.input\?\.tm_/) @@ -298,7 +300,10 @@ describe("approving saves exactly the cases the plan showed", () => { const shown = renderPlannedCases(file) assert.match(shown.body, /\*\*Will be saved to:\*\* github-app in rag\\_assignment \(PR-42, 7\), once approved\./) assert.match(shown.body, /\*\*2 test cases:\*\*/) - assert.match(shown.body, /#### T001 — Base delay\nPriority: high\nBefore you start: Module importable\n1\. Call f\\\(1\\\)\n {3}- Expected: 500/) + assert.match(shown.body, /
T001 — Base delay<\/summary>/) + assert.match(shown.body, /\*\*Priority:\*\* high/) + assert.match(shown.body, /\*\*Before you start:\*\* Module importable/) + assert.match(shown.body, /1\. Call f\\\(1\\\)\n {3}- Expected: 500/) assert.equal(shown.digest, casesDigest(file)) }) @@ -319,18 +324,6 @@ describe("approving saves exactly the cases the plan showed", () => { assert.match(renderPlannedCases(none).body, /Will be saved to:\*\* nowhere/) }) - test("a plan too long for one comment still lists every title", () => { - const big = Buffer.from( - JSON.stringify({ - destination: { project: "p", folder: "f" }, - cases: Array.from({ length: 300 }, (_, i) => ({ id: `T${i}`, title: `case ${i}`, steps: [{ step: "s".repeat(200), result: "r" }] })), - }), - ) - const shown = renderPlannedCases(big).body - assert.ok(shown.length < 60_000, String(shown.length)) - assert.match(shown, /- T299 — case 299/) - }) - test("an unreadable file shows no generated plan", () => { assert.equal(renderPlannedCases(Buffer.from("not json")), undefined) assert.equal(renderPlannedCases(Buffer.from("{}")), undefined) @@ -360,7 +353,8 @@ describe("approving saves exactly the cases the plan showed", () => { const check = fn.indexOf('casesDigest(bytes) !== save.casesSha256) return unavailable("changed")') assert.ok(check > 0, "no fingerprint check") assert.ok(check < fn.indexOf('"tm", "add"'), "the check must come before tm add") - assert.match(main, /renderPlannedCases\(readBytes\(join\(CASES_DIR, CASES_DATA_FILE\)\)\)/) + assert.match(main, /renderPlannedCases\(casesBytes, \{ maxChars: COMMENT_LIMIT - summaryShown\.length/) + assert.match(main, /readBytes\(join\(CASES_DIR, CASES_DATA_FILE\)\)/) }) }) @@ -487,3 +481,144 @@ describe("failure detail is kept, and kept out of the comment", () => { assert.match(fn, /logRedacted\("tc tm add", stderr \|\| r\.error\?\.message\)/) }) }) + +describe("the plan shows every field approving saves", () => { + const doc = (cases, extra = {}) => Buffer.from(JSON.stringify({ destination: { project: "rag", folder: "gh" }, cases, ...extra })) + + test("a BDD case shows its feature, background and scenario", () => { + const shown = renderPlannedCases( + doc([ + { + id: "T001", + title: "Login works", + format: "bdd", + feature: "Login", + background: "Given the app is open", + bddScenario: "Scenario: good password\n When I sign in\n Then I see my home page", + }, + ]), + ) + assert.equal(shown.complete, true) + assert.deepEqual(shown.unshown, []) + assert.match(shown.body, /\*\*Format:\*\* BDD/) + assert.match(shown.body, /\*\*Feature:\*\* Login/) + assert.match(shown.body, /\*\*Background:\*\* Given the app is open/) + assert.match(shown.body, /\*\*Scenario:\*\*\n\n```text\nScenario: good password\n {2}When I sign in\n {2}Then I see my home page\n```/) + }) + + test("description, type, tags, references and custom fields are all shown, as plain text", () => { + const shown = renderPlannedCases( + doc([ + { + id: "T002", + title: "Cap", + description: "Checks the cap\nSee @octocat", + type: "Functional", + tags: ["smoke", "retry"], + references: ["https://evil.test/x", "JIRA-1"], + customFields: { Team: "QA", Owner: "@mallory" }, + }, + ]), + ) + assert.deepEqual(shown.unshown, []) + assert.match(shown.body, /\*\*Description:\*\*\n\n```text\nChecks the cap\nSee @octocat\n```/, "in a code block: no mention") + assert.match(shown.body, /\*\*Type:\*\* Functional/) + assert.match(shown.body, /\*\*Tags:\*\* smoke, retry/) + assert.match(shown.body, /- https:\u200b\/\/evil\.test\/x/, "a reference is not a link") + assert.match(shown.body, /- JIRA-1/) + assert.match(shown.body, /- Team: QA/) + assert.match(shown.body, /- Owner: @\u200bmallory/) + }) + + test("a case already in Test Management is labelled, since saving skips it", () => { + assert.match(renderPlannedCases(doc([{ id: "TC-9", title: "Old", folder: "F-1" }])).body, /Already in Test Management \(F-1\): saving skips this case/) + }) + + test("an unknown field is listed in the plan, and refused before saving", () => { + const shown = renderPlannedCases(doc([{ id: "T003", title: "x", links: ["https://evil.test"] }])) + assert.deepEqual(shown.unshown, ["T003: links"]) + assert.match(shown.body, /can't save its cases exactly as shown/) + const text = renderCasesUnavailable("unshown", "T003: links") + assert.match(text, /fields the plan couldn't show/) + const main = readFileSync(new URL("../src/main.mjs", import.meta.url), "utf8") + const fn = main.slice(main.indexOf("function saveExactly("), main.indexOf("function pickBinding(")) + const check = fn.indexOf('if (unshown.length) return unavailable("unshown"') + assert.ok(check > fn.indexOf("casesDigest(bytes)"), "after the fingerprint check") + assert.ok(check < fn.indexOf('"tm", "add"'), "before tm add") + }) + + test("a field of the wrong shape counts as not shown", () => { + assert.deepEqual(unshownFields({ cases: [{ id: "T1", title: "x", tags: "smoke" }] }), ["T1: tags"]) + assert.deepEqual(unshownFields({ cases: [{ id: "T1", title: "x", customFields: { a: { nested: 1 } } }] }), ["T1: customFields"]) + assert.deepEqual(unshownFields({ cases: [{ id: "T1", title: "x", steps: [{ step: "s", note: "hidden" }] }] }), ["T1: steps"]) + assert.deepEqual(unshownFields({ cases: ["not a case"] }), ["#1: not a test case"]) + }) + + test("an empty unknown field doesn't block", () => { + assert.deepEqual(unshownFields({ cases: [{ id: "T1", title: "x", links: [], extra: "", other: null }] }), []) + }) + + test("the renderer and the check share one list of fields", () => { + const full = { + id: "T1", + title: "t", + folder: "F", + format: "bdd", + type: "ty", + priority: "p", + tags: ["tg"], + description: "de", + preconditions: "pr", + feature: "fe", + background: "bg", + bddScenario: "sc", + steps: [{ step: "st", result: "re" }], + references: ["rf"], + customFields: { k: "v" }, + } + assert.deepEqual(Object.keys(full).sort(), [...RENDERED_CASE_FIELDS].sort(), "the list is exactly these fields") + assert.deepEqual(unshownFields({ cases: [full] }), []) + const body = renderPlannedCases(doc([full])).body + for (const value of ["T1", "t", "F", "BDD", "ty", "p", "tg", "de", "pr", "fe", "bg", "sc", "st", "re", "rf", "k: v"]) { + assert.ok(body.includes(value), `${value} is not shown`) + } + for (const key of RENDERED_CASE_FIELDS) { + assert.deepEqual(unshownFields({ cases: [{ title: "x", [key + "X"]: "v" }] }), [`#1: ${key}X`]) + } + }) + + test("a plan too large for one comment shows what fits, says so, and can't be approved", () => { + const cases = Array.from({ length: 300 }, (_, i) => ({ id: `T${i}`, title: `case ${i}`, steps: [{ step: "s".repeat(300), result: "r" }] })) + const shown = renderPlannedCases(doc(cases), { maxChars: 20_000 }) + assert.equal(shown.complete, false) + assert.ok(shown.body.length <= 20_000, String(shown.body.length)) + assert.match(shown.body, /too large to show in full/) + assert.match(shown.body, /can't be approved to save exactly as shown/) + const marker = proposalMarkerFor("abc", "o", "generate", "1", { cases: shown.digest, shown: "partial" }) + assert.match(marker, / shown=partial -->$/) + const waiting = findProposal( + [{ id: 5, user: { login: "browserstack[bot]", type: "Bot" }, created_at: new Date().toISOString(), body: `p\n\n${marker}` }], + "abc", + Date.now(), + "browserstack[bot]", + ) + assert.equal(waiting.casesShown, "partial") + assert.deepEqual( + approvedSavePlan({ workflow: "generate", replyText: "go", runId: "1", casesSha256: shown.digest, casesShown: waiting.casesShown }), + { mode: "unavailable", reason: "partial" }, + ) + assert.match(renderCasesUnavailable("partial"), /too large to show every case/) + }) + + test("a plan shown from the agent's text can't be approved for an exact save, and says why", () => { + assert.deepEqual(approvedSavePlan({ workflow: "generate", replyText: "go", runId: "1", casesShown: "markdown" }), { + mode: "unavailable", + reason: "unchecked", + }) + const text = renderCasesUnavailable("unchecked") + assert.match(text, /couldn't be shown from its cases file/) + assert.doesNotMatch(text, /older version/) + const main = readFileSync(new URL("../src/main.mjs", import.meta.url), "utf8") + assert.match(main, /if \(isGenerate\) casesShown = "markdown"/) + }) +}) From 52da1a1e256089a45b3e1bc2cfc18dbc22b33519 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 16 Nov 2026 15:32:51 +0530 Subject: [PATCH 28/31] test-companion: a staged rename is committed as a move, not a copy `git status` reports a staged rename as `R old -> new`, and only the new path was kept. The commit is built from the base, so after `git mv a b` the pushed commit had both a and b. The old path of a rename is now included, so it is recorded as deleted. A copy (C) still keeps its source. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/push-work.mjs | 14 ++++++++++---- test-companion/test/push-work.test.mjs | 26 ++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index e7d852f..1f793f8 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -102,11 +102,17 @@ export function changedPaths(porcelain) { return String(porcelain ?? "") .split("\n") .filter((line) => line.trim()) - .map(stripStatus) - .filter(Boolean) - .map((path) => { + .flatMap((line) => { + const path = stripStatus(line) + if (!path) return [] const arrow = path.indexOf(" -> ") - return arrow === -1 ? path : path.slice(arrow + 4) + if (arrow === -1) return [path] + // A rename (R) moved a file: its old path is gone and must be recorded as deleted, + // or a commit started from the base would keep both. A copy (C) leaves the old one. + const renamed = line.trimStart().startsWith("R") + const from = path.slice(0, arrow) + const to = path.slice(arrow + 4) + return renamed ? [from, to] : [to] }) .map(unquote) } diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index db66d6e..26fafef 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -83,7 +83,8 @@ describe("changedPaths — reading git status", () => { }) test("a rename reports where the file landed, not where it came from", () => { - assert.deepEqual(changedPaths("R old/a.js -> new/a.js"), ["new/a.js"]) + // Both: the old path is recorded as deleted, so the commit doesn't keep a copy. + assert.deepEqual(changedPaths("R old/a.js -> new/a.js"), ["old/a.js", "new/a.js"]) }) test("a quoted path with a space is unquoted", () => { @@ -275,7 +276,7 @@ describe("changedPaths — paths git had to escape", () => { }) test("a rename to an escaped path reports the decoded destination", () => { - assert.deepEqual(changedPaths('R old.js -> "caf\\303\\251.spec.ts"'), ["café.spec.ts"]) + assert.deepEqual(changedPaths('R old.js -> "caf\\303\\251.spec.ts"'), ["old.js", "café.spec.ts"]) }) }) @@ -1260,3 +1261,24 @@ describe("a hand-back is never built on the pull_request merge commit", () => { assert.ok(refuse < fn.indexOf('git(["push"'), "refused before any push") }) }) + +describe("a staged rename is a move, not a copy", () => { + const identity = { name: "Test Companion", email: BOT_EMAIL } + + test("a copy keeps its source", () => { + assert.deepEqual(changedPaths("C a.js -> b.js"), ["b.js"]) + }) + + test("after `git mv a b`, the commit has b and not a", () => { + const { dir, git, tip } = repoWithBranch({ "a.ts": "new\n" }) + assert.ok(git(["mv", "a.ts", "b.ts"]).ok) + const status = git(["-c", "core.quotePath=false", "status", "--porcelain", "--untracked-files=all"]) + const paths = changedPaths(status.out) + assert.deepEqual(paths, ["a.ts", "b.ts"]) + const out = commitPaths({ git, base: tip, paths, identity, message: "m", indexFile: join(dir, "scratch") }) + assert.ok(out.sha, JSON.stringify(out)) + assert.equal(git(["cat-file", "-e", `${out.sha}:a.ts`]).ok, false, "the old path was kept") + assert.equal(git(["show", `${out.sha}:b.ts`]).out, "new") + rmSync(dir, { recursive: true, force: true }) + }) +}) From 9d0f22bb06a55dd6d9325bfddf09cefcc2fd6c03 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 16 Nov 2026 15:34:07 +0530 Subject: [PATCH 29/31] test-companion: say a save saved nothing when that's certain, and only then When `tc tm add` stopped without a report, the reply said cases "may already be saved" even when it had failed before saving anything, such as a sign-in, connection or folder lookup error. The CLI prints its report as soon as it has started creating cases, so a run that exited by itself with an error and no report failed before saving, and one that never started saved nothing. Those now say nothing was saved and leave the plan approvable. A kill, a timeout, a crash, or a clean exit with no report may have stopped mid-save: those still use the plan up and say to check Test Management first, so a second `go` can't save twice. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/approved-cases.mjs | 23 +++++++++++++--- test-companion/src/main.mjs | 11 ++++---- test-companion/test/approved-cases.test.mjs | 30 +++++++++++++++++++-- 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/test-companion/src/approved-cases.mjs b/test-companion/src/approved-cases.mjs index 2eff128..7762dda 100644 --- a/test-companion/src/approved-cases.mjs +++ b/test-companion/src/approved-cases.mjs @@ -367,7 +367,7 @@ const UNAVAILABLE_WHY = { "no-destination": "the plan recorded no Test Management destination, so there is nowhere to save its cases", "old-cli": `saving a plan's own cases needs Test Companion CLI ${CASES_DESTINATION_CLI} or later, and this run has an older one`, "old-cli-plan": `this plan was made with a Test Companion CLI older than ${CASES_DESTINATION_CLI}, which doesn't record where to save its cases`, - "save-failed": "Test Management refused the cases before any were saved", + "save-failed": "the save stopped before any case was saved", "lookup-failed": "I couldn't look up the plan's saved cases on GitHub just now", } @@ -480,13 +480,30 @@ export function renderSaveInterrupted({ project, folder } = {}, why) { } /** - * Whether `tc tm add` said, in its own words, that it saved nothing. Only then is the plan - * left approvable; any other stop may have saved some cases. + * Whether `tc tm add` said, in its own words, that it saved nothing. */ export function tmAddSavedNothing(stderr) { return /\bnothing was saved\b/i.test(String(stderr ?? "")) } +/** + * How a `tc tm add` that printed no report stopped: "nothing-saved" or "uncertain". Only + * "nothing-saved" leaves the plan approvable; anything else may have saved some cases. + * + * The CLI prints its report as soon as it has started creating cases. So a process that ran + * and exited by itself with an error, and printed no report, failed before saving (reading + * the file, signing in, finding the project or folder). A process that never started saved + * nothing too. A kill, a timeout, a crash (a Go panic) or a clean exit with no report could + * have stopped mid-save. + */ +export function tmAddStop({ status, signal, errorCode, stderr } = {}) { + if (tmAddSavedNothing(stderr)) return "nothing-saved" + if (errorCode === "ENOENT" || errorCode === "EACCES") return "nothing-saved" + if (signal || errorCode) return "uncertain" + if (/\bpanic:|\bgoroutine \d+/.test(String(stderr ?? ""))) return "uncertain" + return Number.isInteger(status) && status !== 0 ? "nothing-saved" : "uncertain" +} + /** A note for the reply when a go with edits had to generate again. */ export const EDITED_GO_NOTE = "Your go included changes, and saving the plan as it was can't apply them, so the cases were generated again with your changes as guidance. Check the list below; it may differ from the plan." diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index d22fe0f..c0100d1 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -28,7 +28,7 @@ import { readCasesDestination, unshownFields, renderSaveInterrupted, - tmAddSavedNothing, + tmAddStop, missingDestinationReason, renderPlannedCases, casesDigest, @@ -1817,10 +1817,11 @@ function saveExactly(plan, save, sha, base, finish, unavailable) { logRedacted("tc tm add", stderr || r.error?.message) const said = stderr.trim().split("\n").filter(Boolean).at(-1) const why = redactSecrets(said || (r.signal ? `stopped by ${r.signal}` : r.error?.message) || "", secretsNow()) - // Only the CLI's own "nothing was saved" leaves the plan approvable. A save that was - // killed, timed out or printed no report may have saved some, so the plan is used up: - // a second go could save them twice. - if (!r.error && !r.signal && tmAddSavedNothing(stderr)) return unavailable("save-failed", why) + // Approvable again only when nothing can have been saved. A save that was killed, timed + // out or crashed may have saved some, so the plan is used up: a second go could save them twice. + if (tmAddStop({ status: r.status, signal: r.signal, errorCode: r.error?.code, stderr }) === "nothing-saved") { + return unavailable("save-failed", why) + } return finish({ ...base, outcome: OUTCOMES.RUN_BROKE, answer: renderSaveInterrupted(where, why) }, true) } if (report.failed.length > 0) logRedacted("tc tm add", stderr) diff --git a/test-companion/test/approved-cases.test.mjs b/test-companion/test/approved-cases.test.mjs index 2c4373b..3e6d0c3 100644 --- a/test-companion/test/approved-cases.test.mjs +++ b/test-companion/test/approved-cases.test.mjs @@ -24,6 +24,7 @@ import { browserstackLink, renderSaveInterrupted, tmAddSavedNothing, + tmAddStop, } from "../src/approved-cases.mjs" import { OUTCOMES } from "../src/outcome.mjs" import { isApprovalOnly } from "../src/intent.mjs" @@ -221,7 +222,7 @@ describe("what the reply says", () => { const text = renderCasesUnavailable("expired") assert.match(text, /I did not save anything: the plan's saved cases have expired\./) assert.match(text, /I have not generated a new set/) - assert.match(renderCasesUnavailable("save-failed", "folder not found"), /before any were saved \(folder not found\)\./) + assert.match(renderCasesUnavailable("save-failed", "folder not found"), /before any case was saved \(folder not found\)\./) }) }) @@ -454,7 +455,7 @@ describe("a save that stopped without a report uses the plan up", () => { test("the wiring: killed, timed out or no report is a used-up Run broke; only 'nothing was saved' stays approvable", () => { const main = readFileSync(new URL("../src/main.mjs", import.meta.url), "utf8") const fn = main.slice(main.indexOf("function saveExactly("), main.indexOf("function pickBinding(")) - assert.match(fn, /if \(!r\.error && !r\.signal && tmAddSavedNothing\(stderr\)\) return unavailable\("save-failed", why\)/) + assert.match(fn, /if \(tmAddStop\(\{ status: r\.status, signal: r\.signal, errorCode: r\.error\?\.code, stderr \}\) === "nothing-saved"\) \{\n\s*return unavailable\("save-failed", why\)/) assert.match(fn, /return finish\(\{ \.\.\.base, outcome: OUTCOMES\.RUN_BROKE, answer: renderSaveInterrupted\(where, why\) \}, true\)/) }) }) @@ -622,3 +623,28 @@ describe("the plan shows every field approving saves", () => { assert.match(main, /if \(isGenerate\) casesShown = "markdown"/) }) }) + +describe("a save that stopped says nothing was saved only when that's certain", () => { + test("failing before saving (sign-in, connection, finding the folder) means nothing was saved", () => { + assert.equal(tmAddStop({ status: 1, stderr: "Error: not signed in" }), "nothing-saved") + assert.equal(tmAddStop({ status: 1, stderr: "Error: folder \"x\" not found in project PR-1" }), "nothing-saved") + assert.equal(tmAddStop({ status: 1, stderr: "Error: nothing was saved: this file records no Test Management destination" }), "nothing-saved") + }) + + test("a CLI that never started saved nothing", () => { + assert.equal(tmAddStop({ status: null, errorCode: "ENOENT" }), "nothing-saved") + }) + + test("a kill, a timeout, a crash or a clean exit with no report may have saved some", () => { + assert.equal(tmAddStop({ status: null, signal: "SIGTERM", errorCode: "ETIMEDOUT" }), "uncertain") + assert.equal(tmAddStop({ status: null, signal: "SIGKILL" }), "uncertain") + assert.equal(tmAddStop({ status: 2, stderr: "panic: runtime error\n\ngoroutine 1 [running]:" }), "uncertain") + assert.equal(tmAddStop({ status: 0, stderr: "" }), "uncertain") + }) + + test("the wording matches", () => { + assert.match(renderCasesUnavailable("save-failed", "not signed in"), /the save stopped before any case was saved \(not signed in\)/) + assert.doesNotMatch(renderCasesUnavailable("save-failed"), /may already/) + assert.match(renderSaveInterrupted({ project: "p", folder: "f" }, "stopped by SIGTERM"), /may already be in f in p/) + }) +}) From 095fccbc6488ed25c5f3ad06bc40f73d970ed6d0 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 16 Nov 2026 15:37:55 +0530 Subject: [PATCH 30/31] test-companion: check the pushed tree against the base as listed before the run The agent can write .git. Git checks the hash of every object it reads, so a forged loose or packed object is refused by git itself. But some files are trusted without that check, and the base the Action builds on is read after the agent ran: - Before the agent runs, the Action now lists the base's whole tree and reads .git/objects/info/alternates and http-alternates, keeping both in its own memory, which the agent can't reach. - Before pushing a saved commit or a hand-back pull request, it refuses if the alternates files changed, and compares the new commit's whole tree with that listing: every path that differs must be one the Action chose (for a saved commit) and one it would ever commit. - Its git calls no longer use the commit-graph or multi-pack-index files, which git reads without checking the objects (core.commitGraph=false, core.multiPackIndex=false). Variables that point git at other objects were already never passed on. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/main.mjs | 48 ++++++++++++++ test-companion/src/push-work.mjs | 44 +++++++++++++ test-companion/test/push-work.test.mjs | 86 +++++++++++++++++++++++++- 3 files changed, 177 insertions(+), 1 deletion(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index c0100d1..2138d8b 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -105,6 +105,8 @@ import { isRequestedByPerson, leaveOut, commitPaths, + parseLsTree, + checkTreeAgainstBase, leaveOutMessage, listFiles, renderPushNote, @@ -245,6 +247,32 @@ let approvedPlan /** Which failed run a build-reading job looked at, and what it left out. */ let buildNote = "" +/** The base's whole tree, listed before the agent ran. */ +let baseListing = new Map() +/** Where git would look for borrowed objects, and what those files said before the agent ran. */ +const ALTERNATES_FILES = ["objects/info/alternates", "objects/info/http-alternates"] +let alternatesPaths = [] +let alternatesBefore = "" + +function readAlternates(paths) { + return paths + .map((p) => { + try { + return readFileSync(p, "utf8") + } catch { + return "" + } + }) + .join("\0") +} + +/** Why git's object store can't be trusted any more, or undefined. */ +function alternatesChanged() { + return readAlternates(alternatesPaths) === alternatesBefore + ? undefined + : "the run changed where git borrows objects from (.git/objects/info/alternates), so what git reads can't be trusted" +} + /** A generate plan's room: the run's report above the cases is kept to this. */ const PLAN_SUMMARY_LIMIT = 12_000 /** The proposal's own text and marker, around the plan. */ @@ -940,6 +968,11 @@ async function main() { // Where HEAD was before the agent touched anything. headBeforeRun = git(["rev-parse", "HEAD"]).out + // What the base holds, and where git may borrow objects from, before the agent can write .git. + // Kept in this process, which the agent can't reach. + baseListing = headBeforeRun ? parseLsTree(git(["ls-tree", "-r", "-z", "--full-tree", headBeforeRun]).raw) : new Map() + alternatesPaths = ALTERNATES_FILES.map((f) => git(["rev-parse", "--git-path", f]).out).filter(Boolean) + alternatesBefore = readAlternates(alternatesPaths) notice(`tc ${args.join(" ")}`) // Remove stale files so a run that produces nothing can't report an earlier run's work. for (const f of [SUMMARY_FILE, RESULT_FILE, `${RESULT_FILE}.json`, CASES_DIR]) { @@ -1163,6 +1196,8 @@ function saveWork(saving, plan) { "changes are in the job's workspace only.", ].join(" ") } + const borrowed = alternatesChanged() + if (borrowed) return `**Could not save this work to \`${branch}\`:** ${borrowed}. Nothing was committed or pushed.` // quotePath=false keeps non-ASCII paths as-is; -uall lists new directories file by file. const status = git(["-c", "core.quotePath=false", "status", "--porcelain", "--untracked-files=all"]) if (!status.ok) return undefined @@ -1197,6 +1232,12 @@ function saveWork(saving, plan) { if (!made.sha) { return renderPushNote({ branch, files: push, skipped: skip, error: made.error || "nothing to commit" }) } + // The agent can write .git/objects, so git's view of the base can be forged. The new commit + // must differ from the base as listed before the run only in the files chosen above. + const forged = checkTreeAgainstBase(baseListing, parseLsTree(git(["ls-tree", "-r", "-z", "--full-tree", made.sha]).raw), { + allowed: push, + }) + if (forged) return `**Could not save this work to \`${branch}\`:** ${forged}. Nothing was pushed.` // Push to a URL built from GITHUB_REPOSITORY, not `origin`: an insteadOf rewrite in // .git/config could send the token elsewhere. @@ -1380,6 +1421,8 @@ function openHandedOffPr(path, plan) { if (unexpected.length) { return `**Did not open a pull request:** the repository's git config carries ${listFiles(unexpected)}, which \`actions/checkout\` does not write.` } + const borrowed = alternatesChanged() + if (borrowed) return `**Did not open a pull request:** ${borrowed}.` const tip = git(["rev-parse", "--verify", "--quiet", `refs/heads/${req.branch}^{commit}`]) if (!tip.ok || !tip.out) return `**Did not open a pull request:** the run did not leave a \`${req.branch}\` branch.` // The squash goes onto the commit the run started on, and only if the agent's branch builds @@ -1432,6 +1475,11 @@ function openHandedOffPr(path, plan) { rmSync(indexFile, { force: true }) } if (left.error) return `**Could not open the pull request:** ${left.error}.` + // Checked against the base as listed before the run, not as git reads it now. + if (left.sha) { + const forged = checkTreeAgainstBase(baseListing, parseLsTree(git(["ls-tree", "-r", "-z", "--full-tree", left.sha]).raw)) + if (forged) return `**Did not open a pull request:** ${forged}.` + } if (left.empty) { return dropped.length ? `**Did not open a pull request:** the run's only changes were files Test Companion never commits: ${listFiles(dropped)}.` diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index 1f793f8..79edbbe 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -409,6 +409,12 @@ export const GIT_SAFE_ARGS = Object.freeze([ // which callers read as an error. "-c", "advice.graftFileDeprecated=false", + // The agent can write .git/objects/info/commit-graph and the multi-pack-index. Git reads a + // commit's parents and tree from them without checking the objects, so don't use them. + "-c", + "core.commitGraph=false", + "-c", + "core.multiPackIndex=false", ]) /** @@ -514,6 +520,44 @@ export function commitPaths({ git, base, paths, identity, message, indexFile }) return commit.ok && commit.out ? { sha: commit.out } : { error: commit.err || "could not commit" } } +/** + * `git ls-tree -r -z` output as path → "mode type id". Used to compare a commit's whole tree + * with the base's, as listed before the agent ran. + */ +export function parseLsTree(raw) { + const out = new Map() + for (const entry of String(raw ?? "").split("\0")) { + const tab = entry.indexOf("\t") + if (tab > 0) out.set(entry.slice(tab + 1), entry.slice(0, tab)) + } + return out +} + +/** Paths whose entry differs between two listings (added, removed or changed), sorted. */ +export function treeChanges(before, after) { + const paths = new Set([...before.keys(), ...after.keys()]) + return [...paths].filter((p) => before.get(p) !== after.get(p)).sort() +} + +/** + * Why a commit about to be pushed can't be trusted, or undefined. `before` is the base's + * listing taken before the agent ran; `after` is the new commit's. The agent can write + * .git/objects, so git's own view of the base can be forged: comparing against the listing + * taken earlier catches a changed base tree or objects fed in from elsewhere. Every path that + * differs must be one the Action chose (`allowed`, when given) and one it would ever commit. + */ +export function checkTreeAgainstBase(before, after, { allowed } = {}) { + if (!before || before.size === 0) return "the commit the run started on couldn't be listed before the run" + const changed = treeChanges(before, after) + const chosen = allowed ? new Set(allowed) : undefined + const unchosen = chosen ? changed.filter((p) => !chosen.has(p)) : [] + if (unchosen.length) return `it would also change files nobody chose: ${listFiles(unchosen)}` + const isLink = (p) => String(after.get(p) ?? "").startsWith("120000 ") + const { skip } = splitPushable(changed, { isSymlink: isLink }) + if (skip.length) return `it would change files Test Companion never commits: ${listFiles(skip)}` + return undefined +} + // Agent-chosen text on one line, so it can't add a trailer such as Co-authored-by. const oneLine = (s) => String(s ?? "").replace(/[\x00-\x1f\x7f]+/g, " ").trim() diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index 26fafef..03411f0 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -1,7 +1,7 @@ import { test, describe } from "node:test" import assert from "node:assert/strict" import { execFileSync, spawnSync } from "node:child_process" -import { mkdtempSync, mkdirSync, rmSync, readdirSync, readFileSync, writeFileSync, symlinkSync, unlinkSync, existsSync } from "node:fs" +import { mkdtempSync, mkdirSync, rmSync, readdirSync, readFileSync, writeFileSync, symlinkSync, unlinkSync, existsSync, copyFileSync, chmodSync } from "node:fs" import { tmpdir } from "node:os" import { dirname, join } from "node:path" import { @@ -29,6 +29,9 @@ import { savingFor, leaveOut, commitPaths, + parseLsTree, + treeChanges, + checkTreeAgainstBase, startedOnMergeCommit, checkHandoffDescends, REWROTE_HISTORY, @@ -1282,3 +1285,84 @@ describe("a staged rename is a move, not a copy", () => { rmSync(dir, { recursive: true, force: true }) }) }) + +describe("the base is checked against a listing taken before the agent ran", () => { + const identity = { name: "Test Companion", email: BOT_EMAIL } + const ls = (git, rev) => parseLsTree(spawnSync("git", ["ls-tree", "-r", "-z", "--full-tree", rev], { cwd: git.dir, encoding: "utf8" }).stdout) + + test("listings compare path by path", () => { + const before = parseLsTree("100644 blob aaa\ta.ts\x00100644 blob bbb\tb.ts\0") + const after = parseLsTree("100644 blob aaa\ta.ts\x00100644 blob ccc\tb.ts\x00120000 blob ddd\tlink\0") + assert.deepEqual(treeChanges(before, after), ["b.ts", "link"]) + assert.match(checkTreeAgainstBase(before, after, { allowed: ["b.ts"] }), /nobody chose: `link`/) + assert.match(checkTreeAgainstBase(before, after, { allowed: ["b.ts", "link"] }), /never commits: `link`/) + assert.equal(checkTreeAgainstBase(before, parseLsTree("100644 blob aaa\ta.ts\x00100644 blob ccc\tb.ts\0"), { allowed: ["b.ts"] }), undefined) + assert.match(checkTreeAgainstBase(new Map(), after), /couldn't be listed/) + }) + + test("git itself refuses a forged object, loose or packed", () => { + const { dir, git, tip } = repoWithBranch({ "a.ts": "new\n" }) + const tree = git(["rev-parse", `${tip}^{tree}`]).out + const file = join(dir, ".git/objects", tree.slice(0, 2), tree.slice(2)) + // Swap the base tree's object file for another tree's. + const other = git(["mktree"], {}, "").out + chmodSync(file, 0o644) + copyFileSync(join(dir, ".git/objects", other.slice(0, 2), other.slice(2)), file) + const r = git(["ls-tree", "-r", tip]) + assert.equal(r.ok, false) + assert.match(r.err, /hash mismatch/) + rmSync(dir, { recursive: true, force: true }) + }) + + test("a commit built on a base other than the one listed is refused", () => { + // What a forged read of the base would produce: a commit whose tree differs from the + // listed base in more than the chosen files. + const { dir, git, base, tip } = repoWithBranch({ "a.ts": "new\n", ".github/actions/x/action.yml": "runs: evil\n" }) + git.dir = dir + const listedBase = ls(git, base) + writeFileSync(join(dir, "a.ts"), "newer\n") + const out = commitPaths({ git, base: tip, paths: ["a.ts"], identity, message: "m", indexFile: join(dir, "scratch") }) + assert.ok(out.sha, JSON.stringify(out)) + assert.match(checkTreeAgainstBase(listedBase, ls(git, out.sha), { allowed: ["a.ts"] }), /nobody chose: `\.github\/actions\/x\/action\.yml`/) + rmSync(dir, { recursive: true, force: true }) + }) + + test("commit-graph and multi-pack-index files are not used", () => { + const settings = GIT_SAFE_ARGS.filter((a) => a !== "-c") + assert.ok(settings.includes("core.commitGraph=false")) + assert.ok(settings.includes("core.multiPackIndex=false")) + }) + + test("git never inherits a variable that points it at other objects", () => { + const got = gitEnvFrom({ GIT_ALTERNATE_OBJECT_DIRECTORIES: "/tmp/evil", GIT_OBJECT_DIRECTORY: "/tmp/evil", GIT_DIR: "/tmp/evil", GIT_INDEX_FILE: "/tmp/evil" }) + for (const k of ["GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_OBJECT_DIRECTORY", "GIT_DIR", "GIT_INDEX_FILE"]) assert.equal(got[k], undefined, k) + }) + + test("an alternates file feeds git objects from another folder, which is why it's checked", () => { + const { dir, git } = repoWithBranch({ "a.ts": "new\n" }) + const other = mkdtempSync(join(tmpdir(), "tc-alt-")) + spawnSync("git", ["init", "-q"], { cwd: other }) + const foreign = spawnSync("git", ["hash-object", "-w", "--stdin"], { cwd: other, encoding: "utf8", input: "foreign\n" }).stdout.trim() + assert.equal(git(["cat-file", "-e", foreign]).ok, false) + mkdirSync(join(dir, ".git/objects/info"), { recursive: true }) + writeFileSync(join(dir, ".git/objects/info/alternates"), `${join(other, ".git/objects")}\n`) + assert.equal(git(["cat-file", "-e", foreign]).ok, true) + rmSync(dir, { recursive: true, force: true }) + rmSync(other, { recursive: true, force: true }) + }) + + test("the wiring: listed before the agent, alternates and listings checked before anything is pushed", () => { + const main = readFileSync(new URL("../src/main.mjs", import.meta.url), "utf8") + const start = main.indexOf('headBeforeRun = git(["rev-parse", "HEAD"]).out') + assert.ok(main.indexOf("baseListing = headBeforeRun ? parseLsTree(", start) > start) + assert.ok(main.indexOf("alternatesBefore = readAlternates(", start) > start) + assert.ok(main.indexOf("baseListing = headBeforeRun", start) < main.indexOf("runLockedDown(", start), "listed before the agent runs") + assert.match(main, /const ALTERNATES_FILES = \["objects\/info\/alternates", "objects\/info\/http-alternates"\]/) + const save = main.slice(main.indexOf("function saveWork("), main.indexOf("function isolateAgent(")) + assert.ok(save.indexOf("alternatesChanged()") < save.indexOf('"status", "--porcelain"')) + assert.ok(save.indexOf("checkTreeAgainstBase(baseListing") < save.indexOf('git(["push"')) + const hand = main.slice(main.indexOf("function openHandedOffPr("), main.indexOf("function openHandedOffPr(") + 7000) + assert.ok(hand.indexOf("alternatesChanged()") < hand.indexOf("refs/heads/${req.branch}^{commit}")) + assert.ok(hand.indexOf("checkTreeAgainstBase(baseListing") < hand.indexOf('git(["push"')) + }) +}) From f2cb285d6eb83d5fa8be1d491a2b61096e7918ec Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 16 Nov 2026 15:38:17 +0530 Subject: [PATCH 31/31] test-companion: use a neutral placeholder reference in a test Co-Authored-By: Claude Opus 5.5 --- test-companion/test/approved-cases.test.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-companion/test/approved-cases.test.mjs b/test-companion/test/approved-cases.test.mjs index 3e6d0c3..253542c 100644 --- a/test-companion/test/approved-cases.test.mjs +++ b/test-companion/test/approved-cases.test.mjs @@ -516,7 +516,7 @@ describe("the plan shows every field approving saves", () => { description: "Checks the cap\nSee @octocat", type: "Functional", tags: ["smoke", "retry"], - references: ["https://evil.test/x", "JIRA-1"], + references: ["https://evil.test/x", "ISSUE-1"], customFields: { Team: "QA", Owner: "@mallory" }, }, ]), @@ -526,7 +526,7 @@ describe("the plan shows every field approving saves", () => { assert.match(shown.body, /\*\*Type:\*\* Functional/) assert.match(shown.body, /\*\*Tags:\*\* smoke, retry/) assert.match(shown.body, /- https:\u200b\/\/evil\.test\/x/, "a reference is not a link") - assert.match(shown.body, /- JIRA-1/) + assert.match(shown.body, /- ISSUE-1/) assert.match(shown.body, /- Team: QA/) assert.match(shown.body, /- Owner: @\u200bmallory/) })