Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion agents/frontend-triage/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ handler registry far wider than the tools this agent was given.

- `add_comment_hook` — one comment, public, on the bug being triaged.

That is the whole list, because a comment is the only thing this agent can write.
That is the whole list, because a comment is the only thing the model can write.
It has no tool that changes a bug's fields: `severity` was the one field a ruleset
directed it to set, and that is now a suggestion at the end of the comment for a
human to apply, so `bugzilla.update_bug` left `ENABLED_ACTION_TYPES` rather than
Expand All @@ -234,6 +234,14 @@ A refusal reaches the agent as a tool error it can correct in the same run, and
action never lands in `summary.json`. The action _type_ needs no check:
`ENABLED_ACTION_TYPES` decides which tools the actions server exposes at all.

The one other write is recorded by code after the run. When the plan carries a
`reporter_question`, `record_reporter_question()` in `agent.py` records a
`bugzilla.request_info`: that text as a comment, with a `needinfo?` on the bug's
`creator` in the same `PUT`. The requestee comes from the pre-flight lookup, never
from the model. hackbot-api lists the type in this agent's `always_apply_actions`,
so it is posted even when the diagnosis is held, because a medium- or
low-confidence run is the one that needs the reporter's answer.

Two further hooks shape the comment text as it is recorded:

- `permalink_hook` expands `{{searchfox.permalink}}/<path>#<line>` into
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from hackbot_runtime import HackbotContext, run_async
from pydantic_settings import BaseSettings, SettingsConfigDict

from .agent import FrontendTriageResult, run_frontend_triage
from .agent import FrontendTriageResult, record_reporter_question, run_frontend_triage
from .notify import record_notification
from .preflight import attached_fix, fetch_bug

Expand Down Expand Up @@ -78,6 +78,8 @@ async def main(ctx: HackbotContext) -> FrontendTriageResult:
actions_recorder=ctx.actions,
)

record_reporter_question(ctx.actions, result, bug_fields.get("creator"))

# Recorded last, so it applies after the Bugzilla writes it reports. Only a run
# in a component with a channel records anything -- see notify.py.
record_notification(ctx.actions, result, run_id=ctx.run_id)
Expand Down
48 changes: 47 additions & 1 deletion agents/frontend-triage/hackbot_agents/frontend_triage/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
)
from hackbot_runtime import ActionsRecorder, AgentError, HackbotAgentResult
from hackbot_runtime.actions import ACTIONS_SERVER_NAME
from hackbot_runtime.actions.bugzilla import _COMMENT_FOOTER
from hackbot_runtime.actions.claude_sdk import actions_server_for, actions_to_tool_names
from hackbot_runtime.claude import Reporter
from hackbot_runtime.searchfox import (
Expand Down Expand Up @@ -136,6 +137,9 @@ class FrontendTriageResult(HackbotAgentResult):
# None when the hunt did not run; a populated object with `duplicate_of: null`
# means it ran and found nothing, which is the answer we are measuring.
duplicate_assessment: DuplicateAssessment | None = None
# What the reporter is asked, posted with a needinfo on them by
# `record_reporter_question` whatever the run's confidence.
reporter_question: str | None = None
# This run's verdict on whether its recorded actions may reach the bug without a
# human, computed by `may_apply_unattended`. hackbot-api reads it and still has
# the final say.
Expand Down Expand Up @@ -521,6 +525,47 @@ def may_apply_unattended(plan: dict) -> bool:
return plan.get("confidence") == "high" and plan.get("actionable") is not False


# This text reaches the bug unattended at any confidence, so it is capped. The prompt
# asks for a question or two; tighten the cap if real ones run well under it.
MAX_REPORTER_QUESTION_CHARS = 1000


def record_reporter_question(
recorder: ActionsRecorder, result: FrontendTriageResult, reporter: object
) -> dict | None:
"""Record the run's question for the reporter, with a needinfo on them.

Recorded as ``bugzilla.request_info``, which hackbot-api applies even when the
rest of the run is held: a medium- or low-confidence run is the one that needs
the answer. The requestee is the bug's ``creator``, read by the pre-flight lookup
rather than taken from the model, so nothing the agent read on the bug can choose
who gets flagged.

Records nothing without both a question and a reporter. A question too long to be
one is dropped rather than truncated, since a cut-off question is worse than none.
"""
question = result.reporter_question
if not question or not isinstance(reporter, str) or not reporter.strip():
return None
if len(question) > MAX_REPORTER_QUESTION_CHARS:
print(
f"[frontend_triage] dropping a {len(question)}-character reporter "
f"question (limit {MAX_REPORTER_QUESTION_CHARS})",
file=sys.stderr,
)
return None

return recorder.record(
"bugzilla.request_info",
{
"bug_id": result.bug_id,
"text": f"{question}\n\n---\n\n{_COMMENT_FOOTER}",
"requestee": reporter.strip(),
},
reasoning="The triage needs information only the reporter has.",
)


def parse_plan(text: str | None) -> dict:
"""Extract the structured plan from the agent's final message, if present.

Expand All @@ -546,7 +591,7 @@ def _as_list(value):

def _as_str(value):
# A non-string would fail FrontendTriageResult's validation and lose the whole
# run's result, and these two only route a notification.
# run's result over a field that is not worth it.
return value.strip() or None if isinstance(value, str) else None

actionable = data.get("actionable")
Expand All @@ -569,6 +614,7 @@ def _as_str(value):
"duplicate_assessment": parse_duplicate_assessment(
data.get("duplicate_assessment")
),
"reporter_question": _as_str(data.get("reporter_question")),
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,10 @@
has no way to record a ``bugzilla.update_bug``, a ``bugzilla.create_bug`` or a
Phabricator action in the first place -- ``severity`` is a suggestion in the comment
now, which is what left that tool with no caller.

The one write these hooks do not see is ``bugzilla.request_info``, which code records
after the run (``agent.record_reporter_question``) with the bug's reporter as the only
possible requestee.
"""

import re
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
# dotted so Bugzilla leaves `data` out -- a bare `attachments` returns every
# attachment's base64 body.
BUG_FIELDS = (
"id,product,component,"
"id,product,component,creator,"
"attachments.id,attachments.content_type,attachments.is_obsolete"
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ Before calling any action tool, state in your response:
- **What** action you are recording and **why** (cite the specific rule)
- **Your confidence**: high / medium / low

Record exactly one `bugzilla_add_comment` with your fix plan, ending with the severity sentence described below. That comment is the **only** thing you can write to a bug — you have no tool that changes a field, so a severity, keyword, status or resolution can only be _suggested_ in the comment for a human to apply.
Record exactly one `bugzilla_add_comment` with your fix plan, ending with the severity sentence described below. That comment is the **only** thing you can write to a bug apart from `reporter_question` (below) — you have no tool that changes a field, so a severity, keyword, status or resolution can only be _suggested_ in the comment for a human to apply.

The tool is deliberately narrow, and a call outside what it accepts is refused with the reason (fix it and retry — a refused call records nothing, so it costs you nothing but the turn):

Expand Down Expand Up @@ -206,7 +206,8 @@ After recording your comment, end your final message with a fenced ```json block
"duplicate_of": 1998432,
"confidence": "high | medium | low",
"rationale": "why it is or is not the same defect"
}}
}},
"reporter_question": "Does this still happen in a new profile? If so, which version of Firefox are you on?"
}}
```

Expand All @@ -220,7 +221,9 @@ Field guidance for the handoff:

- **`duplicate_assessment`** — what the `duplicate_hunter` concluded. Set `duplicate_of` to the bug id when it named one and to `null` when it answered `NEW`, with a `rationale` either way. Fill this in **both** cases: a run that looked and found nothing has to be distinguishable from one that never looked, which is what tells us whether the hunt is worth keeping. Set the whole object to null only if the subagent failed to answer.

If you could not localize a root cause, set `root_cause` to null, keep `confidence` low, set `actionable` accordingly, and have your comment ask the specific open questions that block triage.
- **`reporter_question`** — what you need the reporter to tell you to localize or confirm the cause, addressed to them in one or two short questions: steps to reproduce, a version, a profile or pref setting, a screenshot, `about:support`. It is posted as its own comment with a needinfo on the reporter, **whatever your confidence**, so it reaches the bug even when your diagnosis is held for review. Write only the question: no diagnosis, no file links, and do not mention the needinfo, which is set for you. Do not repeat it in your comment. Use `null` when you have nothing to ask the reporter, including when the open question is one only a developer could answer.

If you could not localize a root cause, set `root_cause` to null, keep `confidence` low, set `actionable` accordingly, and put what you need from the reporter in `reporter_question`. Open questions only a developer could answer stay in the comment.

# Additional instructions for this run

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -166,10 +166,13 @@ confidence (see `severity-assessment`).
- **High** (you found the specific code and the cause is clear): record the
plan comment.
- **Medium** (plausible area, cause not pinned down): record the comment with
your best hypothesis and the open questions that would confirm it.
- **Low** (could not localize): record a comment stating what you checked and
the specific information needed to proceed. Set `confidence` to low and
`root_cause` to null in the structured output.
your best hypothesis and the open questions that would confirm it. Questions
only the reporter can answer go in `reporter_question` instead, which reaches
the bug even though this comment is held.
- **Low** (could not localize): record a comment stating what you checked, and
put the specific information you need from the reporter in
`reporter_question`. Set `confidence` to low and `root_cause` to null in the
structured output.

## Already-fixed / duplicate

Expand Down
69 changes: 69 additions & 0 deletions agents/frontend-triage/tests/test_reporter_question.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
"""Tests for the question to the reporter that a run posts whatever its confidence.

hackbot-api applies `bugzilla.request_info` even when the rest of the run is held, so
what gets recorded here is what reaches the bug unattended.
"""

from hackbot_agents.frontend_triage.agent import (
MAX_REPORTER_QUESTION_CHARS,
FrontendTriageResult,
parse_plan,
record_reporter_question,
)
from hackbot_runtime.actions import ActionsRecorder

BUG_ID = 1968342
REPORTER = "reporter@example.com"
QUESTION = "Does this still happen in a new profile?"


def _result(question: str | None = QUESTION) -> FrontendTriageResult:
return FrontendTriageResult(
bug_id=BUG_ID, num_turns=1, total_cost_usd=0.0, reporter_question=question
)


def test_parse_plan_reads_the_reporter_question():
def plan(body: str) -> dict:
return parse_plan(f"```json\n{body}\n```")

assert plan(f'{{"reporter_question": " {QUESTION} "}}')["reporter_question"] == (
QUESTION
)
for body in (
'{"reporter_question": null}',
'{"reporter_question": " "}',
'{"reporter_question": ["a list"]}',
"{}",
):
assert plan(body)["reporter_question"] is None, body


def test_the_question_needinfos_the_reporter():
recorder = ActionsRecorder()
record_reporter_question(recorder, _result(), f" {REPORTER} ")

[action] = recorder.actions
assert action["type"] == "bugzilla.request_info"
assert action["params"]["bug_id"] == BUG_ID
assert action["params"]["requestee"] == REPORTER
assert action["params"]["text"].startswith(QUESTION + "\n\n---\n\n")


def test_nothing_is_recorded_without_a_question_or_a_reporter():
for result, reporter in (
(_result(None), REPORTER),
(_result(), None),
(_result(), ""),
(_result(), {"email": REPORTER}),
):
recorder = ActionsRecorder()
assert record_reporter_question(recorder, result, reporter) is None
assert recorder.actions == []


def test_an_overlong_question_is_dropped_rather_than_cut_off():
recorder = ActionsRecorder()
question = "x" * (MAX_REPORTER_QUESTION_CHARS + 1)
assert record_reporter_question(recorder, _result(question), REPORTER) is None
assert recorder.actions == []
26 changes: 26 additions & 0 deletions services/hackbot-api/app/action_handlers/bugzilla_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,32 @@ async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult
return ActionResult.ok({"bug_id": bug_id, "url": _bug_url(bug_id)})


class RequestInfoHandler:
"""Post a comment and ``needinfo?`` its ``requestee`` in one ``PUT /bug/{id}``.

One transaction, so the flag is attached to the comment that asks the question.
Deliberately not in ``_MERGEABLE_TYPES``: an agent applies this on a different
policy from its other comments (frontend-triage posts it even when the rest of the
run is held), so it must not be folded into one of them.
"""

async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult:
bug_id = params["bug_id"]
body: dict[str, Any] = {
"comment": _comment_body(params),
"comment_tags": [ctx.agent],
"flags": [
{"name": "needinfo", "status": "?", "requestee": params["requestee"]}
],
}
try:
_request("PUT", f"bug/{bug_id}", body)
except Exception as exc:
log.exception("Failed to request info on bug %s", bug_id)
return ActionResult.failed(str(exc))
return ActionResult.ok({"bug_id": bug_id, "url": _bug_url(bug_id)})


class AddAttachmentHandler:
async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult:
bug_id = params["bug_id"]
Expand Down
2 changes: 2 additions & 0 deletions services/hackbot-api/app/action_handlers/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
AddAttachmentHandler,
AddCommentHandler,
CreateBugHandler,
RequestInfoHandler,
UpdateBugHandler,
)
from app.action_handlers.email_handler import SendEmailHandler
Expand All @@ -28,6 +29,7 @@
"bugzilla.add_comment": AddCommentHandler(),
"bugzilla.add_attachment": AddAttachmentHandler(),
"bugzilla.create_bug": CreateBugHandler(),
"bugzilla.request_info": RequestInfoHandler(),

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could use bugzilla.update_bug to add a needinfo. Is there a reason why that would not work here?

"phabricator.submit_patch": SubmitPatchHandler(),
"phabricator.update_patch": UpdatePatchHandler(),
"phabricator.add_comment": PhabricatorAddCommentHandler(),
Expand Down
8 changes: 5 additions & 3 deletions services/hackbot-api/app/agents.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,9 +118,11 @@ def model_to_env(inputs: BaseModel) -> dict[str, str]:
# marked `auto_apply` qualify. Everything else stays for manual apply.
auto_apply_actions=True,
auto_apply_requires_consent=True,
# Except the Slack notification: for a held run it is the only signal that
# an analysis is waiting, so holding it with the comment hides the run.
always_apply_actions=frozenset({"slack.post_message"}),
# Except two that matter most when the run is held. The Slack notification
# is the only signal that an analysis is waiting, so holding it with the
# comment hides the run. The question to the reporter is what a medium- or
# low-confidence run needs answered, so holding it would leave nobody to ask.
always_apply_actions=frozenset({"slack.post_message", "bugzilla.request_info"}),
),
"test-repair": AgentSpec(
name="test-repair",
Expand Down
9 changes: 9 additions & 0 deletions services/hackbot-api/tests/test_actions_applier.py
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,15 @@ def test_the_real_frontend_triage_spec_asks_for_consent():
assert spec.auto_apply_requires_consent is True


def test_a_held_frontend_triage_run_still_asks_the_reporter():
# The run held for review is the one whose question needs an answer, so only
# the diagnosis waits for a human.
spec = AGENT_REGISTRY["frontend-triage"]
held = _run_with_findings(auto_apply=False)
assert _action_auto_applies(spec, held, "bugzilla.request_info")
assert not _action_auto_applies(spec, held, "bugzilla.add_comment")


def test_which_agents_auto_apply_without_asking_for_consent():
# `bug-fix` auto-applies whatever it records, and the apply step dispatches
# against hackbot-api's *global* handler registry — creating bugs, attaching
Expand Down
46 changes: 46 additions & 0 deletions services/hackbot-api/tests/test_bugzilla_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,52 @@ async def test_comment_handlers_add_acting_agent_tag(monkeypatch):
]


async def test_request_info_handler_flags_the_requestee_on_its_comment(monkeypatch):
calls = []
monkeypatch.setattr(
bugzilla_handler, "_request", lambda m, p, b: calls.append((m, p, b))
)
result = await bugzilla_handler.RequestInfoHandler().apply(
{"bug_id": 5, "text": "Which version?", "requestee": "reporter@example.com"},
_ctx(agent="frontend-triage"),
)
# One PUT, so the needinfo is attached to the comment asking the question.
assert result.status == "applied"
assert calls == [
(
"PUT",
"bug/5",
{
"comment": {
"body": "Which version?",
"is_private": False,
"is_markdown": True,
},
"comment_tags": ["frontend-triage"],
"flags": [
{
"name": "needinfo",
"status": "?",
"requestee": "reporter@example.com",
}
],
},
)
]


def test_request_info_never_coalesces_with_a_comment():
# frontend-triage applies it while holding its diagnosis comment, so folding the
# two into one PUT would either post the held comment or hold the question.
planned = bugzilla_handler.plan_coalesced_groups(
[
("bugzilla.add_comment", {"bug_id": 5}),
("bugzilla.request_info", {"bug_id": 5}),
]
)
assert planned == []


async def test_add_attachment_handler_downloads_and_base64_encodes(monkeypatch):
seen = {}
monkeypatch.setattr(
Expand Down