airbyte_ops_mcp.mcp.session_feedback

  1# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
  2"""MCP tool for Devin session feedback reporting.
  3
  4This module exposes the session feedback operation as an MCP tool for AI agents.
  5It is a thin wrapper around the shared dispatch function in the human_in_the_loop
  6module, with hardcoded channel, emoji, and formatting for the feedback use case.
  7
  8Feedback is structured with categories and fields modeled after standard issue/bug
  9report templates to ensure actionable, consistent reports.
 10
 11## MCP reference
 12
 13.. include:: ../../../docs/mcp-generated/session_feedback.md
 14    :start-line: 2
 15"""
 16
 17from __future__ import annotations
 18
 19__all__: list[str] = []
 20
 21import logging
 22from enum import StrEnum
 23from typing import Annotated, Literal
 24
 25import requests
 26from fastmcp import FastMCP
 27from fastmcp_extensions import mcp_tool, register_mcp_tools
 28from pydantic import BaseModel, Field
 29
 30from airbyte_ops_mcp.github_actions import (
 31    WorkflowDispatchResult,
 32    trigger_workflow_dispatch,
 33)
 34from airbyte_ops_mcp.github_api import resolve_ci_trigger_github_token
 35from airbyte_ops_mcp.human_in_the_loop import dispatch_escalation
 36from airbyte_ops_mcp.slack_api import SlackAPIError, SlackURLParseError
 37from airbyte_ops_mcp.slack_posting import parse_slack_thread_url, post_thread_reply
 38
 39logger = logging.getLogger(__name__)
 40
 41# Hardcoded settings for the feedback tool — not agent-controllable.
 42_FEEDBACK_CHANNEL = "C0ACUHRP6B1"
 43_AJ_STEERS_IDENTIFIER = "U05AKF1BCC9"
 44
 45# Triage workflow settings
 46_TRIAGE_REPO_OWNER = "airbytehq"
 47_TRIAGE_REPO_NAME = "airbyte-ops-mcp"
 48_TRIAGE_WORKFLOW_FILE = "devin-session-triage.yml"
 49_TRIAGE_DEFAULT_BRANCH = "main"
 50
 51# --- Category definitions ---
 52
 53_CATEGORY_DISPLAY: dict[str, str] = {
 54    "tool_failure": "Tool Failure",
 55    "missing_guidance": "Missing Guidance",
 56    "suspected_hallucination": "Suspected Hallucination",
 57    "bad_approach": "Bad Approach",
 58    "excessive_iteration": "Excessive Iteration",
 59    "poor_quality": "Poor Quality",
 60    "other_concern": "Other Concern",
 61    "great_results": "Great Results",
 62    "exceeded_expectations": "Exceeded Expectations",
 63    "fast_completion": "Fast Completion",
 64    "good_communication": "Good Communication",
 65    "other_positive_feedback": "Other Positive Feedback",
 66}
 67
 68
 69class FeedbackCategory(StrEnum):
 70    """Feedback categories for Devin session reports."""
 71
 72    # Negative categories
 73    TOOL_FAILURE = "tool_failure"
 74    MISSING_GUIDANCE = "missing_guidance"
 75    SUSPECTED_HALLUCINATION = "suspected_hallucination"
 76    BAD_APPROACH = "bad_approach"
 77    EXCESSIVE_ITERATION = "excessive_iteration"
 78    POOR_QUALITY = "poor_quality"
 79    OTHER_CONCERN = "other_concern"
 80
 81    # Positive categories
 82    GREAT_RESULTS = "great_results"
 83    EXCEEDED_EXPECTATIONS = "exceeded_expectations"
 84    FAST_COMPLETION = "fast_completion"
 85    GOOD_COMMUNICATION = "good_communication"
 86    OTHER_POSITIVE_FEEDBACK = "other_positive_feedback"
 87
 88    def is_negative(self) -> bool:
 89        """Return True if this is a negative feedback category."""
 90        return self in _NEGATIVE_MEMBERS
 91
 92    def display_name(self) -> str:
 93        """Return the human-readable display name for this category."""
 94        return _CATEGORY_DISPLAY.get(self.value, self.value)
 95
 96
 97_NEGATIVE_MEMBERS = frozenset(
 98    {
 99        FeedbackCategory.TOOL_FAILURE,
100        FeedbackCategory.MISSING_GUIDANCE,
101        FeedbackCategory.SUSPECTED_HALLUCINATION,
102        FeedbackCategory.BAD_APPROACH,
103        FeedbackCategory.EXCESSIVE_ITERATION,
104        FeedbackCategory.POOR_QUALITY,
105        FeedbackCategory.OTHER_CONCERN,
106    }
107)
108
109_SEVERITY_DISPLAY: dict[str, str] = {
110    "low": "Low",
111    "medium": "Medium",
112    "high": "High",
113    "critical": "Critical",
114}
115
116
117def _feedback_emoji(feedback_type: str) -> str:
118    """Return the header emoji for the given feedback type."""
119    return ":tada:" if feedback_type == "positive" else ":warning:"
120
121
122def _feedback_label(feedback_type: str) -> str:
123    """Return the header label for the given feedback type."""
124    type_display = "Positive" if feedback_type == "positive" else "Negative"
125    return f"Devin Session Feedback ({type_display})"
126
127
128def _build_feedback_body(
129    *,
130    feedback_type: str,
131    category: str,
132    task_description: str,
133    expected_behavior: str | None,
134    observed_behavior: str | None,
135    what_went_well: str | None,
136    severity: str | None,
137    steps_to_reproduce: str | None,
138) -> str:
139    """Build a Slack mrkdwn message body from structured feedback fields."""
140    lines: list[str] = []
141
142    cat = FeedbackCategory(category)
143    lines.append(f"*Category:* {cat.display_name()}")
144
145    if severity:
146        sev_display = _SEVERITY_DISPLAY.get(severity, severity)
147        lines.append(f"*Severity:* {sev_display}")
148
149    lines.append("")
150    lines.append(f"*Task:* {task_description}")
151
152    if feedback_type == "negative":
153        if expected_behavior:
154            lines.append("")
155            lines.append(f"*Expected Behavior:* {expected_behavior}")
156        if observed_behavior:
157            lines.append("")
158            lines.append(f"*Observed Behavior:* {observed_behavior}")
159        if steps_to_reproduce:
160            lines.append("")
161            lines.append(f"*Steps to Reproduce:* {steps_to_reproduce}")
162    else:
163        if what_went_well:
164            lines.append("")
165            lines.append(f"*What Went Well:* {what_went_well}")
166
167    if feedback_type == "negative":
168        lines.append("")
169        lines.append(
170            "_Auto-triage: a Devin session with v3 analyze mode will inspect this session._"
171        )
172
173    return "\n".join(lines)
174
175
176def _validate_negative_fields(
177    expected_behavior: str | None,
178    observed_behavior: str | None,
179) -> str | None:
180    """Return an error message if required negative feedback fields are missing."""
181    missing: list[str] = []
182    if not expected_behavior:
183        missing.append("expected_behavior")
184    if not observed_behavior:
185        missing.append("observed_behavior")
186    if missing:
187        return f"Negative feedback requires: {', '.join(missing)}."
188    return None
189
190
191def _validate_positive_fields(
192    what_went_well: str | None,
193) -> str | None:
194    """Return an error message if required positive feedback fields are missing."""
195    if not what_went_well:
196        return "Positive feedback requires: what_went_well."
197    return None
198
199
200def _dispatch_triage_workflow(
201    session_url: str,
202    feedback_context: str,
203    reporting_user: str,
204    cc_persons: str = "",
205    header_emoji: str = "",
206    header_label: str = "",
207) -> WorkflowDispatchResult | None:
208    """Dispatch the v3 session triage workflow.
209
210    The triage workflow launches a Devin session with v3 analyze mode and
211    posts a single Slack notification via the HITL reusable workflow.
212    Formatting params (emoji, header, cc) are passed through to the HITL
213    notification so the caller doesn't need to post separately.
214
215    Returns the dispatch result, or None if dispatch fails.
216    """
217    token = resolve_ci_trigger_github_token()
218    inputs: dict[str, str] = {
219        "session_url": session_url,
220        "feedback_context": feedback_context,
221        "reporting_user": reporting_user,
222    }
223    if cc_persons:
224        inputs["cc_persons"] = cc_persons
225    if header_emoji:
226        inputs["header_emoji"] = header_emoji
227    if header_label:
228        inputs["header_label"] = header_label
229    try:
230        return trigger_workflow_dispatch(
231            owner=_TRIAGE_REPO_OWNER,
232            repo=_TRIAGE_REPO_NAME,
233            workflow_file=_TRIAGE_WORKFLOW_FILE,
234            ref=_TRIAGE_DEFAULT_BRANCH,
235            inputs=inputs,
236            token=token,
237        )
238    except requests.HTTPError:
239        logger.exception("Failed to dispatch triage workflow")
240        return None
241
242
243class SessionFeedbackResponse(BaseModel):
244    """Response from the session feedback tool."""
245
246    success: bool = Field(description="Whether the workflow was triggered successfully")
247    message: str = Field(description="Human-readable status message")
248    workflow_url: str | None = Field(
249        default=None,
250        description="URL to view the GitHub Actions workflow file",
251    )
252    run_id: int | None = Field(
253        default=None,
254        description="GitHub Actions workflow run ID",
255    )
256    run_url: str | None = Field(
257        default=None,
258        description="Direct URL to the GitHub Actions workflow run",
259    )
260    triage_run_url: str | None = Field(
261        default=None,
262        description="URL to the auto-triage workflow run",
263    )
264
265
266@mcp_tool(
267    read_only=False,
268    idempotent=False,
269    open_world=True,
270)
271def devin_session_feedback(
272    feedback_type: Annotated[
273        Literal["positive", "negative"],
274        Field(
275            description=(
276                "Type of feedback: 'positive' for a good experience or 'negative' for a "
277                "bad experience. Use 'positive' when the user expresses satisfaction, "
278                "praise, or a success story. Use 'negative' when the user reports a problem, "
279                "frustration, or failure."
280            ),
281        ),
282    ],
283    category: Annotated[
284        FeedbackCategory,
285        Field(
286            description=(
287                "Feedback category. "
288                "For NEGATIVE feedback, use one of: "
289                "'tool_failure' (a specific tool/integration broke), "
290                "'missing_guidance' (Devin lacked instructions or context), "
291                "'suspected_hallucination' (Devin fabricated information or made incorrect claims), "
292                "'bad_approach' (Devin took a fundamentally wrong strategy), "
293                "'excessive_iteration' (too many loops/retries before success), "
294                "'poor_quality' (output quality below expectations), "
295                "'other_concern'. "
296                "For POSITIVE feedback, use one of: "
297                "'great_results' (task completed with high quality), "
298                "'exceeded_expectations' (went above and beyond), "
299                "'fast_completion' (completed quickly and efficiently), "
300                "'good_communication' (kept user well-informed), "
301                "'other_positive_feedback'."
302            ),
303        ),
304    ],
305    task_description: Annotated[
306        str,
307        Field(
308            description=(
309                "Brief description of what the user asked Devin to do. "
310                "This sets the context for the feedback."
311            ),
312        ),
313    ],
314    agent_session_url: Annotated[
315        str,
316        Field(
317            description=(
318                "Your agent session URL so the team can view the full context. "
319                "Use the session URL from your system prompt."
320            ),
321        ),
322    ],
323    reporting_user: Annotated[
324        str,
325        Field(
326            description=(
327                "The person providing the feedback. Accepts an email address "
328                "(e.g. 'aj@airbyte.io'), a GitHub handle prefixed with @ "
329                "(e.g. '@aaronsteers'), or a Slack user ID (e.g. 'U05AKF1BCC9')."
330            ),
331        ),
332    ],
333    expected_behavior: Annotated[
334        str | None,
335        Field(
336            default=None,
337            description=(
338                "What should have happened. REQUIRED for negative feedback. "
339                "Describe the expected outcome clearly."
340            ),
341        ),
342    ],
343    observed_behavior: Annotated[
344        str | None,
345        Field(
346            default=None,
347            description=(
348                "What actually happened. REQUIRED for negative feedback. "
349                "Describe the actual outcome, including any error messages or unexpected results."
350            ),
351        ),
352    ],
353    what_went_well: Annotated[
354        str | None,
355        Field(
356            default=None,
357            description=(
358                "What specifically was good about the experience. REQUIRED for positive feedback. "
359                "Be specific about what Devin did well."
360            ),
361        ),
362    ],
363    severity: Annotated[
364        Literal["low", "medium", "high", "critical"] | None,
365        Field(
366            default=None,
367            description=(
368                "Severity of the issue. Recommended for negative feedback. "
369                "'low' = minor inconvenience, 'medium' = notable impact, "
370                "'high' = significant blocker, 'critical' = complete failure."
371            ),
372        ),
373    ],
374    steps_to_reproduce: Annotated[
375        str | None,
376        Field(
377            default=None,
378            description=(
379                "Optional steps to reproduce the issue. Helpful for negative feedback "
380                "to enable the team to investigate."
381            ),
382        ),
383    ],
384    session_to_evaluate: Annotated[
385        str | None,
386        Field(
387            default=None,
388            description=(
389                "Optional Devin session URL to evaluate/triage. Use this when reporting "
390                "feedback about a *different* session (not your own). If omitted, "
391                "agent_session_url is used as the session to triage (i.e., the reporter "
392                "is reporting on itself)."
393            ),
394        ),
395    ],
396) -> SessionFeedbackResponse:
397    """Report structured feedback about a Devin session experience via Slack.
398
399    Posts a formatted feedback message to the #hydra-feedback Slack channel,
400    tagging the reporting user and @AJ Steers. The message includes a clickable
401    button for the Devin session link. For negative feedback, a triage workflow
402    is automatically dispatched to launch a Devin session with v3 analyze mode
403    that can inspect the original session's full conversation history.
404
405    IMPORTANT: This feedback will be logged publicly in Slack. Inform the user
406    that their feedback is visible to the team and they may be contacted for
407    additional details.
408
409    Use this tool when a user explicitly asks to report a positive or negative
410    experience with their Devin session. Before calling this tool, let the user
411    know:
412    - Their feedback will be posted publicly in the #hydra-feedback Slack channel
413    - They may be contacted by the team for more details
414    - Both the reporting user and @AJ Steers will be tagged in the message
415    - For negative feedback, a triage session will be automatically launched to inspect the reported session
416
417    The Slack message is sent by a GitHub Actions workflow so that Slack
418    credentials are never exposed to the calling agent.
419    """
420    # Validate category matches feedback type.
421    cat = FeedbackCategory(category)
422    is_negative_feedback = feedback_type == "negative"
423    if cat.is_negative() != is_negative_feedback:
424        expected_kind = "negative" if is_negative_feedback else "positive"
425        valid = [
426            c.value for c in FeedbackCategory if c.is_negative() == is_negative_feedback
427        ]
428        return SessionFeedbackResponse(
429            success=False,
430            message=(
431                f"Invalid category '{category}' for {feedback_type} feedback. "
432                f"Valid {expected_kind} categories: {', '.join(valid)}."
433            ),
434        )
435
436    # Validate required fields based on feedback type.
437    if feedback_type == "negative":
438        validation_error = _validate_negative_fields(
439            expected_behavior=expected_behavior,
440            observed_behavior=observed_behavior,
441        )
442    else:
443        validation_error = _validate_positive_fields(
444            what_went_well=what_went_well,
445        )
446
447    if validation_error:
448        return SessionFeedbackResponse(
449            success=False,
450            message=validation_error,
451        )
452
453    message_body = _build_feedback_body(
454        feedback_type=feedback_type,
455        category=category,
456        task_description=task_description,
457        expected_behavior=expected_behavior,
458        observed_behavior=observed_behavior,
459        what_went_well=what_went_well,
460        severity=severity,
461        steps_to_reproduce=steps_to_reproduce,
462    )
463
464    # For negative feedback, dispatch triage workflow (which also posts to Slack
465    # via the HITL reusable workflow — single message with triage button).
466    # For positive feedback, dispatch HITL directly (no triage needed).
467    if is_negative_feedback:
468        triage_session_url = session_to_evaluate or agent_session_url
469        # _dispatch_triage_workflow catches exceptions internally and returns None
470        triage_result = _dispatch_triage_workflow(
471            session_url=triage_session_url,
472            feedback_context=message_body,
473            reporting_user=reporting_user,
474            cc_persons=_AJ_STEERS_IDENTIFIER,
475            header_emoji=_feedback_emoji(feedback_type),
476            header_label=_feedback_label(feedback_type),
477        )
478        if triage_result is not None:
479            view_url = triage_result.run_url or triage_result.workflow_url
480            return SessionFeedbackResponse(
481                success=True,
482                message=(
483                    "Feedback submitted. Auto-triage workflow launched. "
484                    "A Slack notification will be posted to #hydra-feedback "
485                    "once the triage session starts. "
486                    f"View workflow progress at: {view_url}"
487                ),
488                workflow_url=triage_result.workflow_url,
489                run_id=triage_result.run_id,
490                run_url=triage_result.run_url,
491                triage_run_url=view_url,
492            )
493        # Triage dispatch failed — fall back to direct HITL notification
494        # so negative feedback is still recorded in Slack.
495        logger.warning(
496            "Triage workflow dispatch failed; falling back to direct HITL dispatch."
497        )
498
499    # Positive feedback (or negative feedback fallback): dispatch HITL directly
500    result = dispatch_escalation(
501        target_person=reporting_user,
502        message=message_body,
503        agent_session_url=agent_session_url,
504        cc=[_AJ_STEERS_IDENTIFIER],
505        channel_override=_FEEDBACK_CHANNEL,
506        header_emoji=_feedback_emoji(feedback_type),
507        header_label=_feedback_label(feedback_type),
508    )
509
510    view_url = result.run_url or result.workflow_url
511    return SessionFeedbackResponse(
512        success=True,
513        message=(
514            f"Feedback submitted and posted to #hydra-feedback. "
515            f"The reporting user and @AJ Steers have been tagged. "
516            f"View progress at: {view_url}"
517        ),
518        workflow_url=result.workflow_url,
519        run_id=result.run_id,
520        run_url=result.run_url,
521    )
522
523
524# ---------------------------------------------------------------------------
525# Non-interactive thread disclaimer
526# ---------------------------------------------------------------------------
527
528_FOLLOWUP_HEADER = "🤖 *Automated Triage Update*"
529_FOLLOWUP_FOOTER_TEMPLATE = (
530    "_ℹ️ This thread is not monitored by Devin. "
531    "Replies here will not be seen by any agent. "
532    "For follow-up, use the <{agent_session_url}|linked session> or create a new task._"
533)
534
535
536def _wrap_followup_message(message: str, *, agent_session_url: str) -> str:
537    """Wrap a follow-up message with session link and non-interactive disclaimer."""
538    footer = _FOLLOWUP_FOOTER_TEMPLATE.format(agent_session_url=agent_session_url)
539    return f"{_FOLLOWUP_HEADER}\n\n{message}\n\n{footer}"
540
541
542# ---------------------------------------------------------------------------
543# Feedback follow-up tool
544# ---------------------------------------------------------------------------
545
546
547class SessionFeedbackFollowupResponse(BaseModel):
548    """Response from the session feedback follow-up tool."""
549
550    success: bool = Field(description="Whether the follow-up was posted successfully")
551    message: str = Field(description="Human-readable status message")
552    reply_ts: str | None = Field(
553        default=None,
554        description="Timestamp of the posted reply (Slack ts format)",
555    )
556
557
558@mcp_tool(
559    read_only=False,
560    idempotent=False,
561    open_world=True,
562)
563def devin_session_feedback_followup(
564    thread_url: Annotated[
565        str,
566        Field(
567            description=(
568                "Slack thread URL from the original feedback post in #hydra-feedback. "
569                "This is the thread where follow-up context will be appended. "
570                "Example: https://airbytehq-team.slack.com/archives/C0ACUHRP6B1/p1773062711122019"
571            ),
572        ),
573    ],
574    message: Annotated[
575        str,
576        Field(
577            description=(
578                "Follow-up message text in Slack mrkdwn format. "
579                "Typically a triage report or additional context about the "
580                "feedback being investigated. "
581                "Supports *bold*, _italic_, `code`, ```code blocks```, "
582                "> blockquotes, and <url|label> links."
583            ),
584        ),
585    ],
586    agent_session_url: Annotated[
587        str,
588        Field(
589            description=(
590                "Your agent session URL for audit trail. "
591                "Use the session URL from your system prompt."
592            ),
593        ),
594    ],
595) -> SessionFeedbackFollowupResponse:
596    """Post a follow-up to an existing feedback thread in #hydra-feedback.
597
598    This is the "second call" in the feedback workflow: after
599    `devin_session_feedback` creates the initial report, this tool appends
600    triage findings or additional context as a threaded reply.
601
602    Each reply is wrapped with a disclaimer clarifying that the thread is
603    non-interactive and not monitored by any agent.
604
605    Workspace validation ensures only URLs from the expected Slack
606    workspace are accepted.
607    """
608    try:
609        channel_id, thread_ts = parse_slack_thread_url(thread_url)
610    except SlackURLParseError as exc:
611        return SessionFeedbackFollowupResponse(
612            success=False,
613            message=str(exc),
614        )
615
616    wrapped_message = _wrap_followup_message(
617        message, agent_session_url=agent_session_url
618    )
619
620    try:
621        reply_ts = post_thread_reply(
622            channel_id=channel_id,
623            thread_ts=thread_ts,
624            message=wrapped_message,
625        )
626    except SlackAPIError as exc:
627        return SessionFeedbackFollowupResponse(
628            success=False,
629            message=f"Slack API error: {exc}",
630        )
631
632    logger.info(
633        "Feedback follow-up posted: channel=%s thread_ts=%s agent=%s",
634        channel_id,
635        thread_ts,
636        agent_session_url,
637    )
638    return SessionFeedbackFollowupResponse(
639        success=True,
640        message=f"Follow-up posted to feedback thread in channel {channel_id}.",
641        reply_ts=reply_ts,
642    )
643
644
645def register_session_feedback_tools(app: FastMCP) -> None:
646    """Register session feedback tools with the FastMCP app."""
647    register_mcp_tools(app, mcp_module=__name__)