//! With/without-tools ablation analysis.
//!
//! Answers the question a benchmark like greynewell/mcpbr asks: does
//! attaching an MCP server's tools actually help the agent? Each task is run
//! twice through the same agent loop, once with the server's tools advertised
//! and once with them withheld, and the two outcomes are compared. The report
//! is the uplift: how many tasks the tools turned from fail to pass, how often
//! the model reached for a tool when it had one, and what the extra tool
//! round-trips cost in tokens and wall-clock.
//!
//! This is distinct from tool-bench, which mutates tool descriptions to measure
//! selection sensitivity; ablation measures whether the tools help at all.
//!
//! The analysis is pure and deterministic: the live harness drives both arms
//! and feeds their measured outcomes in. Keeping it pure means the uplift math
//! is unit-tested with synthetic outcomes, with no model or network in the
//! loop.

use serde::Serialize;

/// One arm's measured outcome for a single task (tools attached, or withheld).
#[derive(Debug, Clone)]
pub struct ArmOutcome {
    /// Whether the task's success criteria passed in this arm.
    pub passed: bool,
    /// Number of tool calls the model made in this arm. Always zero for the
    /// tools-withheld arm (the model is offered no tools).
    pub tool_calls: usize,
    /// Dollar cost of this arm's run, when a real provider priced it. Zero
    /// when unknown (the offline stub does not price a run).
    pub cost_usd: f64,
    /// Wall-clock duration of this arm's run, in milliseconds.
    pub duration_ms: u64,
}

/// A single task run through both arms.
#[derive(Debug, Clone)]
pub struct TaskAblation {
    /// Task identifier (the agent test name).
    pub task: String,
    /// Outcome with the server's tools advertised.
    pub with_tools: ArmOutcome,
    /// Outcome with the server's tools withheld.
    pub without_tools: ArmOutcome,
}

/// How the tools affected one task.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum TaskVerdict {
    /// Passed with tools, failed without: the tools made the difference.
    Helped,
    /// Failed with tools, passed without: the tools got in the way.
    Hurt,
    /// Same pass/fail state in both arms: the tools changed nothing.
    Unchanged,
}

/// Per-task uplift detail.
#[derive(Debug, Clone, Serialize)]
pub struct TaskUplift {
    /// Task identifier.
    pub task: String,
    /// Whether the tools-attached arm passed.
    pub with_pass: bool,
    /// Whether the tools-withheld arm passed.
    pub without_pass: bool,
    /// Whether the tools-attached arm actually called at least one tool.
    pub used_tools: bool,
    /// How the tools affected this task.
    pub verdict: TaskVerdict,
}

/// The ablation verdict over all tasks.
#[derive(Debug, Clone, Default, Serialize)]
pub struct AblationReport {
    /// Number of tasks analyzed.
    pub tasks: usize,
    /// Tasks that passed with tools attached.
    pub with_pass: usize,
    /// Tasks that passed with tools withheld.
    pub without_pass: usize,
    /// Tasks the tools turned from fail to pass.
    pub helped: usize,
    /// Tasks the tools turned from pass to fail.
    pub hurt: usize,
    /// Net success uplift in `[-1.0, 1.0]`: `(with_pass - without_pass) / tasks`.
    /// Positive means the tools help; zero means they make no difference;
    /// negative means they hurt.
    pub success_uplift: f64,
    /// Fraction of tasks in `[0.0, 1.0]` where the tools-attached arm reached
    /// for at least one tool. A low rate next to a high uplift hints the model
    /// barely used the tools it nonetheless benefited from.
    pub tool_selection_rate: f64,
    /// Total dollar cost the tools added: `sum(with) - sum(without)`.
    pub cost_delta_usd: f64,
    /// Total wall-clock the tools added, in milliseconds (signed).
    pub latency_delta_ms: i64,
    /// Per-task breakdown.
    pub per_task: Vec<TaskUplift>,
}

impl AblationReport {
    /// True when the net success uplift meets `min_uplift`. A gate of `0.0`
    /// accepts tools that do not hurt; a positive gate demands measured help.
    pub fn passed(&self, min_uplift: f64) -> bool {
        self.success_uplift >= min_uplift
    }
}

/// Analyze paired with/without-tools task outcomes into an uplift report.
pub fn analyze(tasks: &[TaskAblation]) -> AblationReport {
    let mut report = AblationReport {
        tasks: tasks.len(),
        ..Default::default()
    };
    let (mut with_cost, mut without_cost) = (0.0_f64, 0.0_f64);
    let (mut with_ms, mut without_ms) = (0_i64, 0_i64);
    let mut tool_used = 0_usize;
    for task in tasks {
        if task.with_tools.passed {
            report.with_pass += 1;
        }
        if task.without_tools.passed {
            report.without_pass += 1;
        }
        let used_tools = task.with_tools.tool_calls > 0;
        if used_tools {
            tool_used += 1;
        }
        let verdict = match (task.with_tools.passed, task.without_tools.passed) {
            (true, false) => {
                report.helped += 1;
                TaskVerdict::Helped
            }
            (false, true) => {
                report.hurt += 1;
                TaskVerdict::Hurt
            }
            _ => TaskVerdict::Unchanged,
        };
        with_cost += task.with_tools.cost_usd;
        without_cost += task.without_tools.cost_usd;
        with_ms += task.with_tools.duration_ms as i64;
        without_ms += task.without_tools.duration_ms as i64;
        report.per_task.push(TaskUplift {
            task: task.task.clone(),
            with_pass: task.with_tools.passed,
            without_pass: task.without_tools.passed,
            used_tools,
            verdict,
        });
    }
    if !tasks.is_empty() {
        let n = tasks.len() as f64;
        report.success_uplift = (report.with_pass as f64 - report.without_pass as f64) / n;
        report.tool_selection_rate = tool_used as f64 / n;
    }
    report.cost_delta_usd = with_cost - without_cost;
    report.latency_delta_ms = with_ms - without_ms;
    report
}

/// Render the report as a human-friendly summary.
pub fn render(report: &AblationReport) -> String {
    let mut s = format!(
        "Tool ablation: {} task(s), {} passed with tools, {} without\n",
        report.tasks, report.with_pass, report.without_pass
    );
    s.push_str(&format!(
        "  success uplift: {:+.0}% ({} helped, {} hurt)\n",
        report.success_uplift * 100.0,
        report.helped,
        report.hurt
    ));
    s.push_str(&format!(
        "  tool selection rate: {:.0}%\n",
        report.tool_selection_rate * 100.0
    ));
    s.push_str(&format!(
        "  cost added: ${:.4}    latency added: {} ms\n",
        report.cost_delta_usd, report.latency_delta_ms
    ));
    for task in &report.per_task {
        let mark = match task.verdict {
            TaskVerdict::Helped => "helped",
            TaskVerdict::Hurt => "hurt",
            TaskVerdict::Unchanged => "unchanged",
        };
        s.push_str(&format!(
            "    {}: {} (with={}, without={}, used_tools={})\n",
            task.task,
            mark,
            pass_word(task.with_pass),
            pass_word(task.without_pass),
            task.used_tools
        ));
    }
    s
}

fn pass_word(passed: bool) -> &'static str {
    if passed {
        "pass"
    } else {
        "fail"
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn arm(passed: bool, tool_calls: usize, cost_usd: f64, duration_ms: u64) -> ArmOutcome {
        ArmOutcome {
            passed,
            tool_calls,
            cost_usd,
            duration_ms,
        }
    }

    #[test]
    fn tools_that_turn_a_failure_into_a_pass_show_positive_uplift() {
        // The classic mcpbr case: with the weather tool the agent answers
        // correctly; without it, it cannot.
        let tasks = vec![TaskAblation {
            task: "weather".into(),
            with_tools: arm(true, 1, 0.0, 100),
            without_tools: arm(false, 0, 0.0, 40),
        }];
        let report = analyze(&tasks);
        assert_eq!(report.success_uplift, 1.0);
        assert_eq!(report.helped, 1);
        assert_eq!(report.hurt, 0);
        assert_eq!(report.per_task[0].verdict, TaskVerdict::Helped);
        assert!(report.passed(0.5));
    }

    #[test]
    fn a_task_the_tools_do_not_affect_shows_zero_uplift() {
        // An unrelated task the model answers from its own knowledge: both arms
        // pass, the with-tools arm never reaches for a tool.
        let tasks = vec![TaskAblation {
            task: "capital-of-france".into(),
            with_tools: arm(true, 0, 0.0, 50),
            without_tools: arm(true, 0, 0.0, 48),
        }];
        let report = analyze(&tasks);
        assert_eq!(report.success_uplift, 0.0);
        assert_eq!(report.helped, 0);
        assert_eq!(report.tool_selection_rate, 0.0);
        assert_eq!(report.per_task[0].verdict, TaskVerdict::Unchanged);
    }

    #[test]
    fn a_mix_averages_the_uplift_and_counts_tool_selection() {
        let tasks = vec![
            TaskAblation {
                task: "weather".into(),
                with_tools: arm(true, 1, 0.0, 100),
                without_tools: arm(false, 0, 0.0, 40),
            },
            TaskAblation {
                task: "trivia".into(),
                with_tools: arm(true, 0, 0.0, 50),
                without_tools: arm(true, 0, 0.0, 50),
            },
        ];
        let report = analyze(&tasks);
        // One of two tasks improved.
        assert_eq!(report.success_uplift, 0.5);
        // Only the weather task reached for a tool.
        assert_eq!(report.tool_selection_rate, 0.5);
        assert_eq!(report.with_pass, 2);
        assert_eq!(report.without_pass, 1);
    }

    #[test]
    fn tools_that_break_a_task_register_as_hurt() {
        let tasks = vec![TaskAblation {
            task: "regression".into(),
            with_tools: arm(false, 1, 0.0, 120),
            without_tools: arm(true, 0, 0.0, 30),
        }];
        let report = analyze(&tasks);
        assert_eq!(report.success_uplift, -1.0);
        assert_eq!(report.hurt, 1);
        assert_eq!(report.per_task[0].verdict, TaskVerdict::Hurt);
        assert!(!report.passed(0.0));
    }

    #[test]
    fn cost_and_latency_deltas_sum_the_arms() {
        let tasks = vec![TaskAblation {
            task: "weather".into(),
            with_tools: arm(true, 1, 0.0030, 100),
            without_tools: arm(false, 0, 0.0010, 40),
        }];
        let report = analyze(&tasks);
        assert!((report.cost_delta_usd - 0.0020).abs() < 1e-9);
        assert_eq!(report.latency_delta_ms, 60);
    }

    #[test]
    fn an_empty_run_is_a_zero_report() {
        let report = analyze(&[]);
        assert_eq!(report.tasks, 0);
        assert_eq!(report.success_uplift, 0.0);
        assert!(report.passed(0.0));
    }

    #[test]
    fn render_states_the_uplift_and_each_task() {
        let report = analyze(&[TaskAblation {
            task: "weather".into(),
            with_tools: arm(true, 1, 0.0, 100),
            without_tools: arm(false, 0, 0.0, 40),
        }]);
        let text = render(&report);
        assert!(text.contains("success uplift: +100%"));
        assert!(text.contains("weather: helped"));
        assert!(text.contains("tool selection rate: 100%"));
    }
}
