#!/usr/bin/env bash
#
# analyze_training.sh - Post-training performance analysis script
#
# Part of Phase 4 (Tracing & Observability) - entrenar spec
#
# Analyzes renacer ML anomaly detection output to identify:
# - Top bottlenecks (I/O, compute, syscalls)
# - Anomalous syscalls (high latency outliers)
# - Clustering quality (silhouette score)
# - Recommendations for optimization
#
# Usage:
#   ./scripts/analyze_training.sh [profile.json]
#
# Default profile: .pmat/llama-training-profile.json (generated by make profile-llama-anomaly)

set -euo pipefail

# Colors for output
RED='\033[0;31m'
YELLOW='\033[1;33m'
GREEN='\033[0;32m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

PROFILE_FILE="${1:-.pmat/llama-training-profile.json}"

echo "🔍 Analyzing LLaMA Training Profile"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

# Check if profile file exists
if [[ ! -f "$PROFILE_FILE" ]]; then
    echo -e "${RED}✗ Error: Profile file not found: $PROFILE_FILE${NC}"
    echo ""
    echo "Generate a profile first:"
    echo "  make profile-llama-anomaly"
    exit 1
fi

echo -e "${BLUE}📁 Profile:${NC} $PROFILE_FILE"
echo ""

# Check if jq is available for JSON parsing
if ! command -v jq &> /dev/null; then
    echo -e "${YELLOW}⚠️  jq not found. Install with: sudo apt-get install jq${NC}"
    echo "Showing raw file contents..."
    echo ""
    cat "$PROFILE_FILE"
    exit 0
fi

# Parse ML anomaly analysis
echo -e "${BLUE}═══ ML Anomaly Detection Results ══${NC}"
echo ""

# Extract clustering metrics
CLUSTERS=$(jq -r '.ml_analysis.clusters // "N/A"' "$PROFILE_FILE" 2>/dev/null || echo "N/A")
SILHOUETTE=$(jq -r '.ml_analysis.silhouette_score // "N/A"' "$PROFILE_FILE" 2>/dev/null || echo "N/A")

echo -e "${GREEN}Clustering Quality:${NC}"
echo "  Clusters: $CLUSTERS"
echo "  Silhouette Score: $SILHOUETTE"

# Interpret silhouette score
if [[ "$SILHOUETTE" != "N/A" ]]; then
    SCORE=$(echo "$SILHOUETTE" | awk '{printf "%.2f", $1}')
    if (( $(echo "$SCORE > 0.7" | bc -l) )); then
        echo -e "  ${GREEN}✓ Excellent cluster separation${NC}"
    elif (( $(echo "$SCORE > 0.5" | bc -l) )); then
        echo -e "  ${YELLOW}~ Good cluster separation${NC}"
    else
        echo -e "  ${RED}✗ Poor cluster separation - noisy data${NC}"
    fi
fi

echo ""

# High-latency cluster detection
echo -e "${YELLOW}High-Latency Cluster:${NC}"
CLUSTER_ID=$(jq -r '.ml_analysis.high_latency_cluster.cluster_id // "N/A"' "$PROFILE_FILE" 2>/dev/null || echo "N/A")
AVG_LATENCY=$(jq -r '.ml_analysis.high_latency_cluster.avg_latency_us // "N/A"' "$PROFILE_FILE" 2>/dev/null || echo "N/A")
SYSCALLS=$(jq -r '.ml_analysis.high_latency_cluster.syscalls // [] | join(", ")' "$PROFILE_FILE" 2>/dev/null || echo "N/A")
COUNT=$(jq -r '.ml_analysis.high_latency_cluster.count // "N/A"' "$PROFILE_FILE" 2>/dev/null || echo "N/A")

if [[ "$CLUSTER_ID" != "N/A" ]]; then
    echo "  Cluster ID: $CLUSTER_ID"
    echo "  Avg Latency: ${AVG_LATENCY}μs"
    echo "  Syscalls: $SYSCALLS"
    echo "  Count: $COUNT"
else
    echo "  No high-latency cluster detected"
fi

echo ""

# Outlier detection
echo -e "${RED}═══ Outlier Detection ══${NC}"
echo ""

OUTLIER_COUNT=$(jq -r '.ml_analysis.outliers // [] | length' "$PROFILE_FILE" 2>/dev/null || echo "0")

if [[ "$OUTLIER_COUNT" -gt 0 ]]; then
    echo -e "${RED}Found $OUTLIER_COUNT outliers:${NC}"
    echo ""
    jq -r '.ml_analysis.outliers[] | "  - \(.syscall): \(.latency_us)μs (z-score: \(.z_score))"' "$PROFILE_FILE" 2>/dev/null || echo "Error parsing outliers"
else
    echo -e "${GREEN}✓ No outliers detected${NC}"
fi

echo ""

# Real-time anomaly summary
echo -e "${BLUE}═══ Real-Time Anomaly Summary ══${NC}"
echo ""

TOTAL_ANOMALIES=$(jq -r '.anomaly_summary.total // "N/A"' "$PROFILE_FILE" 2>/dev/null || echo "N/A")
HIGH_ANOMALIES=$(jq -r '.anomaly_summary.high // "N/A"' "$PROFILE_FILE" 2>/dev/null || echo "N/A")
MEDIUM_ANOMALIES=$(jq -r '.anomaly_summary.medium // "N/A"' "$PROFILE_FILE" 2>/dev/null || echo "N/A")
LOW_ANOMALIES=$(jq -r '.anomaly_summary.low // "N/A"' "$PROFILE_FILE" 2>/dev/null || echo "N/A")

echo "Total Anomalies: $TOTAL_ANOMALIES"
echo ""
echo "Severity Distribution:"
echo -e "  ${RED}🔴 High (>5.0σ):${NC}   $HIGH_ANOMALIES (likely hardware issues)"
echo -e "  ${YELLOW}🟡 Medium (4-5σ):${NC}  $MEDIUM_ANOMALIES (investigate)"
echo -e "  ${GREEN}🟢 Low (3-4σ):${NC}    $LOW_ANOMALIES (noise)"

echo ""

# Recommendations
echo -e "${GREEN}═══ Recommendations ══${NC}"
echo ""

if [[ "$HIGH_ANOMALIES" != "N/A" && "$HIGH_ANOMALIES" -gt 0 ]]; then
    echo -e "${RED}⚠️  High severity anomalies detected!${NC}"
    echo "  - Check for hardware issues (GPU throttling, disk contention)"
    echo "  - Review high-latency syscalls in the outliers section"
    echo "  - Consider running on different hardware"
fi

if [[ "$OUTLIER_COUNT" -gt 5 ]]; then
    echo -e "${YELLOW}⚠️  Multiple outliers detected${NC}"
    echo "  - I/O bottlenecks likely present"
    echo "  - Consider optimizing checkpoint loading"
    echo "  - Review disk I/O patterns"
fi

if [[ "$SILHOUETTE" != "N/A" ]]; then
    SCORE=$(echo "$SILHOUETTE" | awk '{printf "%.2f", $1}')
    if (( $(echo "$SCORE < 0.5" | bc -l) )); then
        echo -e "${YELLOW}⚠️  Poor clustering quality${NC}"
        echo "  - Training performance may be inconsistent"
        echo "  - Consider running longer benchmark for better baseline"
    fi
fi

echo ""
echo -e "${GREEN}✓ Analysis complete!${NC}"
echo ""
echo "Next steps:"
echo "  1. Review Jaeger traces: http://localhost:16686"
echo "  2. Run with renacer OTLP: make profile-llama-otlp"
echo "  3. Optimize identified bottlenecks"
