"""Learning pipeline: candidate capture and approved rule promotion.

Phase 1 goals:
- Separate raw learning observations from approved prompt rules.
- Allow explicit user corrections to auto-promote conservatively.
- Keep self-reflection outputs as candidates only until a later validation phase.
"""

from __future__ import annotations

import asyncio
import json
import re
import threading
import uuid
from datetime import datetime

from backend.config import MEMORY_DIR, _load_settings, _model_ctx
from backend.models import _call_text_model_candidates, _is_llm_error, _unique_model_candidates


_LEARNING_CANDIDATES_FILE = MEMORY_DIR / "learning_candidates.jsonl"
_LEARNING_RULES_FILE = MEMORY_DIR / "learning_rules.json"
_MAX_APPROVED_RULES = 200
_MAX_CANDIDATES = 1000
_LEARNING_REVIEW_TIMEOUT_SECONDS = 25.0
_LEARNING_REVIEW_PER_MODEL_TIMEOUT_SECONDS = 12.0


def _task_scope(task_state: dict | None = None) -> dict:
    task = dict(task_state or {})
    return {
        "task_type": _clean_text(task.get("task_type") or "generic", limit=80) or "generic",
        "provider_scope": _clean_text(task.get("provider_scope") or "default", limit=80) or "default",
        "time_filter": _clean_text(task.get("time_filter") or "none", limit=80) or "none",
    }


def _ensure_parent(path) -> None:
    try:
        path.parent.mkdir(parents=True, exist_ok=True)
    except Exception:
        pass


def _clean_text(value: str, limit: int = 300) -> str:
    return " ".join(str(value or "").strip().split())[:limit]


def _rule_key(text: str) -> str:
    return _clean_text(text, limit=500).casefold()


def _load_rules_store() -> dict:
    try:
        if _LEARNING_RULES_FILE.exists():
            data = json.loads(_LEARNING_RULES_FILE.read_text(encoding="utf-8"))
            if isinstance(data, dict):
                rules = data.get("rules")
                data["rules"] = rules if isinstance(rules, list) else []
                data["version"] = int(data.get("version") or 1)
                return data
    except Exception:
        pass
    return {"version": 1, "rules": []}


def _save_rules_store(store: dict) -> None:
    try:
        _ensure_parent(_LEARNING_RULES_FILE)
        rules = list(store.get("rules") or [])[-_MAX_APPROVED_RULES:]
        payload = {"version": int(store.get("version") or 1), "rules": rules}
        _LEARNING_RULES_FILE.write_text(
            json.dumps(payload, ensure_ascii=False, indent=2),
            encoding="utf-8",
        )
    except Exception:
        pass


def _append_candidate(candidate: dict) -> None:
    try:
        _ensure_parent(_LEARNING_CANDIDATES_FILE)
        with _LEARNING_CANDIDATES_FILE.open("a", encoding="utf-8") as handle:
            handle.write(json.dumps(candidate, ensure_ascii=False) + "\n")
    except Exception:
        pass


def _load_candidates() -> list[dict]:
    rows: list[dict] = []
    try:
        if not _LEARNING_CANDIDATES_FILE.exists():
            return rows
        with _LEARNING_CANDIDATES_FILE.open("r", encoding="utf-8") as handle:
            for line in handle:
                raw = str(line or "").strip()
                if not raw:
                    continue
                parsed = json.loads(raw)
                if isinstance(parsed, dict):
                    rows.append(parsed)
    except Exception:
        return []
    return rows


def _save_candidates(candidates: list[dict]) -> None:
    try:
        _ensure_parent(_LEARNING_CANDIDATES_FILE)
        rows = [row for row in (candidates or []) if isinstance(row, dict)][-_MAX_CANDIDATES:]
        with _LEARNING_CANDIDATES_FILE.open("w", encoding="utf-8") as handle:
            for row in rows:
                handle.write(json.dumps(row, ensure_ascii=False) + "\n")
    except Exception:
        pass


def _normalize_candidate(candidate: dict) -> dict:
    row = dict(candidate or {})
    decision = evaluate_learning_candidate(row)
    status = _clean_text(row.get("status") or decision.get("status") or "candidate", limit=40) or "candidate"
    reason = _clean_text(row.get("review_reason") or decision.get("reason") or "", limit=120)
    row["status"] = status
    if reason:
        row["review_reason"] = reason
    if decision.get("storage_target"):
        row.setdefault("storage_target", str(decision.get("storage_target") or ""))
    if decision.get("activation_mode"):
        row.setdefault("activation_mode", str(decision.get("activation_mode") or ""))
    return row


def _candidate_identity(candidate: dict) -> tuple[str, str, str, str, str]:
    scope = _task_scope(candidate)
    impact_type = _clean_text(candidate.get("impact_type") or "observation", limit=80)
    rule_text = _rule_key(candidate.get("rule_text") or "")
    return (
        impact_type,
        rule_text,
        scope["task_type"],
        scope["provider_scope"],
        scope["time_filter"],
    )


def _historical_signal(candidate: dict) -> dict:
    identity = _candidate_identity(candidate)
    signal = {"approved": 0, "candidate": 0, "rejected": 0}

    for row in _load_candidates():
        if not isinstance(row, dict):
            continue
        if _candidate_identity(row) != identity:
            continue
        status = _clean_text(row.get("status") or "candidate", limit=40) or "candidate"
        if status in signal:
            signal[status] += 1

    for row in get_learning_rules(impact_type=identity[0], status="approved"):
        if not isinstance(row, dict):
            continue
        if _candidate_identity(row) != identity:
            continue
        signal["approved"] += 1

    return signal


def _run_async_from_sync(awaitable):
    try:
        running_loop = asyncio.get_running_loop()
    except RuntimeError:
        running_loop = None

    if running_loop and running_loop.is_running():
        box: dict[str, object] = {}
        error_box: dict[str, BaseException] = {}

        def _runner() -> None:
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)
            try:
                box["value"] = loop.run_until_complete(awaitable)
            except BaseException as exc:  # pragma: no cover - defensive handoff
                error_box["error"] = exc
            finally:
                loop.close()

        thread = threading.Thread(target=_runner, daemon=True)
        thread.start()
        thread.join()
        if error_box:
            raise error_box["error"]
        return box.get("value")

    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(loop)
    try:
        return loop.run_until_complete(awaitable)
    finally:
        loop.close()


def _learning_review_model_candidates() -> list[str]:
    settings = _load_settings()
    return _unique_model_candidates(
        str(settings.get("main_model") or ""),
        str(settings.get("fast_model") or ""),
        str(settings.get("fallback_model") or ""),
    )


def _extract_learning_review_payload(raw_text: str) -> dict:
    cleaned = re.sub(r"<think>[\s\S]*?</think>", "", str(raw_text or "")).strip()
    cleaned = re.sub(r"^```(?:json)?\s*", "", cleaned, flags=re.IGNORECASE)
    cleaned = re.sub(r"\s*```$", "", cleaned).strip()
    match = re.search(r"\{[\s\S]*\}", cleaned)
    if not match:
        return {}
    try:
        payload = json.loads(match.group(0))
    except Exception:
        return {}
    return payload if isinstance(payload, dict) else {}


def _auto_review_learning_candidate(candidate: dict) -> dict:
    default = {
        "approved": False,
        "status": "rejected",
        "reason": "auto_review_unavailable",
        "confidence": float(candidate.get("confidence") or 0.0),
        "storage_target": "learning_candidates",
        "activation_mode": "discarded",
        "review_source": "llm_auto_review",
        "review_model": "",
    }

    impact_type = _clean_text(candidate.get("impact_type") or "observation", limit=80)
    if impact_type not in {"prompt_rule", "planner_rule"}:
        default["reason"] = "not_auto_promotable"
        default["activation_mode"] = "audit_only"
        return default

    signal = _historical_signal(candidate)
    if signal["rejected"] > 0:
        default["reason"] = "matches_previously_rejected_learning"
        return default
    if signal["approved"] > 0:
        default["approved"] = True
        default["status"] = "approved"
        default["reason"] = "matches_previously_approved_learning"
        default["confidence"] = max(default["confidence"], 0.86)
        default["storage_target"] = "learning_rules"
        default["activation_mode"] = "immediate"
        return default

    candidates = _learning_review_model_candidates()
    if not candidates:
        return default

    scope = _task_scope(candidate)
    settings = _load_settings()
    prompt = (
        "Decide whether this proposed agent learning should become a definitive durable rule.\n"
        "Approve only when the learning is specific, reusable across similar tasks, safe, and likely to improve future behavior.\n"
        "Reject vague, overly situational, redundant, risky, or low-signal rules.\n"
        "Return strict JSON with exactly these keys: approved (boolean), confidence (0 to 1), reason (short sentence).\n\n"
        f"Source: {_clean_text(candidate.get('source') or 'unknown', limit=80)}\n"
        f"Kind: {_clean_text(candidate.get('kind') or 'candidate', limit=80)}\n"
        f"Impact type: {impact_type}\n"
        f"Task type: {scope['task_type']}\n"
        f"Provider scope: {scope['provider_scope']}\n"
        f"Time filter: {scope['time_filter']}\n"
        f"Rule text: {_clean_text(candidate.get('rule_text') or '', limit=500)}\n"
        f"Context: {_clean_text(candidate.get('context') or '', limit=220) or '-'}\n"
        f"Original excerpt: {_clean_text(candidate.get('original_excerpt') or '', limit=220) or '-'}\n"
        f"Evidence: {json.dumps(list(candidate.get('evidence') or [])[:5], ensure_ascii=False)}\n"
        f"Historical approved matches: {signal['approved']}\n"
        f"Historical rejected matches: {signal['rejected']}\n"
        f"Historical pending matches: {signal['candidate']}\n"
    )

    try:
        raw_reply, used_model = _run_async_from_sync(
            asyncio.wait_for(
                _call_text_model_candidates(
                    prompt,
                    candidates,
                    system=(
                        "You validate durable agent learnings. "
                        "Return only strict JSON and be conservative about approvals."
                    ),
                    num_ctx=_model_ctx(4096, settings),
                    thinking=False,
                    per_model_timeout=_LEARNING_REVIEW_PER_MODEL_TIMEOUT_SECONDS,
                ),
                timeout=_LEARNING_REVIEW_TIMEOUT_SECONDS,
            )
        )
    except Exception:
        return default

    if _is_llm_error(str(raw_reply or "")):
        return default

    payload = _extract_learning_review_payload(str(raw_reply or ""))
    if not payload:
        default["reason"] = "auto_review_parse_failed"
        return default

    approved = bool(payload.get("approved"))
    try:
        confidence = float(payload.get("confidence") or 0.0)
    except Exception:
        confidence = float(candidate.get("confidence") or 0.0)
    confidence = max(0.0, min(1.0, confidence))
    reason = _clean_text(payload.get("reason") or "", limit=120)
    if not reason:
        reason = "approved_by_llm_review" if approved else "rejected_by_llm_review"

    return {
        "approved": approved,
        "status": "approved" if approved else "rejected",
        "reason": reason,
        "confidence": max(float(candidate.get("confidence") or 0.0), confidence),
        "storage_target": "learning_rules" if approved else "learning_candidates",
        "activation_mode": "immediate" if approved else "discarded",
        "review_source": "llm_auto_review",
        "review_model": _clean_text(used_model or candidates[0], limit=120),
    }


def evaluate_learning_candidate(candidate: dict) -> dict:
    impact_type = _clean_text(candidate.get("impact_type") or "observation", limit=80)
    source = _clean_text(candidate.get("source") or "unknown", limit=80)
    rule_text = _clean_text(candidate.get("rule_text") or "", limit=500)

    decision = {
        "approved": False,
        "status": "rejected",
        "reason": "auto_review_rejected",
        "confidence": float(candidate.get("confidence") or 0.0),
        "storage_target": "learning_candidates",
        "activation_mode": "discarded",
    }

    if not rule_text:
        decision["status"] = "rejected"
        decision["reason"] = "empty_rule_text"
        decision["activation_mode"] = "discarded"
        return decision

    if impact_type not in {"prompt_rule", "planner_rule"}:
        decision["reason"] = "not_auto_promotable"
        decision["activation_mode"] = "audit_only"
        return decision

    if source == "user_feedback":
        decision["approved"] = True
        decision["status"] = "approved"
        decision["reason"] = "explicit_user_correction"
        decision["confidence"] = max(decision["confidence"], 0.95)
        decision["storage_target"] = "learning_rules"
        decision["activation_mode"] = "immediate"
        return decision

    if source == "reviewed_feedback":
        decision["approved"] = True
        decision["status"] = "approved"
        decision["reason"] = "reviewed_feedback"
        decision["confidence"] = max(decision["confidence"], 0.8)
        decision["storage_target"] = "learning_rules"
        decision["activation_mode"] = "immediate"
        return decision

    return _auto_review_learning_candidate(candidate)


def finalize_pending_learning_candidates(max_items: int = 100) -> dict:
    candidates = _load_candidates()
    changed = False
    reviewed = 0
    promoted = 0

    for index, row in enumerate(candidates):
        if reviewed >= max(1, int(max_items or 1)):
            break
        if not isinstance(row, dict):
            continue
        current_status = _clean_text(row.get("status") or "candidate", limit=40) or "candidate"
        if current_status != "candidate":
            continue

        candidate = dict(row)
        decision = evaluate_learning_candidate(candidate)
        final_status = _clean_text(decision.get("status") or current_status, limit=40) or current_status
        if final_status == "candidate":
            continue

        reviewed += 1
        changed = True
        candidate["status"] = final_status
        candidate["review_reason"] = _clean_text(decision.get("reason") or "", limit=120)
        candidate["storage_target"] = _clean_text(decision.get("storage_target") or "learning_candidates", limit=80)
        candidate["activation_mode"] = _clean_text(decision.get("activation_mode") or "discarded", limit=80)
        candidate["reviewed_at"] = datetime.now().isoformat()
        candidate["review_source"] = _clean_text(decision.get("review_source") or "llm_auto_review", limit=80)
        if decision.get("review_model"):
            candidate["review_model"] = _clean_text(decision.get("review_model") or "", limit=120)
        rule = promote_learning_candidate(candidate, decision)
        if isinstance(rule, dict) and rule.get("id"):
            promoted += 1
            candidate["promoted_rule_id"] = str(rule.get("id"))
        candidates[index] = candidate

    if changed:
        _save_candidates(candidates)

    return {"reviewed": reviewed, "promoted": promoted, "changed": changed}


def promote_learning_candidate(candidate: dict, decision: dict) -> dict | None:
    if not bool((decision or {}).get("approved")):
        return None

    store = _load_rules_store()
    rules = list(store.get("rules") or [])
    rule_text = _clean_text(candidate.get("rule_text") or "", limit=500)
    if not rule_text:
        return None

    key = _rule_key(rule_text)
    now_iso = datetime.now().isoformat()
    source = _clean_text(candidate.get("source") or "unknown", limit=80)
    kind = _clean_text(candidate.get("kind") or "candidate", limit=80)
    context = _clean_text(candidate.get("context") or "", limit=220)
    impact_type = _clean_text(candidate.get("impact_type") or "prompt_rule", limit=80)
    rule_scope = {
        "task_type": _clean_text(candidate.get("task_type") or "generic", limit=80) or "generic",
        "provider_scope": _clean_text(candidate.get("provider_scope") or "default", limit=80) or "default",
        "time_filter": _clean_text(candidate.get("time_filter") or "none", limit=80) or "none",
    }

    for row in rules:
        if not isinstance(row, dict):
            continue
        if _rule_key(row.get("rule_text") or "") != key:
            continue
        if _clean_text(row.get("impact_type") or "", limit=80) != impact_type:
            continue
        if any(
            _clean_text(row.get(name) or "", limit=80) != value
            for name, value in rule_scope.items()
        ):
            continue
        row["updated_at"] = now_iso
        row["status"] = "approved"
        row["source_count"] = int(row.get("source_count") or 1) + 1
        if source:
            row["last_source"] = source
        if kind:
            row["last_kind"] = kind
        if context:
            row["last_context"] = context
        row["approval_reason"] = _clean_text((decision or {}).get("reason") or "", limit=120)
        _save_rules_store(store)
        return row

    rule = {
        "id": str(uuid.uuid4()),
        "created_at": now_iso,
        "updated_at": now_iso,
        "status": "approved",
        "impact_type": impact_type,
        "rule_text": rule_text,
        "source": source,
        "kind": kind,
        "context": context,
        **rule_scope,
        "source_count": 1,
        "approval_reason": _clean_text((decision or {}).get("reason") or "", limit=120),
    }
    rules.append(rule)
    store["rules"] = rules
    _save_rules_store(store)
    return rule


def record_learning_candidate(
    *,
    source: str,
    kind: str,
    impact_type: str,
    rule_text: str,
    context: str = "",
    original_excerpt: str = "",
    evidence: list[str] | None = None,
    confidence: float = 0.0,
    task_state: dict | None = None,
) -> dict:
    scope = _task_scope(task_state)
    candidate = {
        "id": str(uuid.uuid4()),
        "timestamp": datetime.now().isoformat(),
        "source": _clean_text(source, limit=80),
        "kind": _clean_text(kind, limit=80),
        "impact_type": _clean_text(impact_type, limit=80),
        "rule_text": _clean_text(rule_text, limit=500),
        "context": _clean_text(context, limit=220),
        "original_excerpt": _clean_text(original_excerpt, limit=220),
        "evidence": [
            _clean_text(item, limit=220)
            for item in (evidence or [])
            if _clean_text(item, limit=220)
        ][:5],
        "confidence": float(confidence or 0.0),
        **scope,
    }
    decision = evaluate_learning_candidate(candidate)
    candidate["status"] = _clean_text(decision.get("status") or "candidate", limit=40) or "candidate"
    candidate["review_reason"] = _clean_text(decision.get("reason") or "", limit=120)
    candidate["storage_target"] = _clean_text(decision.get("storage_target") or "learning_candidates", limit=80)
    candidate["activation_mode"] = _clean_text(decision.get("activation_mode") or "pending_validation", limit=80)
    if candidate["status"] in {"approved", "rejected"}:
        candidate["reviewed_at"] = candidate["timestamp"]
        candidate["review_source"] = _clean_text(decision.get("review_source") or source, limit=80)
        if decision.get("review_model"):
            candidate["review_model"] = _clean_text(decision.get("review_model") or "", limit=120)
    rule = promote_learning_candidate(candidate, decision)
    if isinstance(rule, dict) and rule.get("id"):
        candidate["promoted_rule_id"] = str(rule.get("id"))
    _append_candidate(candidate)
    return {"candidate": candidate, "decision": decision, "rule": rule}


def get_learning_candidates(
    *,
    status: str = "candidate",
    impact_type: str = "",
    source: str = "",
    max_items: int = 50,
) -> list[dict]:
    rows = [_normalize_candidate(row) for row in _load_candidates() if isinstance(row, dict)]
    wanted_status = _clean_text(status, limit=40)
    wanted_impact = _clean_text(impact_type, limit=80)
    wanted_source = _clean_text(source, limit=80)
    result: list[dict] = []
    for row in rows:
        row_status = _clean_text(row.get("status") or "", limit=40)
        if wanted_status in {"reviewed", "final"}:
            if row_status not in {"approved", "rejected"}:
                continue
        elif wanted_status and row_status != wanted_status:
            continue
        if wanted_impact and _clean_text(row.get("impact_type") or "", limit=80) != wanted_impact:
            continue
        if wanted_source and _clean_text(row.get("source") or "", limit=80) != wanted_source:
            continue
        result.append(row)
    result.sort(
        key=lambda row: str(row.get("reviewed_at") or row.get("timestamp") or ""),
        reverse=True,
    )
    return result[: max(1, int(max_items or 1))]


def review_learning_candidate(
    *,
    candidate_id: str,
    approved: bool,
    review_reason: str = "",
    reviewer: str = "manual_review",
) -> dict | None:
    wanted_id = _clean_text(candidate_id, limit=80)
    if not wanted_id:
        return None

    candidates = _load_candidates()
    for index, row in enumerate(candidates):
        if not isinstance(row, dict):
            continue
        if _clean_text(row.get("id") or "", limit=80) != wanted_id:
            continue

        candidate = _normalize_candidate(row)
        reason = _clean_text(
            review_reason or ("manual_approval" if approved else "manual_rejection"),
            limit=120,
        )
        reviewed_at = datetime.now().isoformat()
        decision = {
            "approved": bool(approved),
            "status": "approved" if approved else "rejected",
            "reason": reason,
            "confidence": float(candidate.get("confidence") or 0.0),
        }
        if approved:
            decision["confidence"] = max(decision["confidence"], 0.8)

        rule = promote_learning_candidate(candidate, decision) if approved else None
        candidate["status"] = str(decision["status"])
        candidate["review_reason"] = reason
        candidate["reviewed_at"] = reviewed_at
        candidate["review_source"] = _clean_text(reviewer, limit=80)
        candidate["storage_target"] = "learning_rules" if approved else "learning_candidates"
        candidate["activation_mode"] = "manual_review" if approved else "discarded"
        if isinstance(rule, dict) and rule.get("id"):
            candidate["promoted_rule_id"] = str(rule.get("id"))

        candidates[index] = candidate
        _save_candidates(candidates)
        return {"candidate": candidate, "decision": decision, "rule": rule}

    return None


def get_learning_rules(*, impact_type: str = "", status: str = "approved") -> list[dict]:
    store = _load_rules_store()
    rows = [row for row in (store.get("rules") or []) if isinstance(row, dict)]
    wanted_impact = _clean_text(impact_type, limit=80)
    wanted_status = _clean_text(status, limit=40)
    result: list[dict] = []
    for row in rows:
        if wanted_status and _clean_text(row.get("status") or "", limit=40) != wanted_status:
            continue
        if wanted_impact and _clean_text(row.get("impact_type") or "", limit=80) != wanted_impact:
            continue
        result.append(row)
    return result


def get_applicable_learning_rules(
    *,
    task_state: dict | None = None,
    impact_type: str = "",
    status: str = "approved",
    max_items: int = 10,
) -> list[dict]:
    rules = get_learning_rules(impact_type=impact_type, status=status)
    scope = _task_scope(task_state)

    def _score(row: dict) -> int:
        score = 0
        row_task = _clean_text(row.get("task_type") or "generic", limit=80) or "generic"
        row_provider = _clean_text(row.get("provider_scope") or "default", limit=80) or "default"
        row_time = _clean_text(row.get("time_filter") or "none", limit=80) or "none"
        if row_task == scope["task_type"]:
            score += 4
        elif row_task == "generic":
            score += 1
        else:
            return -1
        if row_provider == scope["provider_scope"]:
            score += 2
        elif row_provider == "default":
            score += 0
        else:
            return -1
        if row_time == scope["time_filter"]:
            score += 1
        elif row_time == "none":
            score += 0
        else:
            return -1
        return score

    ranked = [
        (row, _score(row))
        for row in rules
        if isinstance(row, dict)
    ]
    ranked = [item for item in ranked if item[1] >= 0]
    ranked.sort(
        key=lambda item: (
            item[1],
            int(item[0].get("source_count") or 0),
            str(item[0].get("updated_at") or item[0].get("created_at") or ""),
        ),
        reverse=True,
    )
    return [row for row, _score_value in ranked[: max(1, int(max_items or 1))]]


def get_prompt_learning_rules(max_items: int = 15) -> list[dict]:
    rules = get_learning_rules(impact_type="prompt_rule", status="approved")
    return rules[-max(1, int(max_items or 1)):]