"""
Memory system: semantic embeddings, session CRUD, user profile enrichment,
agent identity (soul/user files, system prompt).
"""

import asyncio
import json
import re
import threading
import time
from datetime import datetime, timedelta
from pathlib import Path

import requests

from backend.config import (
    OLLAMA_URL, MEMORY_DIR, MEMORY_INDEX_FILE, SOUL_FILE, IDENTITY_FILE, USER_FILE,
    NVIDIA_PREFIX, OPENAI_PREFIX, GEMINI_PREFIX,
    _load_settings, _BUILTIN_TOOL_NAMES, _model_ctx,
)
from backend.learning_pipeline import get_prompt_learning_rules, record_learning_candidate
from backend.models import _call_text_model, _call_text_model_candidates, _unique_model_candidates, _is_llm_error

# ── Filler phrase stripping (soul.md §1: no performative help) ────────────────

_FILLER_PATTERNS = [
    re.compile(r"^(?:great\s+question|good\s+question)[.!]?\s*", re.IGNORECASE),
    re.compile(r"^(?:that'?s?\s+a\s+(?:great|good|interesting)\s+question)[.!]?\s*", re.IGNORECASE),
    re.compile(r"^(?:i'?d?\s+be\s+happy\s+to\s+help(?:\s+(?:you\s+)?with\s+that)?)[.!]?\s*", re.IGNORECASE),
    re.compile(r"^(?:sure[,!]?\s*(?:i\s+can\s+help(?:\s+with\s+that)?)?)[.!]?\s*", re.IGNORECASE),
    re.compile(r"^(?:of\s+course[,!]?\s*(?:i\s+can\s+help)?)[.!]?\s*", re.IGNORECASE),
    re.compile(r"^(?:absolutely[,!]?\s*(?:i\s+can\s+help)?)[.!]?\s*", re.IGNORECASE),
    re.compile(r"^(?:certainly[,!]?\s*(?:i\s+can\s+help)?)[.!]?\s*", re.IGNORECASE),
    re.compile(r"^(?:no\s+problem[,!]?\s*)", re.IGNORECASE),
    re.compile(r"^(?:let\s+me\s+(?:help\s+you\s+with\s+that|look\s+into\s+that\s+for\s+you|assist\s+you))[.!]?\s*", re.IGNORECASE),
    re.compile(r"^(?:(?:graag|natuurlijk|zeker)[,!]?\s*(?:ik\s+(?:help|kan)\s+(?:je\s+)?(?:graag|daarmee|daarbij(?:\s+helpen)?)?[.,!]?\s*)?)[.!]?\s*", re.IGNORECASE),
    re.compile(r"^(?:(?:goede|mooie)\s+vraag)[.!]?\s*", re.IGNORECASE),
    re.compile(r"^(?:ik\s+help\s+je\s+(?:graag|daar\s+graag\s+(?:mee|bij)))[.!]?\s*", re.IGNORECASE),
]

_FILLER_CLOSING_PATTERNS = [
    re.compile(r"\s*(?:(?:is\s+there\s+anything\s+else|anything\s+else)\s+(?:I\s+can\s+help\s+(?:you\s+)?with|you(?:'d|\s+would)\s+like\s+(?:to\s+know|me\s+to\s+do))\??)\s*$", re.IGNORECASE),
    re.compile(r"\s*(?:(?:let\s+me\s+know|feel\s+free\s+to\s+ask)\s+if\s+(?:you\s+(?:need|have|want)\s+(?:anything|something)\s+else|there'?s?\s+anything\s+else))[.!]?\s*$", re.IGNORECASE),
    re.compile(r"\s*(?:(?:laat\s+(?:het\s+)?(?:me\s+)?weten|vraag\s+gerust)\s+(?:als|of)\s+(?:je\s+(?:nog\s+)?(?:iets|wat)\s+(?:anders\s+)?(?:wilt|nodig\s+hebt)))[.!]?\s*$", re.IGNORECASE),
    re.compile(r"\s*(?:I\s+hope\s+(?:this|that)\s+helps)[.!]?\s*$", re.IGNORECASE),
    re.compile(r"\s*(?:(?:ik\s+)?hoop\s+dat\s+(?:dit|het)\s+helpt)[.!]?\s*$", re.IGNORECASE),
]


def _strip_filler_phrases(text: str) -> str:
    """Remove performative filler from the start/end of LLM responses (soul.md §1)."""
    if not text:
        return text
    value = text
    for pat in _FILLER_PATTERNS:
        value = pat.sub("", value, count=1)
    for pat in _FILLER_CLOSING_PATTERNS:
        value = pat.sub("", value, count=1)
    value = value.strip()
    return value if value else text


# ── Semantic memory index ─────────────────────────────────────────────────────

def _cosine_similarity(a: list, b: list) -> float:
    if not a or not b or len(a) != len(b):
        return 0.0
    dot   = sum(x * y for x, y in zip(a, b))
    mag_a = sum(x * x for x in a) ** 0.5
    mag_b = sum(x * x for x in b) ** 0.5
    if mag_a == 0.0 or mag_b == 0.0:
        return 0.0
    return dot / (mag_a * mag_b)


def _get_embedding(text: str, model: str | None = None) -> list | None:
    if model is None:
        model = str(_load_settings().get("embed_model") or "").strip()
    if not model:
        return None
    try:
        res = requests.post(
            f"{OLLAMA_URL}/embeddings",
            json={"model": model, "prompt": text[:2000]},
            timeout=15,
        )
        if res.status_code == 200:
            emb = res.json().get("embedding")
            if emb and isinstance(emb, list):
                return emb
    except Exception:
        pass
    return None


_memory_index_cache: dict | None = None
_memory_index_cache_ts: float = 0.0
_MEMORY_INDEX_CACHE_TTL = 10.0
_MEMORY_SUMMARY_MIN_SCORE = 0.25
_MEMORY_SUMMARY_FALLBACK_MIN_SCORE = 0.15
_MEMORY_DEEP_MIN_SCORE = 0.55
_MEMORY_MAX_SUMMARY_MATCHES = 5
_FOLLOWUP_MODEL_TIMEOUT_SECONDS = 45.0
_MEMORY_RECALL_QUERY_RE = re.compile(
    r"\b(?:welke\s+(?:code|informatie|gegevens?)|wat\s+(?:zei\s+ik|noemde\s+ik|was\s+de\s+code)|"
    r"eerdere\s+sessie|vorige\s+sessie|eerder(?:e)?\s+(?:chat|gesprek)|sessiegeheugen|"
    r"remember|recall|what\s+(?:code|did\s+i\s+mention)|previous\s+(?:chat|conversation)|past\s+session)\b",
    re.IGNORECASE,
)


def _followup_model_candidates(settings: dict, session: dict, preferred_model: str = "") -> list[str]:
    return _unique_model_candidates(
        preferred_model,
        session.get("model", ""),
        settings.get("main_model", ""),
        settings.get("fallback_model", ""),
        settings.get("fast_model", ""),
    )


def _memory_query_terms(text: str) -> set[str]:
    words = re.findall(r"\w+", (text or "").lower())
    stopwords = {
        "the", "and", "for", "with", "that", "this", "from", "into", "your",
        "you", "about", "have", "what", "when", "waar", "de", "het", "een",
        "van", "voor", "met", "dat", "dit", "die", "wat", "hoe", "heb", "kun",
        "als", "naar", "mail", "email", "emails",
    }
    return {w for w in words if len(w) >= 3 and w not in stopwords}


def _normalize_memory_metadata(metadata: dict | None) -> dict:
    source = dict(metadata or {})

    def _clean_text(value: object) -> str:
        return re.sub(r"\s+", " ", str(value or "").strip())

    def _clean_list(values: object, limit: int = 12) -> list[str]:
        cleaned: list[str] = []
        seen: set[str] = set()
        for item in values or []:
            text = _clean_text(item)
            if not text:
                continue
            key = text.lower()
            if key in seen:
                continue
            seen.add(key)
            cleaned.append(text)
            if len(cleaned) >= limit:
                break
        return cleaned

    normalized = {
        "display_goal": _clean_text(source.get("display_goal") or source.get("active_goal")),
        "task_type": _clean_text(source.get("task_type")),
        "provider_scope": _clean_text(source.get("provider_scope")),
        "time_filter": _clean_text(source.get("time_filter")),
        "time_label": _clean_text(source.get("time_label")),
        "source_tools": _clean_list(source.get("source_tools") or source.get("tools_used") or []),
        "source_provenance": _clean_list(source.get("source_provenance") or []),
        "result_counts": _clean_list(source.get("result_counts") or []),
        "exact_identifiers": _clean_list(source.get("exact_identifiers") or []),
    }
    return {
        key: value
        for key, value in normalized.items()
        if (isinstance(value, list) and value) or (isinstance(value, str) and value)
    }


def _session_memory_metadata(session: dict) -> dict:
    continuity = _normalize_memory_metadata(session.get("continuity_summary"))
    task = dict(session.get("task_state") or {})
    exact_identifiers: list[str] = []
    for record in list(session.get("last_email_results") or [])[:5]:
        subject = str((record or {}).get("subject") or "").strip()
        if subject:
            exact_identifiers.append(subject)
    for event in list(session.get("last_calendar_results") or [])[:5]:
        title = str((event or {}).get("title") or (event or {}).get("summary") or "").strip()
        if title:
            exact_identifiers.append(title)
    for key in ("time_label", "since", "until"):
        value = str(task.get(key) or "").strip()
        if value:
            exact_identifiers.append(value)
    tool_names = [item["name"] for item in _extract_tools_used(session)]
    fallback = _normalize_memory_metadata({
        "display_goal": str(session.get("goal") or "").strip(),
        "task_type": str(task.get("task_type") or "").strip(),
        "provider_scope": str(task.get("provider_scope") or "").strip(),
        "time_filter": str(task.get("time_filter") or "").strip(),
        "time_label": str(task.get("time_label") or "").strip(),
        "source_tools": tool_names,
        "exact_identifiers": exact_identifiers,
    })
    merged = dict(fallback)
    for key, value in continuity.items():
        merged[key] = value
    return _normalize_memory_metadata(merged)


def _memory_metadata_text(metadata: dict | None) -> str:
    meta = _normalize_memory_metadata(metadata)
    parts: list[str] = []
    for key in ("display_goal", "task_type", "provider_scope", "time_filter", "time_label"):
        value = str(meta.get(key) or "").strip()
        if value:
            parts.append(value)
    for key in ("source_tools", "source_provenance", "result_counts", "exact_identifiers"):
        values = meta.get(key) or []
        if values:
            parts.append(" ".join(str(item) for item in values))
    return " ".join(parts).strip()


def _memory_identifier_hit_count(goal_text: str, metadata: dict | None) -> int:
    goal_lower = str(goal_text or "").lower()
    if not goal_lower:
        return 0
    identifiers = _normalize_memory_metadata(metadata).get("exact_identifiers") or []
    return sum(1 for ident in identifiers if ident.lower() in goal_lower)


def _public_memory_match(match: dict) -> dict:
    metadata = _normalize_memory_metadata(match.get("metadata"))
    payload = {
        "session_id": str(match.get("id") or ""),
        "date": str(match.get("created_at") or "")[:10],
        "goal": (str(match.get("summary") or "") or str(match.get("goal") or ""))[:120],
        "score": round(float(match.get("score", 0.0) or 0.0), 2),
        "completed": bool(match.get("completed", False)),
        "method": str(match.get("method") or ""),
    }
    for key in ("task_type", "provider_scope", "time_filter", "time_label"):
        value = str(metadata.get(key) or "").strip()
        if value:
            payload[key] = value
    for key in ("source_tools", "source_provenance", "result_counts", "exact_identifiers"):
        values = list(metadata.get(key) or [])
        if values:
            payload[key] = values
    return payload


def _is_memory_recall_query(text: str) -> bool:
    return bool(_MEMORY_RECALL_QUERY_RE.search(str(text or "").strip()))


def _memory_match_signature(goal_text: str, summary_text: str, metadata: dict | None = None) -> str:
    metadata_text = _memory_metadata_text(metadata)
    terms = sorted(_memory_query_terms(f"{goal_text} {metadata_text}"))
    if not terms:
        terms = sorted(_memory_query_terms(f"{summary_text} {metadata_text}"))
    if not terms:
        raw = re.sub(r"\s+", " ", " ".join(part for part in (goal_text, summary_text, metadata_text) if part).strip().lower())
        return raw[:120]
    return "|".join(terms[:10])


def _memory_match_sort_key(match: dict) -> tuple:
    return (
        1 if match.get("completed") else 0,
        float(match.get("score", 0.0) or 0.0),
        str(match.get("created_at", "") or ""),
    )


def _select_distinct_memory_matches(ranked: list[dict], limit: int = _MEMORY_MAX_SUMMARY_MATCHES) -> list[dict]:
    grouped: dict[str, dict] = {}
    for match in ranked:
        signature = _memory_match_signature(match.get("goal", ""), match.get("summary", ""), match.get("metadata"))
        current = grouped.get(signature)
        if current is None or _memory_match_sort_key(match) > _memory_match_sort_key(current):
            grouped[signature] = match

    distinct = sorted(grouped.values(), key=_memory_match_sort_key, reverse=True)
    return distinct[:limit]


def _rank_memory_matches(current_goal: str, limit: int = 5) -> tuple[list[dict], str]:
    index = _load_memory_index()
    if not index:
        return [], "none"

    query_emb = _get_embedding(current_goal) if current_goal.strip() else None
    using_semantic = query_emb is not None
    goal_words = _memory_query_terms(current_goal)
    current_is_memory_recall = _is_memory_recall_query(current_goal)
    ranked: list[dict] = []

    for sid, meta in index.items():
        goal_text = str(meta.get("goal", "") or "")
        summary_text = str(meta.get("summary", "") or "")
        metadata = _normalize_memory_metadata(meta.get("metadata"))
        metadata_text = _memory_metadata_text(metadata)
        candidate_words = _memory_query_terms(f"{goal_text} {summary_text} {metadata_text}")
        lexical_overlap = 0.0
        if goal_words:
            lexical_overlap = len(goal_words & candidate_words) / max(len(goal_words), 1)

        if using_semantic:
            emb = meta.get("embedding")
            if not emb:
                # Session lacks embeddings (e.g. embed model was down during indexing).
                # Fall back to keyword-only scoring instead of skipping entirely.
                if not goal_words and not candidate_words:
                    continue
                score = lexical_overlap
                method = "keyword"
            else:
                semantic_score = _cosine_similarity(query_emb, emb)
                score = (semantic_score * 0.82) + (lexical_overlap * 0.18)
                method = "semantic"
        else:
            if not goal_words and not candidate_words:
                continue
            score = lexical_overlap
            method = "keyword"

        if meta.get("completed"):
            score += 0.03

        identifier_hits = _memory_identifier_hit_count(current_goal, metadata)
        if identifier_hits:
            score += min(0.24, 0.12 * identifier_hits)

        candidate_is_memory_recall = _is_memory_recall_query(f"{goal_text}\n{summary_text}")
        if current_is_memory_recall and candidate_is_memory_recall and not bool(meta.get("completed", False)):
            # Earlier failed recall attempts are usually dead ends. When the current
            # query asks to remember a prior session, prefer the source session over
            # older unresolved recall questions that merely restate the same ask.
            score -= 0.35

        ranked.append({
            "id": sid,
            "score": score,
            "goal": goal_text,
            "summary": summary_text,
            "created_at": str(meta.get("created_at", "") or ""),
            "completed": bool(meta.get("completed", False)),
            "method": method,
            "metadata": metadata,
        })

    ranked.sort(key=lambda item: item["score"], reverse=True)
    return ranked[:limit], ("semantic" if using_semantic else "keyword")


def _recall_memory_bundle(current_goal: str, limit: int = 5) -> dict:
    ranked, method = _rank_memory_matches(current_goal, limit=limit)
    if not ranked:
        return {
            "summary": "",
            "summary_count": 0,
            "deep_memory": "",
            "deep_session_id": "",
            "method": method,
            "matches": [],
            "citations": [],
        }

    summary_matches = [m for m in ranked if m["score"] >= _MEMORY_SUMMARY_MIN_SCORE]
    if not summary_matches and ranked[0]["score"] >= _MEMORY_SUMMARY_FALLBACK_MIN_SCORE:
        summary_matches = [ranked[0]]
    summary_matches = _select_distinct_memory_matches(summary_matches, limit=_MEMORY_MAX_SUMMARY_MATCHES)

    if summary_matches:
        lines = [f"Past sessions (most relevant, {method} match — use recall_memory skill for full details):"]
        for match in summary_matches:
            ts = match["created_at"][:10]
            status = "completed" if match["completed"] else "partial"
            summary = match["summary"] or match["goal"]
            lines.append(
                f"  [{ts} | {status} | score={match['score']:.2f} | id={match['id']}] {summary}"
            )
        summary_text = "\n".join(lines)
    else:
        summary_text = ""

    deep_memory = ""
    deep_session_id = ""
    if summary_matches and summary_matches[0]["score"] >= _MEMORY_DEEP_MIN_SCORE:
        deep_session_id = summary_matches[0]["id"]
        deep_memory = _recall_memory_deep(session_id=deep_session_id)

    # Build citation metadata for provenance tracking
    citations = []
    public_matches = []
    for match in summary_matches:
        cite = _public_memory_match(match)
        # Extract tools only for the deep-memory session (top match) to stay lightweight
        if match["id"] == deep_session_id:
            deep_session = _load_session(deep_session_id)
            if deep_session:
                cite["tools_used"] = [t["name"] for t in _extract_tools_used(deep_session)]
        citations.append(cite)
        public_matches.append(dict(cite))

    # Track recall usage for aging
    recalled_ids = [m["id"] for m in summary_matches]
    if recalled_ids:
        threading.Thread(target=_update_recall_stats, args=(recalled_ids,), daemon=True).start()

    return {
        "summary": summary_text,
        "summary_count": len(public_matches),
        "deep_memory": deep_memory,
        "deep_session_id": deep_session_id,
        "method": method,
        "matches": public_matches,
        "citations": citations,
    }


def _load_memory_index() -> dict:
    global _memory_index_cache, _memory_index_cache_ts
    now = time.time()
    if _memory_index_cache is not None and (now - _memory_index_cache_ts) < _MEMORY_INDEX_CACHE_TTL:
        return _memory_index_cache
    if MEMORY_INDEX_FILE.exists():
        try:
            result = json.loads(MEMORY_INDEX_FILE.read_text(encoding="utf-8"))
            _memory_index_cache = result
            _memory_index_cache_ts = now
            return result
        except Exception:
            pass
    return {}


def _save_memory_index(index: dict):
    global _memory_index_cache, _memory_index_cache_ts
    try:
        MEMORY_INDEX_FILE.write_text(json.dumps(index, ensure_ascii=False), encoding="utf-8")
        _memory_index_cache = index
        _memory_index_cache_ts = time.time()
    except Exception:
        pass


def _generate_session_summary(session: dict) -> str:
    """Generate a 1-2 sentence summary of a session via LLM (fast model)."""
    goal = (session.get("goal") or "").strip()
    steps = session.get("steps") or []
    chat_history = session.get("chat_history") or []
    completed = session.get("completed", False)
    recent_chat_lines = []
    for msg in chat_history[-4:]:
        role = "User" if msg.get("role") == "user" else "Assistant"
        content = str(msg.get("content") or "").strip()
        if content:
            recent_chat_lines.append(f"- {role}: {content[:220]}")
    recent_chat = "\n".join(recent_chat_lines)
    last_assistant = ""
    for msg in reversed(chat_history):
        if msg.get("role") == "assistant":
            last_assistant = str(msg.get("content") or "").strip()
            if last_assistant:
                break
    fallback = f"{goal} ({len(steps)} steps, {'completed' if completed else 'partial'})"
    if last_assistant:
        fallback = f"{goal} -> {last_assistant[:220]}"
    if not goal:
        return fallback
    try:
        settings = _load_settings()
        fast_m = (settings.get("fast_model") or settings.get("main_model") or "").strip()
        if not fast_m:
            return fallback
        steps_text = "\n".join(
            f"- {s.get('step', '')}" for s in steps[:8]
        )
        prompt = (
            "Summarize the following agent session in 1-2 concise sentences. "
            "Focus on WHAT was accomplished and the key outcome.\n\n"
            f"Goal: {goal}\n"
            f"Status: {'completed' if completed else 'partial'}\n"
            f"Steps:\n{steps_text or '(none)'}\n\n"
            f"Recent chat:\n{recent_chat or '(none)'}\n\n"
            "Summary:"
        )
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
        try:
            result = loop.run_until_complete(
                _call_text_model(prompt, fast_m, system="You are a concise summarizer.", num_ctx=2048)
            )
        finally:
            loop.close()
        summary = (result or "").strip()
        summary = re.sub(r'<think>[\s\S]*?</think>', '', summary).strip()
        if summary and len(summary) > 10:
            return summary[:300]
    except Exception as e:
        print(f"[memory] Summary generation failed: {e}")
    return fallback


def _index_session(session: dict):
    sid  = session.get("id")
    goal = session.get("goal", "").strip()
    if not sid or not goal:
        return
    steps_hint = " | ".join(s.get("step", "") for s in session.get("steps", [])[:6])
    chat_hint = " | ".join(
        f"{'user' if msg.get('role') == 'user' else 'assistant'}: {str(msg.get('content') or '').strip()[:160]}"
        for msg in (session.get("chat_history") or [])[-4:]
        if str(msg.get("content") or "").strip()
    )
    metadata = _session_memory_metadata(session)
    metadata_hint = _memory_metadata_text(metadata)
    embed_hint = " | ".join(part for part in (steps_hint, chat_hint, metadata_hint) if part)
    embed_text = f"{goal}. {embed_hint}" if embed_hint else goal
    emb = _get_embedding(embed_text)
    if emb is None:
        return
    summary = _generate_session_summary(session)
    index = _load_memory_index()
    index[sid] = {
        "goal":       goal,
        "embedding":  emb,
        "created_at": session.get("created_at", ""),
        "completed":  session.get("completed", False),
        "summary":    summary,
        "metadata":   metadata,
    }
    _save_memory_index(index)


def _reindex_all_sessions():
    index = _load_memory_index()
    changed = False
    for f in MEMORY_DIR.glob("*.json"):
        if f.name.startswith("_"):
            continue
        try:
            s = json.loads(f.read_text(encoding="utf-8"))
            sid = s.get("id")
            has_steps = bool(s.get("steps"))
            has_chat = any(str(msg.get("content") or "").strip() for msg in (s.get("chat_history") or []))
            if not sid or (not has_steps and not has_chat):
                continue
            if s.get("no_learn"):
                continue
            goal = s.get("goal", "").strip()
            if not goal:
                continue
            existing = index.get(sid, {})
            needs_embed = sid not in index
            needs_summary = not existing.get("summary")
            metadata = _session_memory_metadata(s)
            needs_metadata = _normalize_memory_metadata(existing.get("metadata")) != metadata
            if not needs_embed and not needs_summary and not needs_metadata:
                continue
            emb = existing.get("embedding")
            if needs_embed or needs_metadata:
                steps_hint = " | ".join(st.get("step", "") for st in s.get("steps", [])[:6])
                chat_hint = " | ".join(
                    f"{'user' if msg.get('role') == 'user' else 'assistant'}: {str(msg.get('content') or '').strip()[:160]}"
                    for msg in (s.get("chat_history") or [])[-4:]
                    if str(msg.get("content") or "").strip()
                )
                metadata_hint = _memory_metadata_text(metadata)
                embed_hint = " | ".join(part for part in (steps_hint, chat_hint, metadata_hint) if part)
                embed_text = f"{goal}. {embed_hint}" if embed_hint else goal
                emb = _get_embedding(embed_text)
                if not emb:
                    continue
            summary = existing.get("summary", "")
            if needs_summary:
                summary = _generate_session_summary(s)
            index[sid] = {
                "goal":       goal,
                "embedding":  emb,
                "created_at": s.get("created_at", ""),
                "completed":  s.get("completed", False),
                "summary":    summary,
                "metadata":   metadata,
            }
            changed = True
            action = "indexed+summarized" if needs_embed else ("metadata-refreshed" if needs_metadata and not needs_summary else "summarized")
            print(f"[memory] {action} session {sid[:8]}... '{goal[:60]}'")
        except Exception:
            pass
    for sid, entry in list(index.items()):
        if _normalize_memory_metadata(entry.get("metadata")):
            continue
        fallback_metadata = _normalize_memory_metadata({
            "display_goal": str(entry.get("goal") or entry.get("summary") or "").strip(),
            "exact_identifiers": [str(entry.get("summary") or "").strip()] if str(entry.get("summary") or "").strip() else [],
        })
        if not fallback_metadata:
            continue
        entry["metadata"] = fallback_metadata
        index[sid] = entry
        changed = True
    if changed:
        _save_memory_index(index)
    print("[memory] Reindex complete.")


# ── User profile enrichment ──────────────────────────────────────────────────

def _extract_user_facts(text: str) -> list[str]:
    if not text:
        return []
    facts: list[str] = []
    raw = text.strip()
    _NAME = r"[A-Za-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF][A-Za-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF'\-]*"
    _NAMES = rf"{_NAME}(?:\s+{_NAME}){{0,3}}"
    _VAL = r"([^\n\.!?]{2,80})"

    # -- Auto-detect communication language --
    _NL_MARKERS = re.compile(
        r"\b(hallo|hoi|bedankt|alsjeblieft|graag|goedemorgen|goedemiddag|"
        r"goedenavond|dankjewel|dankuwel|ja\b|nee\b|misschien|natuurlijk|"
        r"ik wil|ik heb|ik ben|ik zoek|kun je|kunt u|wat is|hoe kan|"
        r"even|dus|omdat|maar|ook|voor|niet|meer|deze|mijn)\b",
        re.IGNORECASE,
    )
    nl_hits = len(_NL_MARKERS.findall(raw))
    if nl_hits >= 2:
        facts.append("Communication language: Dutch")

    # -- Name --
    _NAME_STOP = {"en", "ik", "in", "van", "de", "het", "een", "die", "dat", "maar",
                  "ook", "of", "als", "om", "te", "bij", "woon", "werk",
                  "and", "i", "the", "a", "an", "is", "am", "from", "who", "live", "work"}
    for pat in [
        rf"\bik heet\s+({_NAMES})",
        rf"\bmijn naam is\s+({_NAMES})",
        rf"\bmy name is\s+({_NAMES})",
        rf"\bi'?m\s+({_NAMES})\b",
        rf"\bnoem me\s+({_NAMES})",
        rf"\bcall me\s+({_NAMES})",
    ]:
        m = re.search(pat, raw, re.IGNORECASE)
        if m:
            parts = m.group(1).strip().split()
            while parts and parts[-1].lower() in _NAME_STOP:
                parts.pop()
            val = " ".join(parts)
            if len(val) > 1 and val.lower() not in ("a", "an", "the", "de", "het", "een"):
                facts.append(f"Name: {val}")
            break

    # -- Location / timezone --
    for pat in [
        rf"\bik woon in\s+{_VAL}", rf"\bik kom uit\s+{_VAL}",
        rf"\bi live in\s+{_VAL}", rf"\bi'?m from\s+{_VAL}",
        rf"\bik ben gevestigd in\s+{_VAL}", rf"\bbased in\s+{_VAL}",
        rf"\bmy city is\s+{_VAL}", rf"\bmijn stad is\s+{_VAL}",
    ]:
        m = re.search(pat, raw, re.IGNORECASE)
        if m:
            facts.append(f"Location: {m.group(1).strip(' :,-')}")
            break
    for pat in [r"\bmy timezone is\s+" + _VAL, r"\bmijn tijdzone is\s+" + _VAL]:
        m = re.search(pat, raw, re.IGNORECASE)
        if m:
            facts.append(f"Timezone: {m.group(1).strip()}")
            break

    # -- Work / profession --
    for pat in [
        rf"\bik werk bij\s+{_VAL}", rf"\bik werk als\s+{_VAL}",
        rf"\bik ben een?\s+{_VAL}", rf"\bi work (?:at|for)\s+{_VAL}",
        rf"\bi'?m a\s+{_VAL}", rf"\bmy (?:job|role|title|position) is\s+{_VAL}",
        rf"\bmijn functie is\s+{_VAL}", rf"\bmijn beroep is\s+{_VAL}",
        rf"\bi work as\s+{_VAL}",
    ]:
        m = re.search(pat, raw, re.IGNORECASE)
        if m:
            facts.append(f"Work: {m.group(1).strip(' :,-')}")
            break

    # -- Company / organisation --
    for pat in [
        rf"\bmijn bedrijf is\s+{_VAL}", rf"\bmy company is\s+{_VAL}",
        rf"\bi work (?:at|for)\s+{_VAL}", rf"\bik werk voor\s+{_VAL}",
    ]:
        m = re.search(pat, raw, re.IGNORECASE)
        if m:
            facts.append(f"Company: {m.group(1).strip(' :,-')}")
            break

    # -- Language preference --
    for pat in [
        rf"\bik spreek (?:graag |het liefst )?{_VAL}",
        rf"\bmy (?:preferred |native )?language is\s+{_VAL}",
        rf"\bik praat (?:het liefst )?{_VAL}",
        rf"\bantwoord (?:alsjeblieft |graag )?in het\s+{_VAL}",
        rf"\breply in\s+{_VAL}",
    ]:
        m = re.search(pat, raw, re.IGNORECASE)
        if m:
            facts.append(f"Language: {m.group(1).strip(' :,-')}")
            break

    # -- Hobbies / interests --
    for pat in [
        rf"\bmijn hobbies? zijn\s+{_VAL}", rf"\bmijn hobby(?:'s)? is\s+{_VAL}",
        rf"\bmy hobbies? (?:are|is)\s+{_VAL}",
        rf"\bi (?:like|enjoy|love)\s+{_VAL}",
        rf"\bik hou van\s+{_VAL}", rf"\bik vind .* leuk\b",
        rf"\bmijn interesses zijn\s+{_VAL}",
        rf"\bi'?m (?:interested|passionate) (?:in|about)\s+{_VAL}",
    ]:
        m = re.search(pat, raw, re.IGNORECASE)
        if m:
            val = m.group(1).strip(" :,-") if m.lastindex else ""
            if val:
                facts.append(f"Interests: {val}")
            break

    # -- Age / birthday --
    for pat in [
        r"\bik ben\s+(\d{1,3})\s*(?:jaar|years?)",
        r"\bi'?m\s+(\d{1,3})\s*(?:years? old)",
        r"\bmy age is\s+(\d{1,3})",
        r"\bmijn leeftijd is\s+(\d{1,3})",
    ]:
        m = re.search(pat, raw, re.IGNORECASE)
        if m:
            facts.append(f"Age: {m.group(1)}")
            break
    for pat in [
        rf"\bmijn verjaardag is\s+{_VAL}", rf"\bmy birthday is\s+{_VAL}",
        rf"\bik ben geboren op\s+{_VAL}", rf"\bi was born on\s+{_VAL}",
    ]:
        m = re.search(pat, raw, re.IGNORECASE)
        if m:
            facts.append(f"Birthday: {m.group(1).strip()}")
            break

    # -- Email --
    m = re.search(r"\bmy e?-?mail(?:\s+(?:is|address))?\s*[:=]?\s*([^\s,;]+@[^\s,;]+)", raw, re.IGNORECASE)
    if not m:
        m = re.search(r"\bmijn e?-?mail(?:\s+(?:is|adres))?\s*[:=]?\s*([^\s,;]+@[^\s,;]+)", raw, re.IGNORECASE)
    if m:
        facts.append(f"Email: {m.group(1).strip()}")

    # -- Tools / tech stack --
    for pat in [
        rf"\bik gebruik\s+{_VAL}", rf"\bi use\s+{_VAL}",
        rf"\bmy (?:tech )?stack (?:is|includes)\s+{_VAL}",
        rf"\bik werk met\s+{_VAL}",
    ]:
        m = re.search(pat, raw, re.IGNORECASE)
        if m:
            facts.append(f"Tools: {m.group(1).strip(' :,-')}")
            break

    # -- Communication / style preferences --
    for pat in [
        rf"\bik wil (?:graag )?(?:dat je |dat u )?\s*{_VAL}",
        rf"\bplease (?:always |usually )?{_VAL}",
        rf"\bik prefereer\s+{_VAL}",
        rf"\bi prefer\s+{_VAL}",
    ]:
        m = re.search(pat, raw, re.IGNORECASE)
        if m:
            facts.append(f"Preference: {m.group(1).strip(' :,-')}")
            break

    # -- Family --
    for pat in [
        rf"\bik heb\s+(\d+)\s*(?:kind(?:eren)?|kids?|children)",
        rf"\bi have\s+(\d+)\s*(?:kids?|children)",
    ]:
        m = re.search(pat, raw, re.IGNORECASE)
        if m:
            facts.append(f"Family: {m.group(1)} children")
            break

    # -- Relationship / household --
    for pat in [
        rf"\bik woon samen met\s+({_NAMES})",
        rf"\bik woon met\s+({_NAMES})\s+samen",
        rf"\bi live together with\s+({_NAMES})",
        rf"\bi live with\s+({_NAMES})",
        rf"\bmijn partner is\s+({_NAMES})",
        rf"\bmy partner is\s+({_NAMES})",
    ]:
        m = re.search(pat, raw, re.IGNORECASE)
        if m:
            facts.append(f"Lives with: {m.group(1).strip()}")
            break

    # -- OS / platform --
    for pat in [
        rf"\bik gebruik\s+(windows|mac|linux|macos|ubuntu)[^\n]*",
        rf"\bi (?:use|run|am on)\s+(windows|mac|linux|macos|ubuntu)[^\n]*",
        rf"\bmy (?:os|operating system) is\s+{_VAL}",
    ]:
        m = re.search(pat, raw, re.IGNORECASE)
        if m:
            facts.append(f"OS: {m.group(1).strip()}")
            break

    return facts


def _update_user_profile_from_text(text: str) -> list[str]:
    facts = _extract_user_facts(text)
    if not facts:
        print(f"[user-profile] regex extraction: no facts found in: {text[:80]!r}")
        return []
    try:
        existing = USER_FILE.read_text(encoding="utf-8") if USER_FILE.exists() else "# User Profile\n\n## Facts\n"
    except Exception:
        existing = "# User Profile\n\n## Facts\n"
    if "## Facts" not in existing:
        existing = existing.rstrip() + "\n\n## Facts\n"
    existing_lower = existing.lower()
    to_add = [fact for fact in facts if fact.lower() not in existing_lower]
    if not to_add:
        print(f"[user-profile] regex extraction: facts already present: {facts}")
        return []
    updated = existing.rstrip() + "\n" + "\n".join(f"- {fact}" for fact in to_add) + "\n"
    try:
        USER_FILE.write_text(updated, encoding="utf-8")
        print(f"[user-profile] regex extraction added {len(to_add)} fact(s): {to_add}")
    except Exception as e:
        print(f"[user-profile] regex extraction write failed: {e}")
        return []
    return to_add


def _enrich_user_profile_llm(conversation_text: str):
    if not conversation_text or len(conversation_text) < 20:
        print("[user-profile] LLM enrichment skipped: text too short.")
        return
    try:
        settings = _load_settings()
        fast_m = (settings.get("fast_model") or settings.get("main_model") or "").strip()
        if not fast_m:
            return
        if fast_m.startswith(NVIDIA_PREFIX):
            if not (settings.get("nvidia_api_key") or "").strip():
                return
        elif fast_m.startswith(OPENAI_PREFIX):
            if not (settings.get("openai_api_key") or "").strip():
                return
        elif fast_m.startswith(GEMINI_PREFIX):
            if not (settings.get("gemini_api_key") or "").strip():
                return
        existing = ""
        try:
            existing = USER_FILE.read_text(encoding="utf-8") if USER_FILE.exists() else ""
        except Exception:
            pass
        system_prompt = (
            "You are an internal user-profile enrichment agent. "
            "Given a conversation between a user and an AI assistant, extract NEW facts or insights about the user "
            "that are NOT already in the existing profile.\n"
            "Focus on:\n"
            "- Explicit facts: name, location, timezone, language, work/role/company, email, age, family\n"
            "- Implicit patterns: preferred communication language (Dutch/English), expertise areas, "
            "technical tools/stack, recurring topics of interest, industry/domain knowledge\n"
            "- Preferences: communication style, level of detail they prefer, "
            "how they phrase requests, what kind of output format they like\n"
            "- Goals & projects: what they are working on, long-term objectives, current priorities\n\n"
            "Return ONLY a bullet list of new facts (one per line, starting with '- '). "
            "Keep each fact concise (one line). "
            "If there are no new facts, return exactly: NONE"
        )
        prompt = (
            f"=== EXISTING PROFILE ===\n{existing or '(empty)'}\n\n"
            f"=== CONVERSATION ===\n{conversation_text[:6000]}\n\n"
            "Extract new user insights not already in the profile:"
        )
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
        try:
            print(f"[user-profile] LLM enrichment starting with model={fast_m}")
            result = loop.run_until_complete(
                _call_text_model(prompt, fast_m, system=system_prompt, num_ctx=4096, thinking=False)
            )
            print(f"[user-profile] LLM enrichment result: {result[:200] if result else '(empty)'}")
        finally:
            loop.close()
        if not result or "NONE" in result.upper().split("\n")[0]:
            return
        new_facts = []
        for line in result.strip().splitlines():
            line = line.strip()
            if line.startswith("- "):
                fact = line[2:].strip()
                if fact and len(fact) > 2:
                    new_facts.append(fact)
        if not new_facts:
            return
        try:
            current = USER_FILE.read_text(encoding="utf-8") if USER_FILE.exists() else "# User Profile\n\n## Facts\n"
        except Exception:
            current = "# User Profile\n\n## Facts\n"
        if "## Insights" not in current:
            current = current.rstrip() + "\n\n## Insights\n"
        current_lower = current.lower()
        to_add = [f for f in new_facts if f.lower() not in current_lower]
        if not to_add:
            return
        updated = current.rstrip() + "\n" + "\n".join(f"- {fact}" for fact in to_add) + "\n"
        USER_FILE.write_text(updated, encoding="utf-8")
        print(f"[user-profile] LLM enrichment added {len(to_add)} insight(s): {to_add}")

        # Compact user.md if it has grown too large
        try:
            line_count = updated.count("\n")
            if line_count > 40:
                _compact_user_profile(fast_m)
        except Exception:
            pass

    except Exception as e:
        import traceback
        print(f"[user-profile] LLM enrichment failed: {e}")
        traceback.print_exc()


def _compact_user_profile(model: str = ""):
    """Deduplicate and compress user.md via LLM when it grows beyond ~40 lines."""
    try:
        if not USER_FILE.exists():
            return
        current = USER_FILE.read_text(encoding="utf-8").strip()
        if not current or current.count("\n") <= 30:
            return
        if not model:
            settings = _load_settings()
            model = (settings.get("fast_model") or settings.get("main_model") or "").strip()
        if not model:
            return
        prompt = (
            "Compact the following user profile by:\n"
            "1. Removing duplicate or near-duplicate facts\n"
            "2. Merging related facts into single concise lines\n"
            "3. Keeping the most important/recent information\n"
            "4. Preserving the ## Facts and ## Insights section headers\n"
            "5. Target: max 25 bullet points total\n\n"
            f"=== CURRENT PROFILE ===\n{current}\n\n"
            "Return the compacted profile (maintain markdown format with ## headers and - bullet points):"
        )
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
        try:
            result = loop.run_until_complete(
                _call_text_model(prompt, model, system="You are a concise editor. Return only the compacted profile.", num_ctx=4096)
            )
        finally:
            loop.close()
        result = (result or "").strip()
        result = re.sub(r'<think>[\s\S]*?</think>', '', result).strip()
        if not result or len(result) < 20 or "## Facts" not in result:
            return
        if not result.startswith("#"):
            result = "# User Profile\n\n" + result
        USER_FILE.write_text(result + "\n", encoding="utf-8")
        print(f"[user-profile] Compacted: {current.count(chr(10))} -> {result.count(chr(10))} lines")
    except Exception as e:
        print(f"[user-profile] Compaction failed: {e}")


# ── Memory consolidation ─────────────────────────────────────────────────────

def _consolidate_memory_sessions(similarity_threshold: float = 0.85):
    """Merge near-duplicate sessions in the memory index.

    Sessions with embedding cosine similarity above threshold and the same
    completion status get merged: the newer summary replaces the older, and
    the older session file is archived.
    """
    index = _load_memory_index()
    if len(index) < 2:
        return
    entries = [(sid, data) for sid, data in index.items() if data.get("embedding")]
    merged_away: set[str] = set()
    for i, (sid_a, data_a) in enumerate(entries):
        if sid_a in merged_away:
            continue
        emb_a = data_a["embedding"]
        for j in range(i + 1, len(entries)):
            sid_b, data_b = entries[j]
            if sid_b in merged_away:
                continue
            emb_b = data_b.get("embedding")
            if not emb_b:
                continue
            sim = _cosine_similarity(emb_a, emb_b)
            if sim < similarity_threshold:
                continue
            # Same completion status?
            if data_a.get("completed") != data_b.get("completed"):
                continue
            # Keep the newer one
            ts_a = data_a.get("created_at", "")
            ts_b = data_b.get("created_at", "")
            keep, drop = (sid_a, sid_b) if ts_a >= ts_b else (sid_b, sid_a)
            merged_away.add(drop)
            # Archive the dropped session file
            drop_path = MEMORY_DIR / f"{drop}.json"
            archive_dir = MEMORY_DIR / "archive"
            archive_dir.mkdir(exist_ok=True)
            if drop_path.exists():
                try:
                    drop_path.rename(archive_dir / f"{drop}.json")
                except Exception:
                    pass
            print(f"[memory] Consolidated {drop[:8]} into {keep[:8]} (similarity={sim:.2f})")
    if merged_away:
        for sid in merged_away:
            index.pop(sid, None)
        _save_memory_index(index)
        print(f"[memory] Consolidation complete: merged {len(merged_away)} duplicate(s)")


# ── Episode-level recall ─────────────────────────────────────────────────────

def _episode_recall(entity_a: str, entity_b: str = "", limit: int = 5) -> list[dict]:
    """Find sessions where specific entities (people, topics) were discussed.

    Searches session goals, summaries, and chat history for co-occurrence
    of entity_a (and optionally entity_b).
    Returns matching sessions sorted by recency.
    """
    entity_a = entity_a.strip().lower()
    entity_b = entity_b.strip().lower() if entity_b else ""
    if not entity_a:
        return []
    index = _load_memory_index()
    matches: list[dict] = []
    for sid, data in index.items():
        searchable = f"{data.get('goal', '')} {data.get('summary', '')}".lower()
        if entity_a not in searchable:
            continue
        if entity_b and entity_b not in searchable:
            # Also check the full session if summary doesn't have entity_b
            full = _load_session(sid)
            if full:
                chat_text = " ".join(
                    str(m.get("content", "")) for m in (full.get("chat_history") or [])
                ).lower()
                if entity_b not in chat_text:
                    continue
            else:
                continue
        matches.append({
            "id": sid,
            "goal": data.get("goal", ""),
            "summary": data.get("summary", ""),
            "created_at": data.get("created_at", ""),
            "completed": data.get("completed", False),
        })
    matches.sort(key=lambda m: m.get("created_at", ""), reverse=True)
    return matches[:limit]


# ── Preference enforcement ───────────────────────────────────────────────────

_PREFERENCE_CACHE: dict[str, str] = {}
_PREFERENCE_CACHE_TS: float = 0.0
_PREFERENCE_CACHE_TTL = 120.0  # 2 min


def _get_user_preferences() -> str:
    """Extract actionable preferences from user.md to inject into LLM system prompts.

    Reads the ## Insights and ## Facts sections for preference-like statements
    (language, style, format, tone) and returns them as bullet points.
    """
    global _PREFERENCE_CACHE, _PREFERENCE_CACHE_TS
    now = time.time()
    if _PREFERENCE_CACHE and (now - _PREFERENCE_CACHE_TS) < _PREFERENCE_CACHE_TTL:
        return _PREFERENCE_CACHE.get("text", "")
    try:
        if not USER_FILE.exists():
            return ""
        content = USER_FILE.read_text(encoding="utf-8")
    except Exception:
        return ""
    # Extract lines that look like preferences/patterns
    pref_keywords = {
        "prefer", "always", "never", "like", "want", "style", "tone", "format",
        "language", "taal", "voorkeur", "altijd", "nooit", "liever", "graag",
        "communication", "communicatie", "reply", "respond", "brief", "detailed",
    }
    lines: list[str] = []
    for line in content.splitlines():
        stripped = line.strip().lstrip("-•* ")
        if not stripped or stripped.startswith("#"):
            continue
        lower = stripped.lower()
        if any(kw in lower for kw in pref_keywords):
            lines.append(f"- {stripped}")
    result = "\n".join(lines[:15])  # cap at 15 preferences
    _PREFERENCE_CACHE = {"text": result}
    _PREFERENCE_CACHE_TS = now
    return result


# ── Session management ────────────────────────────────────────────────────────
_UUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.IGNORECASE)


def _is_persistable_session_id(session_id: str) -> bool:
    return bool(_UUID_RE.match(str(session_id or "").strip()))


def _session_path(session_id: str) -> Path:
    if not _UUID_RE.match(session_id):
        raise ValueError(f"Invalid session id: {session_id!r}")
    return MEMORY_DIR / f"{session_id}.json"


def _load_session(session_id: str) -> dict | None:
    try:
        p = _session_path(session_id)
    except ValueError:
        return None
    if not p.exists():
        return None
    try:
        return json.loads(p.read_text(encoding="utf-8"))
    except Exception:
        return None


def _save_session(session: dict):
    p = _session_path(session["id"])
    p.write_text(json.dumps(session, ensure_ascii=False, indent=2), encoding="utf-8")
    if not session.get("no_learn"):
        threading.Thread(target=_index_session, args=(dict(session),), daemon=True).start()


def _session_followup_context(session: dict, max_steps: int = 6, max_chars: int = 14000) -> str:
    lines: list[str] = []
    goal = (session.get("goal") or "").strip()
    if goal:
        lines.append(f"Original goal: {goal}")
    final_report = (session.get("final_report") or "").strip()
    if final_report:
        lines.append("\nFinal report:")
        lines.append(final_report)
    steps = session.get("steps") or []
    if steps:
        lines.append("\nRecent execution steps:")
        for idx, st in enumerate(steps[-max_steps:], start=max(1, len(steps) - max_steps + 1)):
            step_txt = str(st.get("step") or "").strip()
            res_txt = str(st.get("result") or "").strip()
            if len(res_txt) > 800:
                res_txt = res_txt[:800] + "..."
            lines.append(f"{idx}. {step_txt}")
            if res_txt:
                lines.append(f"   Result: {res_txt}")
    out = "\n".join(lines).strip()
    if len(out) > max_chars:
        out = out[:max_chars] + "\n..."
    return out


async def _generate_session_followup(session: dict, user_message: str, preferred_model: str = "", tools_desc: str = "", extra_context: str = "", past_memory_text: str = "") -> tuple[str, str, str]:
    settings = _load_settings()
    model_candidates = _followup_model_candidates(settings, session, preferred_model)
    if not model_candidates:
        raise ValueError("No model configured for follow-up chat.")
    history = list(session.get("chat_history") or [])
    history.append({"role": "user", "content": user_message})
    recent_history = history[-30:]
    history_lines = []
    for msg in recent_history:
        role = "User" if msg.get("role") == "user" else "Assistant"
        content = str(msg.get("content") or "").strip()
        if content:
            history_lines.append(f"{role}: {content}")
    transcript = "\n".join(history_lines)
    followup_context = _session_followup_context(session)
    user_context = (session.get("context") or "").strip()
    has_rich_context = bool(user_context)
    _turns_since_workflow = session.get("_chat_turns_since_workflow", 0)
    _workflow_is_stale = bool(_turns_since_workflow > 0 and followup_context and followup_context != "(none)")

    prompt = (
        "You are continuing a conversation.\n"
    )
    if _workflow_is_stale and has_rich_context:
        prompt += (
            "The user previously ran a workflow, but the conversation has since moved on.\n"
            "A document or other rich context was provided more recently.\n"
            "PRIORITIZE the recent chat and user-provided context over the older workflow context.\n"
            "Only use tools if the conversation context does NOT contain the answer.\n\n"
        )
    else:
        prompt += (
            "Use the session context below as source of truth and answer the user's follow-up clearly.\n"
        )
    prompt += (
        "If the follow-up asks you to DO something (send email, search, fetch data, create/update/delete "
        "calendar events, modify files, etc.) you MUST use the available tools via TOOL_CALL IMMEDIATELY.\n"
        "NEVER say you cannot do it. NEVER ask the user to do it themselves. NEVER ask for permission \n"
        "or access — all tools listed in your system prompt are already authorized.\n"
        "If the user refers to something from the conversation (e.g. 'that appointment'), resolve it \n"
        "from the chat history and session context — do NOT ask the user to repeat information.\n"
        "If the latest user turn is a short follow-up fragment, infer the omitted subject from the recent chat and treat it as a continuation of the current topic unless the user explicitly switches topics.\n"
        "Do NOT jump to workspace/file/code search tools for a short ambiguous follow-up unless the user explicitly asks about files, code, workspace, repo, folder, or documents.\n"
        "If you need data first (e.g. an event ID), call the appropriate read/list tool first, \n"
        "then call the update/delete tool with the result.\n"
        "If the follow-up is a simple question you can answer from context, answer directly without tools.\n\n"
        "IMPORTANT: Be concise and direct. Give the answer immediately - no lengthy reasoning, "
        "no step-by-step analysis, no preamble. Short paragraphs or bullet points preferred. "
        "Aim for the shortest correct answer.\n\n"
    )
    if _workflow_is_stale:
        prompt += f"[OLDER WORKFLOW CONTEXT (earlier task — conversation has moved on)]\n{followup_context}\n\n"
    else:
        prompt += f"[SESSION CONTEXT]\n{followup_context or '(none)'}\n\n"
    if user_context:
        prompt += f"[USER-PROVIDED CONTEXT]\n{user_context[:16000]}\n\n"
    if extra_context:
        prompt += f"[ADDITIONAL CONTEXT]\n{extra_context[:8000]}\n\n"
    prompt += f"[RECENT CHAT]\n{transcript or '(none)'}\n\n"
    if past_memory_text:
        prompt += (
            f"[LONG-TERM MEMORY — prior conversations, supplemental only]\n"
            f"{past_memory_text}\n"
            f"Note: use this only if the user explicitly references a past conversation. "
            f"The current chat above always takes priority.\n\n"
        )
    prompt += f"[NEW USER QUESTION]\n{user_message}\n"
    system_prompt = _compose_agent_system()
    if tools_desc:
        system_prompt += tools_desc
    ctx_size = _model_ctx(65536, settings) if has_rich_context else _model_ctx(32768, settings)
    _want_thinking = settings.get("show_thinking", False)
    answer, used_model = await _call_text_model_candidates(
        prompt,
        model_candidates,
        system=system_prompt,
        num_ctx=ctx_size,
        thinking=_want_thinking,
        per_model_timeout=_FOLLOWUP_MODEL_TIMEOUT_SECONDS,
    )
    answer = answer.strip()
    if _is_llm_error(answer):
        raise RuntimeError(answer)
    raw_answer = answer  # Preserve thinking blocks for caller
    answer = re.sub(r'<think>[\s\S]*?</think>', '', answer).strip()
    if not answer:
        answer = "I have no answer yet for this follow-up."
    answer = _strip_filler_phrases(answer)
    history.append({"role": "assistant", "content": answer})
    session["chat_history"] = history
    session["updated_at"] = datetime.now().isoformat()
    _save_session(session)
    return answer, used_model, raw_answer


async def _compact_chat_history(session: dict, model_candidates: list[str]) -> dict:
    """Compact a long chat history into a structured summary + recent messages.

    Progressive: keeps 8 recent messages intact (up from 4) so the user
    doesn't lose immediate conversational context.  Can re-compact: if the
    history already contains an earlier compacted summary, it folds the old
    summary into the new one.
    """
    history = list(session.get("chat_history") or [])
    if len(history) <= 10:
        return {"compacted": False, "reason": "History too short to compact"}

    # Keep last 8 messages intact for immediate context
    keep_recent = 8
    old_messages = history[:-keep_recent]
    recent_messages = history[-keep_recent:]

    # Build transcript of old messages for the LLM
    transcript_lines = []
    for msg in old_messages:
        role = "User" if msg.get("role") == "user" else "Assistant"
        content = str(msg.get("content") or "").strip()
        if content:
            transcript_lines.append(f"{role}: {content}")
    transcript = "\n".join(transcript_lines)

    prompt = (
        "Summarize the following conversation history into a structured note.\n"
        "Use exactly these sections (omit a section only if it has no content):\n\n"
        "## Key Decisions\n"
        "## Open Items / TODOs\n"
        "## Constraints & Rules established\n"
        "## Important identifiers (names, IDs, paths, URLs)\n\n"
        "Keep it concise but preserve ALL actionable details, exact values, and identifiers.\n"
        "Write in the same language as the conversation.\n\n"
        f"--- CONVERSATION ---\n{transcript}\n--- END ---"
    )
    system = "You produce structured conversation summaries. Be concise and factual."

    summary, _used_model = await _call_text_model_candidates(
        prompt, model_candidates, system=system, num_ctx=_model_ctx(16384),
        thinking=False, per_model_timeout=_FOLLOWUP_MODEL_TIMEOUT_SECONDS,
    )
    summary = re.sub(r'<think>[\s\S]*?</think>', '', summary).strip()
    if not summary or _is_llm_error(summary):
        return {"compacted": False, "reason": "LLM failed to generate summary"}

    before_count = len(history)
    compacted_entry = {
        "role": "assistant",
        "content": f"[Earlier conversation summary]\n{summary}",
        "compacted": True,
    }
    session["chat_history"] = [compacted_entry] + recent_messages
    session["compacted_at"] = datetime.now().isoformat()
    session["updated_at"] = datetime.now().isoformat()
    _save_session(session)

    return {
        "compacted": True,
        "before": before_count,
        "after": len(session["chat_history"]),
        "summary_preview": summary[:200],
    }


def _extract_tools_used(session_data: dict) -> list[dict]:
    """Extract unique tool names from last_tool_trace + steps."""
    names: set[str] = set()
    for item in session_data.get("last_tool_trace") or []:
        if isinstance(item, dict) and item.get("tool_name"):
            names.add(str(item["tool_name"]).strip())
    for step in session_data.get("steps") or []:
        if isinstance(step, dict) and step.get("tool_name"):
            names.add(str(step["tool_name"]).strip())
    names.discard("")
    return [{"name": n, "builtin": n in _BUILTIN_TOOL_NAMES} for n in sorted(names)]


def _list_sessions() -> list[dict]:
    sessions = []
    for f in sorted(MEMORY_DIR.glob("*.json"), key=lambda x: x.stat().st_mtime, reverse=True):
        try:
            s = json.loads(f.read_text(encoding="utf-8"))
            sessions.append({
                "id": s["id"],
                "goal": s.get("goal", ""),
                "model": s.get("model", ""),
                "created_at": s.get("created_at", ""),
                "completed": s.get("completed", False),
                "step_count": len(s.get("steps", [])),
                "source": s.get("source", ""),
                "tools_used": _extract_tools_used(s),
                "no_learn": bool(s.get("no_learn", False)),
            })
        except Exception:
            pass
    return sessions


# ── Memory pruning / archiving ────────────────────────────────────────────────

_MEMORY_ARCHIVE_DIR = MEMORY_DIR / "archive"
_PRUNE_AGE_DAYS = 30

# ── Corrections log (corrigeren) ──────────────────────────────────────────────

_CORRECTIONS_FILE = MEMORY_DIR / "corrections.json"
_MAX_CORRECTIONS = 50


def _load_corrections() -> list[dict]:
    try:
        if _CORRECTIONS_FILE.exists():
            data = json.loads(_CORRECTIONS_FILE.read_text(encoding="utf-8"))
            if isinstance(data, list):
                return data
    except Exception:
        pass
    return []


def _save_corrections(corrections: list[dict]):
    try:
        _CORRECTIONS_FILE.write_text(
            json.dumps(corrections[-_MAX_CORRECTIONS:], ensure_ascii=False, indent=2),
            encoding="utf-8",
        )
    except Exception as e:
        print(f"[corrections] Save failed: {e}")


def _add_correction(original: str, correction: str, context: str = "", source: str = "user_feedback"):
    original_short = str(original or "").strip()[:220]
    correction_short = str(correction or "").strip()[:220]
    context_short = str(context or "").strip()[:220]
    source_short = str(source or "user_feedback").strip()[:80] or "user_feedback"
    entry = {
        "timestamp": datetime.now().isoformat(),
        "original": original[:500],
        "correction": correction[:500],
        "context": context[:300],
        "source": source_short,
    }
    corrections = _load_corrections()
    corrections.append(entry)
    _save_corrections(corrections)
    log_system_event(
        "correction",
        f"{original[:80]} → {correction[:80]}",
        {
            "context": context_short,
            "original_excerpt": original_short,
            "correction_excerpt": correction_short,
            "source": source_short,
        },
    )
    print(f"[corrections] Stored: {original[:60]!r} → {correction[:60]!r}")


def _detect_and_store_correction(user_message: str, previous_assistant: str, session_goal: str = ""):
    """Use fast LLM to detect if user_message is correcting the assistant."""
    if not user_message or not previous_assistant:
        return
    if len(user_message) < 5:
        return
    try:
        settings = _load_settings()
        fast_m = (settings.get("fast_model") or settings.get("main_model") or "").strip()
        if not fast_m:
            return
        prompt = (
            "Determine if the user is correcting, disagreeing with, or fixing something the assistant said.\n"
            "Examples of corrections: 'nee, ik bedoel X', 'dat klopt niet', 'niet Y maar Z', "
            "'no, I meant X', 'that's wrong', 'actually it should be X'\n\n"
            f"Assistant said: {previous_assistant[:800]}\n"
            f"User replied: {user_message[:800]}\n\n"
            "If this IS a correction, respond with exactly:\n"
            "CORRECTION: <what was wrong> -> <what is correct>\n"
            "If this is NOT a correction (just a follow-up question, new topic, etc.), respond with exactly:\n"
            "NONE"
        )
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
        try:
            result = loop.run_until_complete(
                _call_text_model(prompt, fast_m, system="You detect user corrections. Be strict — only flag genuine corrections.", num_ctx=2048, thinking=False)
            )
        finally:
            loop.close()
        result = re.sub(r'<think>[\s\S]*?</think>', '', (result or "")).strip()
        if result and result.upper().startswith("CORRECTION:"):
            correction_text = result[len("CORRECTION:"):].strip()
            _add_correction(
                original=previous_assistant[:300],
                correction=correction_text,
                context=session_goal,
                source="user_feedback",
            )
            record_learning_candidate(
                source="user_feedback",
                kind="correction",
                impact_type="prompt_rule",
                rule_text=correction_text,
                context=session_goal,
                original_excerpt=previous_assistant[:220],
                evidence=[user_message[:220]],
                confidence=0.95,
                task_state=None,
            )
            _invalidate_agent_system_cache()
    except Exception as e:
        print(f"[corrections] Detection failed: {e}")


def _format_corrections_for_prompt(max_items: int = 15) -> str:
    corrections = _load_corrections()
    approved_rules = get_prompt_learning_rules(max_items=max_items)
    seen: set[str] = set()
    lines = [
        "=== Learned Corrections ===",
        "Apply these approved lessons and prior user corrections when they fit the current task:",
    ]

    for rule in approved_rules:
        text = " ".join(str(rule.get("rule_text") or "").strip().split())
        key = text.casefold()
        if not text or key in seen:
            continue
        seen.add(key)
        lines.append(f"- [approved] {text}")

    for c in corrections[-max_items:]:
        text = " ".join(str(c.get("correction") or "").strip().split())
        key = text.casefold()
        if not text or key in seen:
            continue
        seen.add(key)
        date = str(c.get("timestamp", ""))[:10]
        lines.append(f"- [{date}] {text}")

    if len(lines) <= 2:
        return ""
    return "\n".join(lines)


# ── Post-run self-reflection (verbeteren) ─────────────────────────────────────

def _run_self_reflection(session: dict):
    """Evaluate a completed session and store lessons learned."""
    session_id = str(session.get("id") or "").strip()
    persist_reflection = _is_persistable_session_id(session_id)
    goal = (session.get("goal") or "").strip()
    final_report = (session.get("final_report") or "").strip()
    completed = session.get("completed", False)
    steps = session.get("steps") or []
    task_state = session.get("task_state") if isinstance(session.get("task_state"), dict) else None
    if not goal or not steps:
        return
    try:
        settings = _load_settings()
        fast_m = (settings.get("fast_model") or settings.get("main_model") or "").strip()
        if not fast_m:
            return

        steps_text = "\n".join(
            f"- {s.get('step', '')}: {(s.get('result') or '')[:200]}"
            for s in steps[:10]
        )
        prompt = (
            "Reflect on this completed agent session. Be specific and actionable.\n\n"
            f"Goal: {goal}\n"
            f"Status: {'completed' if completed else 'incomplete'}\n"
            f"Steps:\n{steps_text}\n"
            f"Final report:\n{(final_report or '(none)')[:2000]}\n\n"
            "Answer these three questions (1 sentence each):\n"
            "1. WORKED: What approach worked well that should be repeated?\n"
            "2. IMPROVE: What could be done better next time?\n"
            "3. PATTERN: Is this a recurring pattern worth remembering? If yes, what rule?\n\n"
            "Format:\n"
            "WORKED: ...\n"
            "IMPROVE: ...\n"
            "PATTERN: ... (or PATTERN: NONE if no pattern)\n"
        )
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
        try:
            result = loop.run_until_complete(
                _call_text_model(prompt, fast_m, system="You reflect on AI agent runs to improve future performance. Be concise.", num_ctx=4096, thinking=False)
            )
        finally:
            loop.close()
        result = re.sub(r'<think>[\s\S]*?</think>', '', (result or "")).strip()
        if not result:
            return

        reflection = {"timestamp": datetime.now().isoformat(), "raw": result[:1000]}
        for line in result.splitlines():
            line = line.strip()
            if line.upper().startswith("WORKED:"):
                reflection["worked"] = line[7:].strip()[:300]
            elif line.upper().startswith("IMPROVE:"):
                reflection["improve"] = line[8:].strip()[:300]
            elif line.upper().startswith("PATTERN:"):
                pattern_text = line[8:].strip()
                if pattern_text.upper() != "NONE":
                    reflection["pattern"] = pattern_text[:300]

        session["reflection"] = reflection
        if persist_reflection:
            _save_session(session)
        log_system_event(
            "reflection",
            f"Session {session.get('id', '?')[:8]}: {reflection.get('worked', '-')[:120]}",
            {
                "session_id": str(session.get("id") or "")[:80],
                "goal": goal[:220],
                "completed": str(bool(completed)),
                "step_count": str(len(steps)),
                "worked": reflection.get("worked", ""),
                "improve": reflection.get("improve", ""),
                "pattern": reflection.get("pattern", ""),
                "final_report_excerpt": (final_report or "")[:220],
            },
        )
        print(f"[reflection] Session {session.get('id', '?')[:8]}: worked={reflection.get('worked', '-')[:60]}")

        reflection_candidates = []
        if reflection.get("worked"):
            reflection_candidates.append(
                record_learning_candidate(
                    source="self_reflection",
                    kind="reflection_worked",
                    impact_type="observation",
                    rule_text=reflection.get("worked", ""),
                    context=goal,
                    original_excerpt=(final_report or "")[:220],
                    evidence=[steps_text[:220]],
                    confidence=0.3,
                    task_state=task_state,
                )
            )
        if reflection.get("improve"):
            reflection_candidates.append(
                record_learning_candidate(
                    source="self_reflection",
                    kind="reflection_improve",
                    impact_type="planner_rule",
                    rule_text=reflection.get("improve", ""),
                    context=goal,
                    original_excerpt=(final_report or "")[:220],
                    evidence=[steps_text[:220]],
                    confidence=0.35,
                    task_state=task_state,
                )
            )
        if reflection.get("pattern"):
            reflection_candidates.append(
                record_learning_candidate(
                    source="self_reflection",
                    kind="reflection_pattern",
                    impact_type="prompt_rule",
                    rule_text=reflection.get("pattern", ""),
                    context=goal,
                    original_excerpt=(final_report or "")[:220],
                    evidence=[steps_text[:220]],
                    confidence=0.35,
                    task_state=task_state,
                )
            )
        if reflection_candidates:
            if any(
                isinstance(item, dict)
                and bool((item.get("decision") or {}).get("approved"))
                and str((item.get("candidate") or {}).get("impact_type") or "") == "prompt_rule"
                for item in reflection_candidates
            ):
                _invalidate_agent_system_cache()
            reflection["learning_candidates"] = [
                {
                    "candidate_id": str((item.get("candidate") or {}).get("id") or ""),
                    "status": str((item.get("decision") or {}).get("status") or "candidate"),
                    "reason": str((item.get("decision") or {}).get("reason") or ""),
                }
                for item in reflection_candidates
                if isinstance(item, dict)
            ]
            session["reflection"] = reflection
            if persist_reflection:
                _save_session(session)
    except Exception as e:
        print(f"[reflection] Failed: {e}")


# ── Memory aging & usage tracking (opruimen) ─────────────────────────────────

def _update_recall_stats(session_ids: list[str]):
    """Bump recall_count and last_recalled for sessions that were matched."""
    if not session_ids:
        return
    index = _load_memory_index()
    changed = False
    now_iso = datetime.now().isoformat()
    for sid in session_ids:
        entry = index.get(sid)
        if not entry:
            continue
        entry["recall_count"] = (entry.get("recall_count") or 0) + 1
        entry["last_recalled"] = now_iso
        changed = True
    if changed:
        _save_memory_index(index)


def _prune_with_usage(max_age_days: int = _PRUNE_AGE_DAYS) -> dict:
    """Enhanced pruning that considers usage frequency.

    - Never-recalled sessions: archive after max_age_days
    - Rarely-recalled (1-2 times): archive after max_age_days * 1.5
    - Frequently-recalled (3+): archive after max_age_days * 3
    - Also purges archived entries older than 180 days from the index entirely
    """
    index = _load_memory_index()
    if not index:
        return {"archived": 0, "cleaned_index": 0, "purged": 0, "errors": []}

    now = datetime.now()
    _MEMORY_ARCHIVE_DIR.mkdir(exist_ok=True)
    archived = 0
    purged = 0
    errors: list[str] = []

    for f in sorted(MEMORY_DIR.glob("*.json")):
        if f.name.startswith("_") or f.name == "corrections.json":
            continue
        try:
            data = json.loads(f.read_text(encoding="utf-8"))
            if not isinstance(data, dict):
                continue
            created = data.get("created_at", "")
            if not created:
                continue
            try:
                created_dt = datetime.fromisoformat(created)
            except (ValueError, TypeError):
                continue

            sid = data.get("id", f.stem)
            entry = index.get(sid, {})
            recall_count = entry.get("recall_count", 0) or 0

            # Determine effective age threshold based on usage
            if recall_count >= 3:
                effective_max = max_age_days * 3
            elif recall_count >= 1:
                effective_max = int(max_age_days * 1.5)
            else:
                effective_max = max_age_days

            age_days = (now - created_dt).days
            if age_days < effective_max:
                continue

            dest = _MEMORY_ARCHIVE_DIR / f.name
            f.rename(dest)
            archived += 1

            if sid in index:
                idx_entry = index[sid]
                index[sid] = {
                    "goal": idx_entry.get("goal", ""),
                    "created_at": idx_entry.get("created_at", ""),
                    "completed": idx_entry.get("completed", False),
                    "summary": idx_entry.get("summary", ""),
                    "metadata": dict(idx_entry.get("metadata") or {}),
                    "recall_count": idx_entry.get("recall_count", 0),
                    "archived": True,
                }
        except Exception as e:
            errors.append(f"{f.name}: {e}")

    # Purge very old archived index entries (>180 days) to keep index lean
    purge_ids = []
    for sid, entry in index.items():
        if not entry.get("archived"):
            continue
        created = entry.get("created_at", "")
        if not created:
            purge_ids.append(sid)
            continue
        try:
            created_dt = datetime.fromisoformat(created)
            if (now - created_dt).days > 180:
                purge_ids.append(sid)
        except (ValueError, TypeError):
            pass
    for sid in purge_ids:
        del index[sid]
        purged += 1

    # Remove orphan entries
    orphan_ids = []
    for sid in list(index.keys()):
        entry = index[sid]
        if entry.get("archived"):
            continue
        if not (MEMORY_DIR / f"{sid}.json").exists():
            orphan_ids.append(sid)
    for sid in orphan_ids:
        del index[sid]

    if archived or purged or orphan_ids:
        _save_memory_index(index)
        log_system_event("memory_aging", f"Prune: {archived} archived, {purged} purged, {len(orphan_ids)} orphans", {
            "max_age_days": str(max_age_days),
        })
        print(f"[memory] Usage-aware prune: {archived} archived, {purged} purged from index, {len(orphan_ids)} orphans removed")

    return {"archived": archived, "cleaned_index": len(orphan_ids), "purged": purged, "errors": errors}


def _recall_memory_deep(session_id: str = "", query: str = "") -> str:
    """Layer 3 recall: load full session details for deep inspection.
    If session_id is given, load that session directly.
    If only query is given, find the best-matching session and return its details.
    """
    if session_id:
        session = _load_session(session_id)
        if not session:
            return f"Session {session_id} not found."
    elif query:
        ranked, _method = _rank_memory_matches(query, limit=1)
        if not ranked:
            return "No sessions found for query."
        best_match = ranked[0]
        best_sid = best_match["id"]
        if best_match["score"] < _MEMORY_SUMMARY_FALLBACK_MIN_SCORE:
            return "No matching session found."
        session = _load_session(best_sid)
        if not session:
            return f"Session file for {best_sid} not found."
    else:
        return "Provide session_id or query."

    lines = [f"Session: {session.get('goal', '')}"]
    lines.append(f"Status: {'completed' if session.get('completed') else 'partial'}")
    lines.append(f"Created: {session.get('created_at', '')}")
    lines.append(f"Model: {session.get('model', '')}")
    report = (session.get("final_report") or "").strip()
    if report:
        lines.append(f"\nFinal report:\n{report[:4000]}")
    chat_history = session.get("chat_history") or []
    if chat_history:
        lines.append("\nRecent chat:")
        for msg in chat_history[-6:]:
            role = "User" if msg.get("role") == "user" else "Assistant"
            content = str(msg.get("content") or "").strip()
            if content:
                lines.append(f"  {role}: {content[:500]}")
    steps = session.get("steps") or []
    if steps:
        lines.append(f"\nSteps ({len(steps)} total):")
        for i, st in enumerate(steps, 1):
            step_txt = st.get("step", "")
            res_txt = (st.get("result") or "")[:600].replace("\n", " ")
            lines.append(f"  {i}. {step_txt}")
            if res_txt:
                lines.append(f"     -> {res_txt}")
    out = "\n".join(lines)
    return out[:12000]


# ── Agent identity & system prompt ────────────────────────────────────────────
def _get_installed_packages() -> str:
    import subprocess, sys
    try:
        out = subprocess.check_output(
            [sys.executable, "-m", "pip", "list", "--format=columns"],
            text=True, timeout=10,
        )
        lines = out.strip().splitlines()[2:]
        names = [l.split()[0] for l in lines if l.strip()]
        return ", ".join(names)
    except Exception:
        return "unknown"


_INSTALLED_PACKAGES = _get_installed_packages()


def _load_soul_text() -> str:
    try:
        if SOUL_FILE.exists():
            return SOUL_FILE.read_text(encoding="utf-8").strip()
    except Exception:
        pass
    return ""


def _load_user_text() -> str:
    try:
        if USER_FILE.exists():
            return USER_FILE.read_text(encoding="utf-8").strip()
        if IDENTITY_FILE.exists():
            return IDENTITY_FILE.read_text(encoding="utf-8").strip()
    except Exception:
        pass
    return ""


AGENT_SYSTEM = f"""\
You are an autonomous AI assistant with persistent memory and a
self-extending tool library.  Your personality, name, tone, and values
are defined in your soul.md file — follow those instructions faithfully.
If soul.md changes, your behaviour changes with it.

Your capabilities:
- PLANNER   : You decompose any goal into a clear, ordered sequence of steps.
- EXECUTOR  : You carry out each step thoroughly, using all available context.
- MEMORY    : You have access to past sessions you have completed. You actively
              recall and apply what you learned - reuse successful patterns,
              avoid known pitfalls, and build on previous results.
- WEB       : When internet access is enabled you can search the web and fetch
              URLs to retrieve up-to-date information.
- VISION    : When images are attached you can analyse their content.
- SKILLS    : You have a self-extending skill library. When an existing skill fits
              the task, use it directly. When no existing skill is sufficient,
              use create_skill (alias of create_tool) to write and permanently
              register a new reusable skill. New skills are saved to tools/ and
              available in ALL future sessions. Always prefer a general-purpose
              skill over ad-hoc code.

Python environment - already installed packages (do NOT install these again):
{_INSTALLED_PACKAGES}

Behaviour guidelines:
- Always stay focused on the current goal.
- When past memory is provided, reference it explicitly and build upon it.
- Be concise and actionable. Do not add filler or disclaimers.
- NEVER start a response with pleasantries, affirmations, or filler. Start with
  the substance: the answer, the data, the action taken.
- If context or memory is relevant, say so - e.g. "Based on a previous session..."
- Only use run_shell to install a package if it is NOT in the list above.
- When a task requires actually doing something (move files, send data, compute
  results), ALWAYS use a tool - never just describe what you would do.
- When a task could benefit from a reusable skill, create it so future sessions
  benefit automatically.

Data grounding rules:
- When conversation history is provided, use it to understand the user's true
  intent — resolve pronouns, follow-up references, and implicit context.
- For factual claims about current events, dates, scores, prices, or real-world
  state: ONLY state facts that come from provided data sources (web search
  results, email/calendar content, file content, tool output).
- NEVER present training-data knowledge as current fact. If no data source
  confirms the information, say "I could not verify this" or use a tool
  (e.g. web_search) to look it up first.
- When answering, clearly distinguish between verified data and general knowledge.
""".strip()


def _local_tz_offset() -> str:
    """Return the local UTC offset string, e.g. '+01:00' or '-05:00'."""
    offset_seconds = -time.timezone if time.daylight == 0 else -time.altzone
    sign = "+" if offset_seconds >= 0 else "-"
    hours, remainder = divmod(abs(offset_seconds), 3600)
    minutes = remainder // 60
    return f"{sign}{hours:02d}:{minutes:02d}"


_agent_system_cache: tuple[float, str] = (0.0, "")
_AGENT_SYSTEM_CACHE_TTL = 30.0


def _invalidate_agent_system_cache() -> None:
    global _agent_system_cache
    _agent_system_cache = (0.0, "")


def invalidate_agent_system_cache() -> None:
    _invalidate_agent_system_cache()


def _current_date_line() -> str:
    """Return a compact date/time string for injection into any prompt."""
    from datetime import datetime as _dt
    now = _dt.now()
    tz_off = _local_tz_offset()
    return f"Today is {now.strftime('%A, %d %B %Y %H:%M')} (UTC{tz_off}, ISO {now.strftime('%Y-%m-%dT%H:%M')})."


def _compose_agent_system() -> str:
    global _agent_system_cache
    now = time.time()
    if _agent_system_cache[1] and (now - _agent_system_cache[0]) < _AGENT_SYSTEM_CACHE_TTL:
        return _agent_system_cache[1]
    tz_off = _local_tz_offset()
    date_line = _current_date_line()
    date_block = (
        f"=== Current date & time ===\n"
        f"{date_line}\n"
        f"IMPORTANT: You ALWAYS know today's date/time — they are shown above. "
        f"NEVER create a planning step to 'determine', 'look up', or 'retrieve' the current date or time. "
        f"Use the date above directly in any tool calls or reasoning that require it.\n"
        f"When presenting times to the user, ALWAYS use the local timezone (UTC{tz_off}). "
        f"NEVER convert times to a different timezone unless the user explicitly asks.\n\n"
    )
    system = date_block + AGENT_SYSTEM
    soul_text = _load_soul_text()
    user_text = _load_user_text()
    if soul_text:
        system = f"{system}\n\n=== Personality (soul.md) ===\n{soul_text}"
    if user_text:
        system = f"{system}\n\n=== Known User Data (user.md) ===\n{user_text}"
    corrections_text = _format_corrections_for_prompt()
    if corrections_text:
        system = f"{system}\n\n{corrections_text}"
    result = system.strip()
    _agent_system_cache = (now, result)
    return result


# ── System log ────────────────────────────────────────────────────────────────

_SYSTEM_LOG_FILE = MEMORY_DIR / "system_log.jsonl"
_SYSTEM_LOG_MAX_DAYS = 30


def _system_log_explanation(entry: dict) -> str:
    cat = str((entry or {}).get("cat") or "").strip().lower()
    msg = str((entry or {}).get("msg") or "").strip()
    details = dict((entry or {}).get("details") or {})
    goal_preview = str(details.get("goal_preview") or "").strip()

    if cat == "hypothesis":
        if "initialized" in msg.lower():
            policy_mode = str(details.get("policy_mode") or "unknown")
            base = "Situation: new run started with initial working hypothesis."
            signal = f"Signal: policy_mode={policy_mode}."
            if goal_preview:
                signal += f" Goal context: '{goal_preview[:120]}'."
            return (
                f"{base}\n"
                f"{signal}\n"
                "Decision: keep hypothesis tracking active during the run.\n"
                "Adjustment: update confidence and route per step based on tool outcomes."
            )
        if "switched" in msg.lower():
            before_h = str(details.get("active_before") or "")
            after_h = str(details.get("active_after") or str(details.get("switched_to") or ""))
            before_c = str(details.get("confidence_before") or "")
            after_c = str(details.get("confidence_after") or "")
            return (
                "Situation: active hypothesis switched.\n"
                f"Signal: {before_h or 'previous hypothesis'} ({before_c or '?'}) -> {after_h or 'new hypothesis'} ({after_c or '?'}).\n"
                "Decision: activate alternate path.\n"
                "Adjustment: route changed due to confidence drop."
            )
        outcome = str(details.get("outcome") or "")
        step_index = str(details.get("step_index") or "?")
        tool = str(details.get("tool") or "unknown tool")
        conf_before = str(details.get("confidence_before") or "?")
        conf_after = str(details.get("confidence_after") or "?")
        decision_mode = str(details.get("decision_mode") or "")
        remediation_mode = str(details.get("remediation_mode") or "")
        if outcome == "support":
            signal_outcome = "result supports current hypothesis"
        elif outcome == "weak":
            signal_outcome = "evidence is limited / weak"
        elif outcome == "error":
            signal_outcome = "error signal lowers confidence"
        else:
            signal_outcome = "neutral signal"
        context = f"Signal: step={step_index}, tool={tool}, {signal_outcome}, confidence {conf_before} -> {conf_after}."
        if decision_mode:
            context += f" Decision mode={decision_mode}."
        if remediation_mode:
            context += f" Remediation={remediation_mode}."
        if goal_preview:
            context += f" Goal context='{goal_preview[:120]}'."
        return (
            "Situation: hypothesis evidence after tool step.\n"
            f"{context}\n"
            f"Decision: mode={decision_mode or 'unknown'}.\n"
            f"Adjustment: {remediation_mode or 'no extra adjustment'}."
        )

    if cat == "run_kpi":
        completed = str(details.get("completed") or "").lower() == "true"
        halted = str(details.get("halted") or "").lower() == "true"
        steps = str(details.get("steps") or "0")
        calls = str(details.get("tool_calls") or "0")
        first_pass = str(details.get("first_pass_success") or "")
        corr_rate = str(details.get("correction_rate") or "")
        tool_eff = str(details.get("tool_call_efficiency") or "")
        runtime_ms = str(details.get("time_to_solution_ms") or "")
        halt_reason = str(details.get("halt_reason") or "")
        if completed and not halted:
            return (
                "Situation: run completed successfully.\n"
                f"Signal: steps={steps}, tool_calls={calls}, runtime_ms={runtime_ms or '?'}, first_pass={first_pass or '?'}, correction_rate={corr_rate or '?'}, tool_efficiency={tool_eff or '?'}.\n"
                "Decision: result is sufficient to finish.\n"
                "Adjustment: KPI values stored as reference for next runs."
            )
        if halted:
            return (
                "Situation: run stopped early.\n"
                f"Signal: halt_reason={halt_reason or 'unknown'}, steps={steps}, tool_calls={calls}, correction_rate={corr_rate or '?'}, tool_efficiency={tool_eff or '?'}.\n"
                "Decision: stop at policy/budget boundary.\n"
                "Adjustment: KPI values indicate where widening or better routing is needed."
            )
        return (
            "Situation: KPI interim snapshot recorded.\n"
            f"Signal: steps={steps}, tool_calls={calls}, first_pass={first_pass or '?'}, correction_rate={corr_rate or '?'}, tool_efficiency={tool_eff or '?'}.\n"
            "Decision: run still in progress.\n"
            "Adjustment: continue monitoring efficiency and corrections."
        )

    if cat == "learning":
        task_type = str(details.get("task_type") or "generic")
        status = str(details.get("playbook_status") or "updated")
        attempts = str(details.get("attempts") or "?")
        success_rate = str(details.get("success_rate") or "?")
        first_pass_rate = str(details.get("first_pass_rate") or "?")
        d_steps = str(details.get("delta_steps") or "0")
        d_calls = str(details.get("delta_tool_calls") or "0")
        d_runtime = str(details.get("delta_runtime_seconds") or "0")
        d_tokens = str(details.get("delta_tokens") or "0")
        return (
            "Situation: learning feedback stored.\n"
            f"Signal: task_type={task_type}, attempts={attempts}, success_rate={success_rate}, first_pass_rate={first_pass_rate}.\n"
            f"Decision: playbook_status={status}.\n"
            f"Adjustment: delta_steps={d_steps}, delta_tool_calls={d_calls}, delta_runtime_seconds={d_runtime}, delta_tokens={d_tokens}."
        )

    if cat == "reflection":
        goal = str(details.get("goal") or "").strip()
        completed = str(details.get("completed") or "").strip().lower()
        step_count = str(details.get("step_count") or "?").strip()
        worked = str(details.get("worked") or "").strip()
        improve = str(details.get("improve") or "").strip()
        pattern = str(details.get("pattern") or "").strip()
        final_report_excerpt = str(details.get("final_report_excerpt") or "").strip()
        if not worked:
            worked = msg
        if not improve:
            improve = "No concrete improvement action recorded yet."
        decision_bits = []
        if pattern:
            decision_bits.append(f"pattern_rule='{pattern[:140]}'")
        else:
            decision_bits.append("no strong pattern detected")
        if completed in {"true", "false"}:
            decision_bits.append(f"run_completed={completed}")
        decision_text = ", ".join(decision_bits)
        signal_line = f"Signal: steps={step_count}, worked='{worked[:140]}', improve='{improve[:140]}'."
        if goal:
            signal_line += f" Goal context='{goal[:120]}'."
        return (
            "Situation: post-run reflection recorded.\n"
            f"{signal_line}\n"
            f"Decision: {decision_text}.\n"
            f"Adjustment: next run should focus on '{improve[:140]}'"
            + (f"; report context='{final_report_excerpt[:120]}'" if final_report_excerpt else "")
            + "."
        )
    if cat == "correction":
        context = str(details.get("context") or "").strip()
        original_excerpt = str(details.get("original_excerpt") or "").strip()
        correction_excerpt = str(details.get("correction_excerpt") or "").strip()
        source = str(details.get("source") or "user_feedback").strip()
        if not original_excerpt and "->" in msg:
            left, _, right = msg.partition("->")
            original_excerpt = left.strip(" -:>\t")
            correction_excerpt = right.strip(" -:>\t")
        if not original_excerpt and "→" in msg:
            left, _, right = msg.partition("→")
            original_excerpt = left.strip(" -:>\t")
            correction_excerpt = right.strip(" -:>\t")
        if not original_excerpt:
            original_excerpt = "Earlier phrasing not explicitly provided"
        if not correction_excerpt:
            correction_excerpt = msg or "Corrected variant not explicitly provided"
        signal_line = (
            f"Signal: source={source}, issue='{original_excerpt[:140]}', correction='{correction_excerpt[:140]}'."
        )
        if context:
            signal_line += f" Context='{context[:120]}'."
        if source == "self_reflection":
            return (
                "Situation: internal reflection produced a reusable rule.\n"
                f"{signal_line}\n"
                "Decision: add this rule to long-term memory for better next runs.\n"
                "Adjustment: weigh this rule up-front in planning and execution."
            )
        return (
            "Situation: user correction recorded for an earlier answer.\n"
            f"{signal_line}\n"
            "Decision: store this correction as a preference/fact rule for future answers.\n"
            "Adjustment: check similar prompts against this correction first to avoid repetition."
        )
    if cat == "memory_aging":
        return "Memory maintenance completed (cleanup/archive) to keep context relevant."

    return "System event recorded."


def humanize_system_log() -> int:
    """Backfill missing human-readable explanations into existing system-log events."""
    changed = 0
    try:
        if not _SYSTEM_LOG_FILE.exists():
            return 0
        rewritten: list[str] = []
        for raw_line in _SYSTEM_LOG_FILE.read_text(encoding="utf-8").splitlines():
            line = raw_line.strip()
            if not line:
                continue
            try:
                entry = json.loads(line)
            except Exception:
                continue
            details = dict(entry.get("details") or {})
            current_expl = str(details.get("uitleg") or "").strip()
            refresh_needed = (
                not current_expl
                or (
                    str(entry.get("cat") or "").strip().lower() in {"hypothesis", "run_kpi", "learning", "reflection", "correction"}
                    and (len(current_expl) < 140 or not current_expl.startswith("Situation:"))
                )
            )
            if refresh_needed:
                details["uitleg"] = _system_log_explanation(entry)
                entry["details"] = {k: str(v)[:500] for k, v in list(details.items())[:10]}
                changed += 1
            rewritten.append(json.dumps(entry, ensure_ascii=False))
        if changed:
            _SYSTEM_LOG_FILE.write_text("\n".join(rewritten) + ("\n" if rewritten else ""), encoding="utf-8")
    except Exception:
        return 0
    return changed


def log_system_event(category: str, summary: str, details: dict | None = None) -> None:
    """Append a timestamped event to the system log (JSONL)."""
    entry = {
        "ts": datetime.now().isoformat(),
        "cat": category,
        "msg": summary[:300],
    }
    if details:
        entry["details"] = {k: str(v)[:500] for k, v in list(details.items())[:10]}
    try:
        MEMORY_DIR.mkdir(parents=True, exist_ok=True)
        with open(_SYSTEM_LOG_FILE, "a", encoding="utf-8") as f:
            f.write(json.dumps(entry, ensure_ascii=False) + "\n")
    except Exception:
        pass


def read_system_log(max_days: int = _SYSTEM_LOG_MAX_DAYS, limit: int = 500) -> list[dict]:
    """Read recent system log entries (newest first)."""
    cutoff = datetime.now() - timedelta(days=max_days)
    entries: list[dict] = []
    try:
        if not _SYSTEM_LOG_FILE.exists():
            return []
        for line in _SYSTEM_LOG_FILE.read_text(encoding="utf-8").splitlines():
            line = line.strip()
            if not line:
                continue
            try:
                entry = json.loads(line)
                ts = datetime.fromisoformat(entry.get("ts", ""))
                if ts >= cutoff:
                    details = dict(entry.get("details") or {})
                    if not str(details.get("uitleg") or "").strip():
                        details["uitleg"] = _system_log_explanation(entry)
                        entry["details"] = details
                    entries.append(entry)
            except (ValueError, KeyError):
                continue
    except Exception:
        pass
    entries.reverse()
    return entries[:limit]


def prune_system_log(max_days: int = _SYSTEM_LOG_MAX_DAYS) -> int:
    """Remove log entries older than max_days. Returns number of kept entries."""
    cutoff = datetime.now() - timedelta(days=max_days)
    kept: list[str] = []
    try:
        if not _SYSTEM_LOG_FILE.exists():
            return 0
        for line in _SYSTEM_LOG_FILE.read_text(encoding="utf-8").splitlines():
            line = line.strip()
            if not line:
                continue
            try:
                entry = json.loads(line)
                ts = datetime.fromisoformat(entry.get("ts", ""))
                if ts >= cutoff:
                    kept.append(line)
            except (ValueError, KeyError):
                continue
        _SYSTEM_LOG_FILE.write_text("\n".join(kept) + ("\n" if kept else ""), encoding="utf-8")
    except Exception:
        pass
    return len(kept)
