"""Build the input `state` for Vansa's email-qualification preset from one outreach thread.

Reference implementation, Python 3.9+ standard library only. It reproduces the pipeline Vansa's email skill (vansa-2 and vansa-3) was
trained with: the same text cleaning, the same `counts`, the 8-message cap and the trimming to about 1,250 tokens.
The published accuracy was measured on states built that way. On the 3,123 held-out threads, the trimming with the
built-in token estimate gives the exact trained state for 91% of them (the rest are trimmed differently; the largest
comes to 1,575 tokens, over the 1,250 budget). build_state(..., fit=False) followed by fit_state(state, count_tokens=...)
with the tokenizer the training data was built with gives the exact state for all 3,123.

    import json, urllib.request
    from email_state import build_state

    state = build_state(
        site="example-garden-blog.com",                 # the website's email domain
        subject="Guest post on example-garden-blog.com",
        emails=[                                         # every email between your team and this one site
            {"from": "us",   "date": "2026-09-10T09:12:00+00:00", "text": "Hi, we would like to publish ..."},
            {"from": "them", "date": "2026-09-11T14:03:00+02:00", "text": "Hi, yes. One guest post is 120 EUR ..."},
        ],
    )
    # any User-Agent: Cloudflare answers 403 (error 1010) to Python-urllib's default one
    req = urllib.request.Request("https://docs.vansa.org/presets/email-qualification.json", headers={"User-Agent": "my-app/1.0"})
    preset = json.load(urllib.request.urlopen(req, timeout=30))
    body = {"model": preset["model"], "state": state, "questions": preset["questions"]}
    # POST body as JSON to https://api.vansa.org/v1/systemone with "Authorization: Bearer vsk_...",
    # "Content-Type: application/json" and a User-Agent header

Each email is a dict:
    from     "us" (your outreach side, the buyer of the post) or "them" (the website)
    date     a datetime, an ISO-8601 string or a raw "Date:" header value; values without a timezone are taken as UTC
    text     the plain-text body (convert HTML first); quoted history and footers are removed here
    kind     optional: "auto_reply", "bounce" or None (= a normal message). When the key is absent, build_state
             detects it with detect_kind() from the optional keys below; pass it yourself when your mail system knows.
             Training also counted a bounce under auto_replies when it carried an auto-reply signal, so with
             kind="bounce" pass subject/headers/text too.
    subject  optional: this email's subject (helps to detect "Automatic reply: ..." and "Undeliverable: ...")
    sender   optional: the From address (helps to detect mailer-daemon / postmaster bounces)
    headers  optional: a dict or an email.message.Message; Auto-Submitted, X-Autoreply, X-Autorespond and
             Content-Type are the reliable auto-reply / bounce signals
"""
from __future__ import annotations

import copy
import email.utils
import json
import re
from datetime import datetime, timezone

ROLES = "us = our outreach team buying a sponsored guest post; them = the website owner or editor"
MAX_MESSAGES = 8          # longer threads keep the first 2 and the last 6 before trimming
STATE_BUDGET = 1250       # tokens for the whole state
CHARS_PER_TOKEN = 2.7     # conservative estimate: never under-counts on 95% of threads, and the rest still fits
WORD_CAPS = (120, 90, 70, 55, 40, 30)
KEEP_LAST = 4

# ---------------------------------------------------------------- cleaning (one email body)
QUOTE_CUTS = [
    r"^On .{5,120}wrote:\s*$",                       # Gmail / Apple
    r"^-{2,}\s*Original Message\s*-{2,}",            # Outlook
    r"^-{2,}\s*Forwarded message\s*-{2,}",
    r"^From:\s.+\n(Sent|Date):\s",                   # Outlook header block
    r"^Le .{5,120}a écrit\s*:",                      # fr
    r"^Am .{5,120}schrieb .{0,80}:",                 # de
    r"^El .{5,120}escribió:",                        # es
    r"^Il .{5,120}ha scritto:",                      # it
    r"^Em .{5,120}escreveu:",                        # pt
    r"^\d{1,2}\.\d{1,2}\.\d{2,4}.{0,60}(wrote|schrieb|napsal|pisze)",
    r"^________________________________",            # Outlook web
    r"^Sent from my (iPhone|iPad|Samsung|Android)",
    r"^-{2,3}\s*$",                                  # signature separator
]
QUOTE_RE = re.compile("|".join(f"(?:{p})" for p in QUOTE_CUTS), re.IGNORECASE | re.MULTILINE)
EXTRA_CUTS = re.compile(r"(?m)^El .{5,200}escribi[oó]:\s*$|^(De|From):\s.+\n(Enviado|Sent|Date|Fecha):\s|^\s*\d{1,2}/\d{1,2}/\d{2,4}.{0,80}(wrote|escribi[oó]):", re.IGNORECASE)
WRAP_FIXES = [
    (re.compile(r"(?m)^(On .{5,200}?)\n\s*(wrote:)\s*$"), r"\1 \2"),
    (re.compile(r"(?m)^(El .{5,200}?)\n\s*(escribi[oó]:)\s*$"), r"\1 \2"),
    (re.compile(r"<\n\s*([^>\n]+@[^>\n]+>)"), r"<\1"),
]
FOOTER_RE = re.compile(r"(unsubscribe|this email and any attachments|confidentiality notice|view (this|it) in your browser)", re.IGNORECASE)
SUBJ_PREFIX_RE = re.compile(r"^\s*((re|fwd?|aw|wg|tr|sv|vs|odp|rv)\s*:\s*)+", re.IGNORECASE)
WS_RE = re.compile(r"[ \t]+")
AUTO_SUBJ_RE = re.compile(r"out of (the )?office|automatic reply|autoreply|auto-reply|automatische antwort|abwesenheit|absence du bureau|respuesta autom[aá]tica|fuera de la oficina|ticket #?\d+ (received|created)", re.IGNORECASE)
AUTO_BODY_RE = re.compile(r"we have received your (message|email|request)|thank you for (contacting|your (message|email))|this (mailbox|inbox) is not monitored|ticket (number|#)|hemos recibido (su|tu) (mensaje|correo)|fuera de la oficina|out of office|no responder a este|do not reply to this", re.IGNORECASE)
BOUNCE_SUBJ_RE = re.compile(r"delivery status notification|undeliverable|mail delivery failed", re.IGNORECASE)


def clean_text(text: str, max_words: int = 150) -> str:
    """New text of one email: quoted history, '>' lines and footers removed, whitespace collapsed, 150 words."""
    text = (text or "").replace("\r", "")
    for rx, rep in WRAP_FIXES:
        text = rx.sub(rep, text)
    m, m2 = QUOTE_RE.search(text), EXTRA_CUTS.search(text)
    if m2 and (not m or m2.start() < m.start()):
        m = m2
    if m:
        text = text[: m.start()]
    lines = []
    for line in text.split("\n"):
        if line.lstrip().startswith(">"):
            continue
        if FOOTER_RE.search(line):
            break
        lines.append(WS_RE.sub(" ", line).strip())
    text = re.sub(r"\n{2,}", "\n", "\n".join(x for x in lines if x)).strip()
    words = text.split(" ")
    return " ".join(words[:max_words]) + " ..." if len(words) > max_words else text


def _signals(subject: str = "", text: str = "", headers=None, sender: str = "") -> tuple[bool, bool]:
    """(bounce, auto_reply), decided independently as in the training pipeline: a bounce that also carries an
    auto-reply signal (e.g. an Auto-Submitted header) is both."""
    h = {str(k).lower(): str(v) for k, v in (headers.items() if headers is not None else []) if v is not None}
    s, subject = str(sender or "").lower(), str(subject or "")
    bounce = bool(h.get("content-type", "").lower().startswith("multipart/report") or "mailer-daemon" in s
                  or "postmaster@" in s or BOUNCE_SUBJ_RE.search(subject))
    auto = bool(h.get("auto-submitted", "").lower() not in ("", "no") or h.get("x-autoreply") or h.get("x-autorespond")
                or AUTO_SUBJ_RE.search(subject) or AUTO_BODY_RE.search(clean_text(str(text or ""))[:200]))
    return bounce, auto


def detect_kind(subject: str = "", text: str = "", headers=None, sender: str = "") -> str | None:
    """"bounce", "auto_reply" or None, as the training pipeline decided it. Headers (a dict or an
    email.message.Message: Auto-Submitted, X-Autoreply, X-Autorespond, Content-Type) are the reliable signal."""
    bounce, auto = _signals(subject, text, headers, sender)
    return "bounce" if bounce else "auto_reply" if auto else None


_ISO_TZ_RE = re.compile(r"([+-]\d{2})(\d{2})$")          # +0200 -> +02:00 (Python < 3.11 needs the colon)
_ISO_FRAC_RE = re.compile(r"(\.\d+)(?=[+-]\d{2}:\d{2}$|$)")  # any number of fraction digits -> 6


def _when(d) -> datetime:
    """datetime, ISO-8601 string (any Python 3.9+) or an RFC 2822 "Date:" header value; naive = UTC."""
    if isinstance(d, str):
        s = d.strip()
        try:
            iso = _ISO_TZ_RE.sub(r"\1:\2", re.sub(r"[zZ]$", "+00:00", s.replace(" ", "T", 1) if re.match(r"\d{4}-\d{2}-\d{2} \d", s) else s))
            iso = _ISO_FRAC_RE.sub(lambda m: m.group(1)[:7].ljust(7, "0"), iso)
            d = datetime.fromisoformat(iso)
        except ValueError:
            try:
                d = email.utils.parsedate_to_datetime(s)      # e.g. "Tue, 02 Jun 2026 09:00:00 +0200"
            except (TypeError, ValueError):                   # 3.9 raises TypeError on garbage
                raise ValueError(f"unsupported date: {s!r}") from None
    if not isinstance(d, datetime):
        raise ValueError(f"unsupported date: {d!r}")
    return d if d.tzinfo else d.replace(tzinfo=timezone.utc)


# ---------------------------------------------------------------- the whole thread
def build_state(site: str, subject: str, emails: list[dict], now: datetime | None = None, fit: bool = True) -> dict:
    """The state for one thread. `now` defaults to the current time (it sets counts.days_since_last)."""
    ms = sorted(({**e, "_d": _when(e["date"])} for e in emails), key=lambda e: e["_d"])
    if not ms:
        raise ValueError("a thread needs at least one email")
    for e in ms:
        if e["from"] not in ("us", "them"):
            raise ValueError('"from" must be "us" or "them"')
        if "kind" not in e:     # auto-replies and bounces must not count as real replies
            e["_bounce"], e["_auto"] = _signals(e.get("subject"), e.get("text"), e.get("headers"), e.get("sender"))
            e["kind"] = "bounce" if e["_bounce"] else "auto_reply" if e["_auto"] else None
        else:                   # a caller-given kind; a bounce still counts as an auto-reply when its fields carry that signal
            e["_bounce"] = e["kind"] == "bounce"
            e["_auto"] = e["kind"] == "auto_reply" or (e["_bounce"] and _signals(e.get("subject"), e.get("text"), e.get("headers"), e.get("sender"))[1])
        e["_t"] = clean_text(str(e.get("text") or ""))
    shown = ms if len(ms) <= MAX_MESSAGES else ms[:2] + ms[-(MAX_MESSAGES - 2):]
    omitted = len(ms) - len(shown)
    messages = []
    for i, e in enumerate(shown, 1):
        if omitted and i == 3:
            messages.append({"n": "...", "omitted": omitted})
        m = {"n": i, "from": e["from"], "date": e["_d"].strftime("%Y-%m-%d"), "text": e["_t"] or "(empty)"}
        if e.get("kind") in ("auto_reply", "bounce"):
            m["kind"] = e["kind"]
        messages.append(m)
    now = _when(now or datetime.now(timezone.utc))
    auto = lambda e: e["_auto"]  # noqa: E731  (a bounce with auto-reply signals counts under both, as in training)
    bounce = lambda e: e["_bounce"]  # noqa: E731
    state = {
        "site": (site or "").strip().lower(),
        "subject": SUBJ_PREFIX_RE.sub("", subject or "")[:120],
        "roles": ROLES,
        "counts": {
            "ours": sum(e["from"] == "us" for e in ms),
            "theirs": sum(e["from"] == "them" and not auto(e) and not bounce(e) for e in ms),
            "auto_replies": sum(auto(e) for e in ms),
            "bounces": sum(bounce(e) for e in ms),
            "days_since_last": max(0, (now - ms[-1]["_d"]).days),
            "last_from": ms[-1]["from"],
        },
        "messages": messages,
    }
    return fit_state(state) if fit else state


# ---------------------------------------------------------------- trimming to the token budget
AMOUNT_RE = re.compile(r"\d[\d.,]*\s?(eur|usd|gbp|mxn|brl|pln|czk|chf|sek|nok|dkk|inr|€|\$|£|euros?|d[oó]lares?|dollars?|pounds?|bucks)\b|[€$£]\s?\d", re.IGNORECASE)
ANSWER_RE = re.compile(r"^\s*(\d+\s*[.)-]\s*)?(yes|no|nope|s[ií]|non|nein|n[ãa]o|oui|ja)\b|^\s*\d+\s*[.)]\s", re.IGNORECASE)
SENT_RE = re.compile(r"(?<=[^\d][.!?])\s+|\n+")


def estimate_tokens(state: dict) -> int:
    return int(len(json.dumps(state, ensure_ascii=False)) / CHARS_PER_TOKEN) + 1


def _cap_words(text: str, cap: int) -> str:
    """First `cap` words, plus up to 3 later sentences that carry an amount or a yes/no answer."""
    words = text.split(" ")
    if len(words) <= cap:
        return text
    head, tail = " ".join(words[:cap]), " ".join(words[cap:])
    frags = []
    for s in SENT_RE.split(tail):
        s = (s or "").strip()
        if not s:
            continue
        if frags and frags[-1].endswith("?"):
            frags[-1] += " " + s
        else:
            frags.append(s)
    kept = []
    for s in frags:
        if AMOUNT_RE.search(s) or ANSWER_RE.match(s):
            kept.append(" ".join(s.split(" ")[:35]))
            if len(kept) >= 3:
                break
    return head + " ..." + "".join(" " + k + " ..." for k in kept)


def _norm(text: str) -> str:
    return re.sub(r"\s+", " ", text).strip().lower()


def _strip_quotes(msgs: list) -> None:
    """Their message that repeats one of ours verbatim is a missed quote: the repeated part is removed."""
    prev = []
    for m in msgs:
        if m.get("from") == "us":
            prev.append(m.get("text", ""))
        elif m.get("from") == "them" and not m.get("kind"):
            t = m.get("text", "")
            for u in prev:
                if len(u) < 40 or _norm(u) not in _norm(t):
                    continue
                t = re.compile(r"\s+".join(re.escape(w) for w in u.split()), re.IGNORECASE).sub(" ", t)
                t = re.sub(r"\s+", " ", t).strip()
            if t != m.get("text", ""):
                m["text"] = t if len(t.split()) >= 3 else "(no new text: their message only repeats ours)"


def fit_state(state: dict, budget: int = STATE_BUDGET, count_tokens=estimate_tokens) -> dict:
    """Shorten every message (120 -> 30 words), keep the first message, their first real reply and the last 4
    ({"n": "...", "omitted": k} replaces the middle), then keep fewer trailing messages until it fits.
    site, subject, roles and counts are never changed."""
    st = copy.deepcopy(state)
    msgs_all = [m for m in st.get("messages", []) if "omitted" not in m]
    _strip_quotes(msgs_all)
    already_omitted = sum(m.get("omitted", 0) for m in st.get("messages", []) if "omitted" in m)
    first_reply = next((i for i, m in enumerate(msgs_all) if m.get("from") == "them" and not m.get("kind")), None)

    def build(cap: int, last: int) -> dict:
        s = dict(st)
        msgs = [dict(m) for m in msgs_all]
        for m in msgs:
            m["text"] = _cap_words(m.get("text", ""), cap)
        head_idx = [0] + ([first_reply] if first_reply not in (None, 0) and first_reply < len(msgs) - last else [])
        if len(msgs) > len(head_idx) + last:
            omitted = len(msgs) - len(head_idx) - last + already_omitted
            msgs = [msgs[i] for i in head_idx] + [{"n": "...", "omitted": omitted}] + msgs[-last:]
        elif already_omitted:
            msgs = [msgs[0], {"n": "...", "omitted": already_omitted}] + msgs[1:]
        for i, m in enumerate(msgs):
            if "omitted" not in m:
                m["n"] = i + 1
        s["messages"] = msgs
        return s

    for last in range(KEEP_LAST, 0, -1):
        for cap in WORD_CAPS:
            cand = build(cap, last)
            if count_tokens(cand) <= budget:
                return cand
    cand = build(WORD_CAPS[-1], 1)
    for m in cand["messages"]:
        if "text" in m:
            m["text"] = m["text"][:200]
    return cand


if __name__ == "__main__":
    demo = build_state("example-garden-blog.com", "Re: Guest post on example-garden-blog.com", [
        {"from": "us", "date": "2026-09-10T09:00:00", "text": "Hi, do you accept guest posts? What is your price for one article?"},
        {"from": "them", "date": "2026-09-11T10:00:00", "text": "Yes, 120 EUR, do-follow, permanent.\n\nOn Wed, Sep 10, 2026 at 9:00 AM Us <us@agency.example> wrote:\n> Hi, do you accept guest posts?"},
    ], now=datetime(2026, 9, 24, tzinfo=timezone.utc))
    print(json.dumps(demo, ensure_ascii=False, indent=2))