from __future__ import annotations

import re
from collections import Counter
import unicodedata

RE_PAGE = re.compile(r"^\s*(page|стр(аница)?)\s*\d+\s*(/|\s+of\s+)\s*\d+\s*$", re.I)
RE_ONLY_PUNCT = re.compile(r"^[\W_]+$", re.U)
RE_MULTI_SPACE = re.compile(r"[ \t]+")
RE_MULTI_NL = re.compile(r"\n{3,}")

_INVISIBLE_CHARS = ["\u00ad", "\u200b", "\u200c", "\u200d", "\ufeff"]
_BIDI_CTRL_RE = re.compile(r"[\u202a-\u202e\u2060\u2066-\u2069\ufffe\uffff]")

def normalize_text(raw: str) -> str:
    text = raw.replace("\r\n", "\n").replace("\r", "\n")
    for ch in _INVISIBLE_CHARS:
        text = text.replace(ch, "")
    text = _BIDI_CTRL_RE.sub("", text)
    # remove most control/format chars but keep line breaks and tabs
    text = "".join(
        ch for ch in text
        if ch in ("\n", "\t") or not unicodedata.category(ch).startswith("C")
    )
    text = "".join(ch for ch in text if ch == "\n" or (ch.isprintable() and ch != "\x0b"))
    lines = [RE_MULTI_SPACE.sub(" ", ln).strip() for ln in text.split("\n")]
    lines = [ln for ln in lines if ln and not RE_PAGE.match(ln) and not RE_ONLY_PUNCT.match(ln)]
    counts = Counter(lines)
    filtered = []
    for ln in lines:
        if counts[ln] >= 4 and len(ln) <= 90:
            continue
        filtered.append(ln)
    text = "\n".join(filtered)
    text = RE_MULTI_NL.sub("\n\n", text).strip()
    return text

def to_fts_text(clean: str) -> str:
    return re.sub(r"\s+", " ", clean).strip()