40 lines
1.4 KiB
Python
40 lines
1.4 KiB
Python
from __future__ import annotations
|
|
|
|
import re
|
|
from collections import Counter
|
|
import unicodedata
|
|
|
|
RE_PAGE = re.compile(r"^\s*(page|стр(аница)?)\s*\d+\s*(/|\s+of\s+)\s*\d+\s*$", re.I)
|
|
RE_ONLY_PUNCT = re.compile(r"^[\W_]+$", re.U)
|
|
RE_MULTI_SPACE = re.compile(r"[ \t]+")
|
|
RE_MULTI_NL = re.compile(r"\n{3,}")
|
|
|
|
_INVISIBLE_CHARS = ["\u00ad", "\u200b", "\u200c", "\u200d", "\ufeff"]
|
|
_BIDI_CTRL_RE = re.compile(r"[\u202a-\u202e\u2060\u2066-\u2069\ufffe\uffff]")
|
|
|
|
def normalize_text(raw: str) -> str:
|
|
text = raw.replace("\r\n", "\n").replace("\r", "\n")
|
|
for ch in _INVISIBLE_CHARS:
|
|
text = text.replace(ch, "")
|
|
text = _BIDI_CTRL_RE.sub("", text)
|
|
# remove most control/format chars but keep line breaks and tabs
|
|
text = "".join(
|
|
ch for ch in text
|
|
if ch in ("\n", "\t") or not unicodedata.category(ch).startswith("C")
|
|
)
|
|
text = "".join(ch for ch in text if ch == "\n" or (ch.isprintable() and ch != "\x0b"))
|
|
lines = [RE_MULTI_SPACE.sub(" ", ln).strip() for ln in text.split("\n")]
|
|
lines = [ln for ln in lines if ln and not RE_PAGE.match(ln) and not RE_ONLY_PUNCT.match(ln)]
|
|
counts = Counter(lines)
|
|
filtered = []
|
|
for ln in lines:
|
|
if counts[ln] >= 4 and len(ln) <= 90:
|
|
continue
|
|
filtered.append(ln)
|
|
text = "\n".join(filtered)
|
|
text = RE_MULTI_NL.sub("\n\n", text).strip()
|
|
return text
|
|
|
|
def to_fts_text(clean: str) -> str:
|
|
return re.sub(r"\s+", " ", clean).strip()
|