| | """Set of default text cleaners"""
|
| |
|
| |
|
| | import re
|
| |
|
| |
|
| | _whitespace_re = re.compile(r"\s+")
|
| |
|
| | rep_map = {
|
| | "：": ",",
|
| | "；": ",",
|
| | "，": ",",
|
| | "。": ".",
|
| | "！": "!",
|
| | "？": "?",
|
| | "\n": ".",
|
| | "·": ",",
|
| | "、": ",",
|
| | "...": ".",
|
| | "…": ".",
|
| | "$": ".",
|
| | "“": "'",
|
| | "”": "'",
|
| | "‘": "'",
|
| | "’": "'",
|
| | "（": "'",
|
| | "）": "'",
|
| | "(": "'",
|
| | ")": "'",
|
| | "《": "'",
|
| | "》": "'",
|
| | "【": "'",
|
| | "】": "'",
|
| | "[": "'",
|
| | "]": "'",
|
| | "—": "",
|
| | "～": "-",
|
| | "~": "-",
|
| | "「": "'",
|
| | "」": "'",
|
| | }
|
| |
|
| | def replace_punctuation(text):
|
| | pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
|
| | replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
|
| | return replaced_text
|
| |
|
| | def lowercase(text):
|
| | return text.lower()
|
| |
|
| |
|
| | def collapse_whitespace(text):
|
| | return re.sub(_whitespace_re, " ", text).strip()
|
| |
|
| | def remove_punctuation_at_begin(text):
|
| | return re.sub(r'^[,.!?]+', '', text)
|
| |
|
| | def remove_aux_symbols(text):
|
| | text = re.sub(r"[\<\>\(\)\[\]\"\«\»\']+", "", text)
|
| | return text
|
| |
|
| |
|
| | def replace_symbols(text, lang="en"):
|
| | """Replace symbols based on the lenguage tag.
|
| |
|
| | Args:
|
| | text:
|
| | Input text.
|
| | lang:
|
| | Lenguage identifier. ex: "en", "fr", "pt", "ca".
|
| |
|
| | Returns:
|
| | The modified text
|
| | example:
|
| | input args:
|
| | text: "si l'avi cau, diguem-ho"
|
| | lang: "ca"
|
| | Output:
|
| | text: "si lavi cau, diguemho"
|
| | """
|
| | text = text.replace(";", ",")
|
| | text = text.replace("-", " ") if lang != "ca" else text.replace("-", "")
|
| | text = text.replace(":", ",")
|
| | if lang == "en":
|
| | text = text.replace("&", " and ")
|
| | elif lang == "fr":
|
| | text = text.replace("&", " et ")
|
| | elif lang == "pt":
|
| | text = text.replace("&", " e ")
|
| | elif lang == "ca":
|
| | text = text.replace("&", " i ")
|
| | text = text.replace("'", "")
|
| | elif lang== "es":
|
| | text=text.replace("&","y")
|
| | text = text.replace("'", "")
|
| | return text
|
| |
|
| | def unicleaners(text, cased=False, lang='en'):
|
| | """Basic pipeline for Portuguese text. There is no need to expand abbreviation and
|
| | numbers, phonemizer already does that"""
|
| | if not cased:
|
| | text = lowercase(text)
|
| | text = replace_punctuation(text)
|
| | text = replace_symbols(text, lang=lang)
|
| | text = remove_aux_symbols(text)
|
| | text = remove_punctuation_at_begin(text)
|
| | text = collapse_whitespace(text)
|
| | text = re.sub(r'([^\.,!\?\-…])$', r'\1.', text)
|
| | return text
|
| |
|
| |
|
