|
import re, os, sys
|
|
from utils import *
|
|
|
|
def get_paragraphs(text, cutoff=10):
|
|
return [ x.strip() for x in re.split(r'\n+', text, flags=re.MULTILINE) if len(x.strip()) > cutoff ]
|
|
|
|
def get_para_sentences(text, cutoff=60):
|
|
para_sents = []
|
|
for para in get_paragraphs(text):
|
|
sents = []; sent = ""
|
|
chunks = re.split(r'\.+', para); n = len(chunks)
|
|
for i in range(0, n):
|
|
sent += chunks[i]
|
|
if i < n - 1: sent += "."
|
|
if len(sent) > cutoff:
|
|
sents.append(sent)
|
|
sent = ""
|
|
if len(sent) > 0: sents.append(sent)
|
|
|
|
para_sents.append(sents)
|
|
return para_sents
|
|
|
|
def get_idx_from_marked_chunk(marked_chunk):
|
|
return int(re.match(r'<C\s*(\d+)>', marked_chunk)[1])
|
|
import random; idx = random.randint(0, 99999)
|
|
assert get_idx_from_marked_chunk(f"<C {idx}> ha ha") == idx
|
|
|
|
|
|
def add_chunk_markers(text, lookup_idx = None, para = True):
|
|
if para: para_chunks = get_paragraphs(text)
|
|
else: para_chunks = get_para_sentences(text)
|
|
|
|
marked_text = ""; chunk_idx = 0
|
|
for chunks in para_chunks:
|
|
if isinstance(chunks, str): chunks = [chunks]
|
|
for idx, chunk in enumerate(chunks):
|
|
marked_chunk = f"<C {chunk_idx}>{chunk.strip()}"
|
|
|
|
chunks[idx] = marked_chunk
|
|
if lookup_idx == chunk_idx: print(marked_chunk); sys.exit()
|
|
|
|
marked_text += f"{marked_chunk}\n"
|
|
chunk_idx += 1
|
|
marked_text += "\n"
|
|
return marked_text.strip(), para_chunks
|
|
|
|
|
|
alphabet = '[0-9a-zaăâáắấàằầảẳẩãẵẫạặậđeêéếèềẻểẽễẹệiíìỉĩịoôơóốớòồờỏổởõỗỡọộợuưúứùừủửũữụựyýỳỷỹỵaăâáắấàằầảẳẩãẵẫạặậđeêéếèềẻểẽễẹệiíìỉĩịoôơóốớòồờỏổởõỗỡọộợuưúứùừủửũữụựyýỳỷỹỵaăâáắấàằầảẳẩãẵẫạặậđeêéếèềẻểẽễẹệiíìỉĩịoôơóốớòồờỏổởõỗỡọộợuưúứùừủửũữụựyýỳỷỹỵaăâáắấàằầảẳẩãẵẫạặậđeêéếèềẻểẽễẹệiíìỉĩịoôơóốớòồờỏổởõỗỡọộợuưúứùừủửũữụựyýỳỷỹỵaăâáắấàằầảẳẩãẵẫạặậđeêéếèềẻểẽễẹệiíìỉĩịoôơóốớòồờỏổởõỗỡọộợuưúứùừủửũữụựyýỳỷỹỵaăâáắấàằầảẳẩãẵẫạặậđeêéếèềẻểẽễẹệiíìỉĩịoôơóốớòồờỏổởõỗỡọộợuưúứùừủửũữụựyýỳỷỹỵ]'
|
|
word = re.compile(f'{alphabet}+', re.IGNORECASE)
|
|
|
|
def hilite(query, source, hilite_color=YELLOW, source_color=GREY, query_color=None):
|
|
for keyword in set(re.findall(word, query)):
|
|
keyword = re.escape(keyword)
|
|
re_keyword = re.compile(rf"(\b{keyword}\b)", flags=re.IGNORECASE | re.MULTILINE)
|
|
if re_keyword.search(source):
|
|
source = re.sub(re_keyword, rf'{hilite_color}\1{source_color}', source)
|
|
if query_color is not None:
|
|
query = re.sub(re_keyword, rf'{hilite_color}\1{query_color}', query)
|
|
return source, query
|
|
|
|
|
|
def pretty_num(x):
|
|
return round(x*100)/100
|
|
|
|
def count_words(x):
|
|
assert isinstance(x, str), f"đầu không phải string {x}"
|
|
return len(x.split())
|
|
|
|
def extract_(text, tag):
|
|
raw = text.split(f"</{tag}>")[0].split(f"<{tag}>")[-1]
|
|
if tag == "summary": return raw.strip()
|
|
splits = re.split(r'[\n,]+', raw)
|
|
splits = [ re.sub(r'^\s*-\s*', '', s).strip() for s in splits ]
|
|
splits = [ s for s in splits if len(s) > 0 ]
|
|
return splits
|
|
|
|
def extract_xmls(text, tags):
|
|
if text is None: return None
|
|
return { tag: extract_(text, tag) for tag in tags }
|
|
|
