init

app.py

@@ -1,7 +1,3 @@
-"""
-This script summarizes from a paragraph(normally 512 length).
-"""
-
 import torch
 import gradio as gr
 
@@ -14,16 +10,9 @@ class SummaryExtractor(object):
         self.device = "cuda" if torch.cuda.is_available() else "cpu"
         self.model = PegasusForConditionalGeneration.from_pretrained('IDEA-CCNL/Randeng-Pegasus-523M-Summary-Chinese').to(self.device)
         self.tokenizer = PegasusTokenizer.from_pretrained("IDEA-CCNL/Randeng-Pegasus-523M-Summary-Chinese")
-
-
         self.text2text_genr = Text2TextGenerationPipeline(self.model, self.tokenizer, device=self.device)
 
     def extract(self, content: str, min=20, max=30) -> str:
-        # inputs = self.tokenizer(content, max_length=512, return_tensors='pt').to(device=self.device)
-        # summary_ids = self.model.generate(inputs['input_ids'])
-        # return self.tokenizer.batch_decode(summary_ids,
-        #                                       skip_special_tokens=True,
-        #                                       clean_up_tokenization_spaces=False)[0]
         return str(self.text2text_genr(content, do_sample=False, min_length=min, max_length=max, num_return_sequences=3)[0]["generated_text"])
 
 
@@ -34,6 +23,8 @@ def randeng_extract(content):
     return t_randeng.extract(content)
 
 def similarity_check(query: str, doc: str):
+    doc_list = doc.split("\n")
+
     return "similarity result"
 
 with gr.Blocks() as app:

article_extractor/data_utils.py

@@ -0,0 +1,321 @@
+# -*- coding: utf-8 -*-
+
+# 用于
+
+import re
+import six
+import unicodedata
+import torch
+import rouge
+import numpy as np
+import random
+# from fengshen.examples.pegasus.pegasus_utils import text_segmentate
+import sys
+
+sys.path.append('../../../../')
+
+rouge = rouge.Rouge()
+
+
+is_py2 = six.PY2
+
+if not is_py2:
+    basestring = str
+
+
+def _is_chinese_char(cp):
+    """Checks whether CP is the codepoint of a CJK character."""
+    # This defines a "chinese character" as anything in the CJK Unicode block:
+    #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
+    #
+    # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
+    # despite its name. The modern Korean Hangul alphabet is a different block,
+    # as is Japanese Hiragana and Katakana. Those alphabets are used to write
+    # space-separated words, so they are not treated specially and handled
+    # like the all of the other languages.
+    if ((cp >= 0x4E00 and cp <= 0x9FFF) or (cp >= 0x3400 and cp <= 0x4DBF)
+            or (cp >= 0x20000 and cp <= 0x2A6DF)
+            or (cp >= 0x2A700 and cp <= 0x2B73F)
+            or (cp >= 0x2B740 and cp <= 0x2B81F)
+            or (cp >= 0x2B820 and cp <= 0x2CEAF)
+            or (cp >= 0xF900 and cp <= 0xFAFF)
+            or (cp >= 0x2F800 and cp <= 0x2FA1F)):
+        return True
+
+    return False
+
+
+def _is_whitespace(char):
+    """Checks whether `char` is a whitespace character."""
+    # \t, \n, and \r are technically control characters but we treat them
+    # as whitespace since they are generally considered as such.
+    if char == " " or char == "\t" or char == "\n" or char == "\r":
+        return True
+    cat = unicodedata.category(char)
+    if cat == "Zs":
+        return True
+    return False
+
+
+def _is_control(char):
+    """Checks whether `char` is a control character."""
+    # These are technically control characters but we count them as whitespace
+    # characters.
+    if char == "\t" or char == "\n" or char == "\r":
+        return False
+    cat = unicodedata.category(char)
+    if cat.startswith("C"):
+        return True
+    return False
+
+
+def _is_punctuation(char):
+    """Checks whether `char` is a punctuation character."""
+    cp = ord(char)
+    # We treat all non-letter/number ASCII as punctuation.
+    # Characters such as "^", "$", and "`" are not in the Unicode
+    # Punctuation class but we treat them as punctuation anyways, for
+    # consistency.
+    if (cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or (
+            cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126):
+        return True
+    cat = unicodedata.category(char)
+    if cat.startswith("P"):
+        return True
+    return False
+
+
+def is_string(s):
+    """判断是否是字符串
+    """
+    return isinstance(s, basestring)
+
+
+def is_stopwords(word, stopwords):
+    if word in stopwords:
+        return True
+    else:
+        return False
+
+
+def text_segmentate(text):
+    en_seg_pattern = '((?:\\!|\\?|\\.|\\n)+(?:\\s)+)'
+    ch_seg_pattern = '((?:？|！|。|\\n)+)'
+    try:
+        text = re.sub(en_seg_pattern, r'\1[SEP]', text)
+        # print("sub text: ", text)
+    except Exception as e:
+        print("input: ", text)
+        raise e
+    text = re.sub(ch_seg_pattern, r'\1[SEP]', text)
+    # print("sub ch text: ", text)
+    text_list = text.split("[SEP]")
+    text_list = list(filter(lambda x: len(x) != 0, text_list))
+    return text_list
+
+
+def load_stopwords(stopwords_path):
+    stopwords_dict = {}
+    with open(stopwords_path, "r") as rf:
+        for line in rf:
+            line = line.strip()
+            if line not in stopwords_dict:
+                stopwords_dict[line] = 0
+            else:
+                pass
+    return stopwords_dict
+
+
+def text_process(text, max_length):
+    """分割文本
+    """
+    texts = text_segmentate(text)
+
+    result, length = [], 0
+    for text in texts:
+        if length + len(text) > max_length * 1.3 and len(result) >= 3:
+            yield result
+            result, length = [], 0
+        result.append(text)
+        length += len(text)
+    if result and len(result) >= 3:
+        yield result
+
+
+def text_process_split_long_content(text, max_length):
+    """分割长文本
+    """
+    texts = text_segmentate(text)
+
+    result, sentence_num = "", 0
+    for text in texts:
+        if len(text) > 500:
+            if len(result) > 300 and sentence_num >= 3:
+                yield result
+                result, sentence_num = "", 0
+            else:
+                result, sentence_num = "", 0
+                continue
+        else:
+            if len(result) + len(text) > max_length * 1.1 and sentence_num >= 3:
+                yield result
+                result, sentence_num = "", 0
+            result += text
+            sentence_num += 1
+
+    if result and sentence_num >= 3:
+        yield result
+
+
+def gather_join(texts, idxs):
+    """取出对应的text，然后拼接起来
+    """
+    return ''.join([texts[i] for i in idxs])
+
+
+def gather_join_f1(texts_token, idsx):
+    join_texts = []
+    for id in idsx:
+        join_texts.extend(texts_token[id])
+    return join_texts
+
+
+def compute_rouge(source, target):
+    """计算rouge-1、rouge-2、rouge-l
+    """
+    source, target = ' '.join(source), ' '.join(target)
+    try:
+        scores = rouge.get_scores(hyps=source, refs=target)
+        return {
+            'rouge-1': scores[0]['rouge-1']['f'],
+            'rouge-2': scores[0]['rouge-2']['f'],
+            'rouge-l': scores[0]['rouge-l']['f'],
+        }
+    except ValueError:
+        return {
+            'rouge-1': 0.0,
+            'rouge-2': 0.0,
+            'rouge-l': 0.0,
+        }
+
+
+def remove_stopwords(texts, stopwords_dict):
+    for i, text in enumerate(texts):
+        texts[i] = list(filter(lambda x: x not in stopwords_dict, text))
+    return texts
+
+
+def pseudo_summary_f1(texts,
+                      stopwords,
+                      tokenizer,
+                      max_length,
+                      rouge_strategy="rouge-l"):
+    """构建伪标签摘要数据集
+    """
+    summary_rate = 0.25
+    max_length = max_length - 1
+    texts_tokens = []
+    sentece_idxs_vec = []
+    for text in texts:
+        if len(texts) == 0:
+            continue
+        try:
+            ids = tokenizer.encode(text.strip())[:-1]
+        except ValueError:
+            print("error, input : ", text)
+            raise ValueError
+        sentece_idxs_vec.append(ids)
+        tokens = [tokenizer._convert_id_to_token(token) for token in ids]
+        texts_tokens.append(tokens)
+
+    texts_tokens_rm = remove_stopwords(texts_tokens, stopwords)
+    source_idxs, target_idxs = list(range(len(texts))), []
+
+    assert len(texts_tokens) == len(texts)
+    # truncate_index = 0
+    while True:
+        sims = []
+        for i in source_idxs:
+            new_source_idxs = [j for j in source_idxs if j != i]
+            new_target_idxs = sorted(target_idxs + [i])
+            new_source = gather_join_f1(texts_tokens_rm, new_source_idxs)
+            new_target = gather_join_f1(texts_tokens_rm, new_target_idxs)
+            sim = compute_rouge(new_source, new_target)[rouge_strategy]
+            sims.append(sim)
+        new_idx = source_idxs[np.argmax(sims)]
+        del sims
+        source_idxs.remove(new_idx)
+        target_idxs = sorted(target_idxs + [new_idx])
+        source = gather_join(texts, source_idxs)
+        target = gather_join(texts, target_idxs)
+        try:
+            if (len(source_idxs) == 1
+                    or 1.0 * len(target) / len(source) > summary_rate):
+                break
+        except ZeroDivisionError as e:
+            print(e.meesage)
+            print(texts)
+            print("source: ", source)
+            print("target: ", target)
+
+    if len(source) < len(target):
+        source, target = target, source
+        source_idxs, target_idxs = target_idxs, source_idxs
+
+    return sentece_idxs_vec, source, target, source_idxs, target_idxs
+
+
+def get_input_mask(sentence_id_vec, indexs):
+    target_idxs = []
+    input_idxs = []
+    kMaskSentenceTokenId = 2
+    kEosTokenId = 1
+    mask_sentence_options_cumulative_prob = [0.9, 0.9, 1, 1]
+    for index in indexs:
+        target_idxs.extend(sentence_id_vec[index])
+        choice = random.uniform(0, 1)
+        if choice < mask_sentence_options_cumulative_prob[0]:
+            # print("mask index: ", index)
+            sentence_id_vec[index] = [kMaskSentenceTokenId]
+        elif choice < mask_sentence_options_cumulative_prob[1]:
+            # print("replace index: ", index)
+            replace_id = random.randint(0, len(sentence_id_vec))
+            sentence_id_vec[index] = sentence_id_vec[replace_id]
+        elif choice < mask_sentence_options_cumulative_prob[2]:
+            pass
+        else:
+            sentence_id_vec[index] = []
+
+    target_idxs.append(kEosTokenId)
+    # print(sentence_id_vec)
+    for index, sentence_id in enumerate(sentence_id_vec):
+        # print(index, sentence_id)
+        if len(sentence_id) == 0:
+            continue
+        input_idxs.extend(sentence_id_vec[index])
+
+    input_idxs.append(kEosTokenId)
+    return input_idxs, target_idxs
+
+
+def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int,
+                       decoder_start_token_id: int):
+    """
+    Shift input ids one token to the right.
+    """
+    shifted_input_ids = input_ids.new_zeros(input_ids.shape)
+    shifted_input_ids[:, 1:] = input_ids[:, :-1].clone()
+    shifted_input_ids[:, 0] = decoder_start_token_id
+
+    if pad_token_id is None:
+        raise ValueError("self.model.config.pad_token_id has to be defined.")
+    # replace possible -100 values in labels by `pad_token_id`
+    shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id)
+
+    return shifted_input_ids
+
+
+def padding_to_maxlength(ids, max_length, pad_id):
+    cur_len = len(ids)
+    len_diff = max_length - cur_len
+    return ids + [pad_id] * len_diff, [1] * cur_len + [0] * len_diff

embed.py

@@ -0,0 +1,36 @@
+"""
+This script turn list of string into embeddings.
+"""
+from transformers import AutoTokenizer, TFAutoModel
+import tensorflow as tf
+
+
+class Embed(object):
+    def __init__(self):
+        self.tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/paraphrase-multilingual-mpnet-base-v2")
+        self.model = TFAutoModel.from_pretrained("sentence-transformers/paraphrase-multilingual-mpnet-base-v2")
+
+    @staticmethod
+    # Mean Pooling - Take attention mask into account for correct averaging
+    def mean_pooling(model_output, attention_mask):
+        token_embeddings = model_output.last_hidden_state
+        input_mask_expanded = tf.cast(tf.tile(tf.expand_dims(attention_mask, -1), [1, 1, token_embeddings.shape[-1]]),
+                                      tf.float32)
+        return tf.math.reduce_sum(token_embeddings * input_mask_expanded, 1) / tf.math.maximum(
+            tf.math.reduce_sum(input_mask_expanded, 1), 1e-9)
+
+    # Encode text
+    def encode(self, texts):
+        # Tokenize sentences
+        encoded_input = self.tokenizer(texts, padding=True, truncation=True, return_tensors='tf')
+
+        # Compute token embeddings
+        model_output = self.model(**encoded_input, return_dict=True)
+
+        # Perform pooling
+        embeddings = Embed.mean_pooling(model_output, encoded_input['attention_mask'])
+
+        # Normalize embeddings
+        embeddings = tf.math.l2_normalize(embeddings, axis=1)
+
+        return embeddings

requirements.txt

@@ -4,4 +4,6 @@ tensorflow
 transformers
 peft
 pandas
-sentence-transformers
+sentence-transformers
+
+rouge