Spaces:
Build error
Build error
File size: 1,476 Bytes
93e5f33 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 |
import numpy, nltk
nltk.download('punkt')
from harvesttext import HarvestText
from lex_rank_util import degree_centrality_scores
from sentence_transformers import SentenceTransformer, util
class LexRank(object):
def __init__(self):
self.model = SentenceTransformer('paraphrase-multilingual-mpnet-base-v2')
self.ht = HarvestText()
def find_central(self, content: str, num=100):
if self.contains_chinese(content):
sentences = self.ht.cut_sentences(content)
else:
sentences = nltk.sent_tokenize(content)
embeddings = self.model.encode(sentences, convert_to_tensor=True).cpu()
# Compute the pair-wise cosine similarities
cos_scores = util.cos_sim(embeddings, embeddings).numpy()
# Compute the centrality for each sentence
centrality_scores = degree_centrality_scores(cos_scores, threshold=None)
# We argsort so that the first element is the sentence with the highest score
most_central_sentence_indices = numpy.argsort(-centrality_scores)
# num = 100
res = []
for index in most_central_sentence_indices:
if num < 0:
break
res.append(sentences[index])
num -= len(sentences[index])
return res
def contains_chinese(self, content: str):
for _char in content:
if '\u4e00' <= _char <= '\u9fa5':
return True
return False
|