Spaces:
Runtime error
Runtime error
marcelomoreno26
commited on
Commit
•
9d54e27
1
Parent(s):
dd204e1
Upload 3 files
Browse files- model_functions.py +91 -0
- preprocessor.py +95 -0
- requirements.txt +5 -0
model_functions.py
ADDED
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import torch
|
2 |
+
from transformers import (AutoModelForSequenceClassification, AutoModelForSeq2SeqLM,
|
3 |
+
AutoConfig, AutoModelForTokenClassification,
|
4 |
+
AutoTokenizer, pipeline)
|
5 |
+
from peft import PeftModel, PeftConfig
|
6 |
+
|
7 |
+
|
8 |
+
|
9 |
+
|
10 |
+
def load_sentiment_analyzer():
|
11 |
+
tokenizer = AutoTokenizer.from_pretrained("aliciiavs/sentiment-analysis-whatsapp2")
|
12 |
+
model = AutoModelForSequenceClassification.from_pretrained("aliciiavs/sentiment-analysis-whatsapp2")
|
13 |
+
|
14 |
+
return tokenizer, model
|
15 |
+
|
16 |
+
def load_summarizer():
|
17 |
+
config = PeftConfig.from_pretrained("marcelomoreno26/bart-large-samsum-adapter")
|
18 |
+
model = AutoModelForSeq2SeqLM.from_pretrained("facebook/bart-large")
|
19 |
+
tokenizer = AutoTokenizer.from_pretrained("facebook/bart-large")
|
20 |
+
tokenizer.pad_token = tokenizer.eos_token
|
21 |
+
model = PeftModel.from_pretrained(model, "marcelomoreno26/bart-large-samsum-adapter", config=config)
|
22 |
+
model = model.merge_and_unload()
|
23 |
+
|
24 |
+
return tokenizer, model
|
25 |
+
|
26 |
+
def load_NER():
|
27 |
+
config = AutoConfig.from_pretrained("hannahisrael03/distilbert-base-uncased-finetuned-wikiann")
|
28 |
+
model = AutoModelForTokenClassification.from_pretrained("hannahisrael03/distilbert-base-uncased-finetuned-wikiann",config=config)
|
29 |
+
tokenizer = AutoTokenizer.from_pretrained("hannahisrael03/distilbert-base-uncased-finetuned-wikiann")
|
30 |
+
pipe = pipeline("ner", model=model, tokenizer=tokenizer, aggregation_strategy="average")
|
31 |
+
|
32 |
+
return pipe
|
33 |
+
|
34 |
+
def get_sentiment_analysis(text, tokenizer, model):
|
35 |
+
inputs = tokenizer(text, padding=True, return_tensors="pt")
|
36 |
+
with torch.no_grad():
|
37 |
+
outputs = model(**inputs)
|
38 |
+
# Get predicted probabilities and predicted label
|
39 |
+
probabilities = torch.softmax(outputs.logits, dim=1)
|
40 |
+
predicted_label = torch.argmax(probabilities, dim=1)
|
41 |
+
# Convert the predicted label tensor to a Python integer
|
42 |
+
predicted_label = predicted_label.item()
|
43 |
+
# Map predicted label index to sentiment label
|
44 |
+
label_dic = {0: 'sadness', 1: 'joy', 2: 'love', 3: 'anger', 4: 'fear', 5: 'surprise'}
|
45 |
+
# Print the predicted sentiment label
|
46 |
+
return label_dic[predicted_label]
|
47 |
+
|
48 |
+
|
49 |
+
def generate_summary(text, tokenizer, model):
|
50 |
+
prefix = "summarize: "
|
51 |
+
encoded_input = tokenizer.encode_plus(prefix + text, return_tensors='pt', add_special_tokens=True)
|
52 |
+
input_ids = encoded_input['input_ids']
|
53 |
+
|
54 |
+
# Check if input_ids exceed the model's max length
|
55 |
+
max_length = 512
|
56 |
+
if input_ids.shape[1] > max_length:
|
57 |
+
# Split the input_ids into manageable segments
|
58 |
+
total_summary = []
|
59 |
+
for i in range(0, input_ids.shape[1], max_length - 50): # We use max_length - 50 to allow for some room for the model to generate context
|
60 |
+
segment_ids = input_ids[:, i:i + max_length]
|
61 |
+
output_ids = model.generate(segment_ids, max_length=150, num_beams=5, early_stopping=True)
|
62 |
+
segment_summary = tokenizer.decode(output_ids[0], skip_special_tokens=True)
|
63 |
+
total_summary.append(segment_summary)
|
64 |
+
|
65 |
+
# Concatenate all segment summaries
|
66 |
+
summary = ' '.join(total_summary)
|
67 |
+
else:
|
68 |
+
# Process as usual
|
69 |
+
output_ids = model.generate(input_ids, max_length=150, num_beams=5, early_stopping=True)
|
70 |
+
summary = tokenizer.decode(output_ids[0], skip_special_tokens=True)
|
71 |
+
|
72 |
+
return summary
|
73 |
+
|
74 |
+
|
75 |
+
def get_NER(text, pipe):
|
76 |
+
# Use pipeline to predict NER
|
77 |
+
results = pipe(text)
|
78 |
+
# Filter duplicates while retaining the highest score for each entity type and word combination
|
79 |
+
unique_entities = {}
|
80 |
+
for ent in results:
|
81 |
+
key = (ent['entity_group'], ent['word'])
|
82 |
+
if key not in unique_entities or unique_entities[key]['score'] < ent['score']:
|
83 |
+
unique_entities[key] = ent
|
84 |
+
|
85 |
+
# Prepare the output, sorted by the start position to maintain the order they appear in the text
|
86 |
+
filtered_results = sorted(unique_entities.values(), key=lambda x: x['start'])
|
87 |
+
# Format the results for a table display
|
88 |
+
formatted_results = [[ent['word'], ent['entity_group']] for ent in filtered_results]
|
89 |
+
|
90 |
+
return formatted_results
|
91 |
+
|
preprocessor.py
ADDED
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import pandas as pd
|
2 |
+
import zipfile
|
3 |
+
import re
|
4 |
+
from io import BytesIO
|
5 |
+
|
6 |
+
|
7 |
+
def detect_file_type(file_path):
|
8 |
+
type = file_path[-3:]
|
9 |
+
print(type)
|
10 |
+
if type in ["txt","zip"]:
|
11 |
+
return type
|
12 |
+
else:
|
13 |
+
return "unknown"
|
14 |
+
|
15 |
+
def preprocess_whatsapp_messages(file_path, file_type):
|
16 |
+
"""
|
17 |
+
Preprocesses the Whatsapp messages zip file into a Pandas Dataframe, all messages in one day go
|
18 |
+
to a row and a timestamp is added.
|
19 |
+
|
20 |
+
Args:
|
21 |
+
file_path (str): Location of the file (zip or txt) of the conversation.
|
22 |
+
|
23 |
+
Returns:
|
24 |
+
str: Dataframe
|
25 |
+
"""
|
26 |
+
|
27 |
+
# Load the zip file and extract text data
|
28 |
+
print(file_type)
|
29 |
+
if file_type == "zip":
|
30 |
+
with zipfile.ZipFile(file_path, 'r') as z:
|
31 |
+
file_name = z.namelist()[0]
|
32 |
+
with z.open(file_name) as file:
|
33 |
+
text_data = file.read().decode('utf-8')
|
34 |
+
else:
|
35 |
+
text_data = BytesIO(file_path.getvalue()).read().decode('utf-8')
|
36 |
+
|
37 |
+
|
38 |
+
# Split the text data into lines
|
39 |
+
lines = text_data.strip().split('\n')
|
40 |
+
|
41 |
+
# Create a DataFrame
|
42 |
+
df = pd.DataFrame(lines, columns=['message'])
|
43 |
+
|
44 |
+
# Process each line to separate timestamp and text
|
45 |
+
df[['timestamp', 'text']] = df['message'].str.split(']', n=1, expand=True)
|
46 |
+
df['timestamp'] = df['timestamp'].str.strip('[')
|
47 |
+
|
48 |
+
# Handle cases where the split might not work (e.g., missing ']' in a line)
|
49 |
+
df.dropna(subset=['timestamp', 'text'], inplace=True)
|
50 |
+
|
51 |
+
# Convert timestamp to datetime and remove the time, keeping only the date
|
52 |
+
df['timestamp'] = pd.to_datetime(df['timestamp'], format='%d/%m/%y, %H:%M:%S', errors='coerce').dt.date
|
53 |
+
|
54 |
+
# Drop rows where the timestamp conversion failed (which results in NaT)
|
55 |
+
df.dropna(subset=['timestamp'], inplace=True)
|
56 |
+
|
57 |
+
# Remove initial WhatsApp system messages in English and Spanish
|
58 |
+
filter_text_en = "Your messages and calls are end-to-end encrypted"
|
59 |
+
filter_text_es = "Los mensajes y las llamadas están cifrados de extremo a extremo"
|
60 |
+
df = df[~df['text'].str.contains(filter_text_en, na=False)]
|
61 |
+
df = df[~df['text'].str.contains(filter_text_es, na=False)]
|
62 |
+
|
63 |
+
# Additional preprocessing steps:
|
64 |
+
# Remove URLs and convert text to lowercase
|
65 |
+
df['text'] = df['text'].apply(lambda x: re.sub(r'https?:\/\/\S+', '', x)) # Remove URLs
|
66 |
+
df['text'] = df['text'].apply(lambda x: x.lower()) # Convert text to lowercase
|
67 |
+
|
68 |
+
# Remove emojis, images, stickers, documents while preserving colons after sender names
|
69 |
+
df['text'] = df['text'].apply(lambda x: re.sub(r'(?<!\w)(:\s|\s:\s|\s:)', '', x)) # Remove colons that are not part of sender's name
|
70 |
+
df['text'] = df['text'].apply(lambda x: re.sub(r'\[image omitted\]', '', x)) # Remove images
|
71 |
+
df['text'] = df['text'].apply(lambda x: re.sub(r'\[sticker omitted\]', '', x)) # Remove stickers
|
72 |
+
df['text'] = df['text'].apply(lambda x: re.sub(r'\[document omitted\]', '', x)) # Remove documents
|
73 |
+
df['text'] = df['text'].apply(lambda x: re.sub(r'<se editó este mensaje.>', '', x)) # Remove editing function (new Whatsapp addition) in Spanish
|
74 |
+
df['text'] = df['text'].apply(lambda x: re.sub(r'<this message was edited.>', '', x)) # Remove editing function (new Whatsapp addition) in English I AM GUESSING IDk
|
75 |
+
|
76 |
+
# Group by date and concatenate all messages from the same date
|
77 |
+
df = df.groupby('timestamp')['text'].apply(lambda x: '\n'.join(x)).reset_index()
|
78 |
+
df.columns = ['date', 'text']
|
79 |
+
df['date'] = pd.to_datetime(df['date'])
|
80 |
+
df['text'] = df['text'].astype(str)
|
81 |
+
|
82 |
+
return df
|
83 |
+
|
84 |
+
def get_dated_input(data, selected_date):
|
85 |
+
'''
|
86 |
+
The Pandas dataframe is processed and the text is extracted.
|
87 |
+
:param data:
|
88 |
+
:param selected_date:
|
89 |
+
:return:
|
90 |
+
'''
|
91 |
+
selected_date = pd.to_datetime(selected_date)
|
92 |
+
data_for_model = data[data['date'].dt.date == selected_date.date()]
|
93 |
+
data_for_model.loc[:, 'text'] = data_for_model['text']
|
94 |
+
first_row_text = data_for_model['text'].iloc[0]
|
95 |
+
return first_row_text
|
requirements.txt
ADDED
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
torch==2.2.2
|
2 |
+
pandas==2.2.2
|
3 |
+
transformers==4.39.3
|
4 |
+
streamlit==1.33.0
|
5 |
+
git+https://github.com/huggingface/peft.git
|