marcelomoreno26 commited on
Commit
9d54e27
1 Parent(s): dd204e1

Upload 3 files

Browse files
Files changed (3) hide show
  1. model_functions.py +91 -0
  2. preprocessor.py +95 -0
  3. requirements.txt +5 -0
model_functions.py ADDED
@@ -0,0 +1,91 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from transformers import (AutoModelForSequenceClassification, AutoModelForSeq2SeqLM,
3
+ AutoConfig, AutoModelForTokenClassification,
4
+ AutoTokenizer, pipeline)
5
+ from peft import PeftModel, PeftConfig
6
+
7
+
8
+
9
+
10
+ def load_sentiment_analyzer():
11
+ tokenizer = AutoTokenizer.from_pretrained("aliciiavs/sentiment-analysis-whatsapp2")
12
+ model = AutoModelForSequenceClassification.from_pretrained("aliciiavs/sentiment-analysis-whatsapp2")
13
+
14
+ return tokenizer, model
15
+
16
+ def load_summarizer():
17
+ config = PeftConfig.from_pretrained("marcelomoreno26/bart-large-samsum-adapter")
18
+ model = AutoModelForSeq2SeqLM.from_pretrained("facebook/bart-large")
19
+ tokenizer = AutoTokenizer.from_pretrained("facebook/bart-large")
20
+ tokenizer.pad_token = tokenizer.eos_token
21
+ model = PeftModel.from_pretrained(model, "marcelomoreno26/bart-large-samsum-adapter", config=config)
22
+ model = model.merge_and_unload()
23
+
24
+ return tokenizer, model
25
+
26
+ def load_NER():
27
+ config = AutoConfig.from_pretrained("hannahisrael03/distilbert-base-uncased-finetuned-wikiann")
28
+ model = AutoModelForTokenClassification.from_pretrained("hannahisrael03/distilbert-base-uncased-finetuned-wikiann",config=config)
29
+ tokenizer = AutoTokenizer.from_pretrained("hannahisrael03/distilbert-base-uncased-finetuned-wikiann")
30
+ pipe = pipeline("ner", model=model, tokenizer=tokenizer, aggregation_strategy="average")
31
+
32
+ return pipe
33
+
34
+ def get_sentiment_analysis(text, tokenizer, model):
35
+ inputs = tokenizer(text, padding=True, return_tensors="pt")
36
+ with torch.no_grad():
37
+ outputs = model(**inputs)
38
+ # Get predicted probabilities and predicted label
39
+ probabilities = torch.softmax(outputs.logits, dim=1)
40
+ predicted_label = torch.argmax(probabilities, dim=1)
41
+ # Convert the predicted label tensor to a Python integer
42
+ predicted_label = predicted_label.item()
43
+ # Map predicted label index to sentiment label
44
+ label_dic = {0: 'sadness', 1: 'joy', 2: 'love', 3: 'anger', 4: 'fear', 5: 'surprise'}
45
+ # Print the predicted sentiment label
46
+ return label_dic[predicted_label]
47
+
48
+
49
+ def generate_summary(text, tokenizer, model):
50
+ prefix = "summarize: "
51
+ encoded_input = tokenizer.encode_plus(prefix + text, return_tensors='pt', add_special_tokens=True)
52
+ input_ids = encoded_input['input_ids']
53
+
54
+ # Check if input_ids exceed the model's max length
55
+ max_length = 512
56
+ if input_ids.shape[1] > max_length:
57
+ # Split the input_ids into manageable segments
58
+ total_summary = []
59
+ for i in range(0, input_ids.shape[1], max_length - 50): # We use max_length - 50 to allow for some room for the model to generate context
60
+ segment_ids = input_ids[:, i:i + max_length]
61
+ output_ids = model.generate(segment_ids, max_length=150, num_beams=5, early_stopping=True)
62
+ segment_summary = tokenizer.decode(output_ids[0], skip_special_tokens=True)
63
+ total_summary.append(segment_summary)
64
+
65
+ # Concatenate all segment summaries
66
+ summary = ' '.join(total_summary)
67
+ else:
68
+ # Process as usual
69
+ output_ids = model.generate(input_ids, max_length=150, num_beams=5, early_stopping=True)
70
+ summary = tokenizer.decode(output_ids[0], skip_special_tokens=True)
71
+
72
+ return summary
73
+
74
+
75
+ def get_NER(text, pipe):
76
+ # Use pipeline to predict NER
77
+ results = pipe(text)
78
+ # Filter duplicates while retaining the highest score for each entity type and word combination
79
+ unique_entities = {}
80
+ for ent in results:
81
+ key = (ent['entity_group'], ent['word'])
82
+ if key not in unique_entities or unique_entities[key]['score'] < ent['score']:
83
+ unique_entities[key] = ent
84
+
85
+ # Prepare the output, sorted by the start position to maintain the order they appear in the text
86
+ filtered_results = sorted(unique_entities.values(), key=lambda x: x['start'])
87
+ # Format the results for a table display
88
+ formatted_results = [[ent['word'], ent['entity_group']] for ent in filtered_results]
89
+
90
+ return formatted_results
91
+
preprocessor.py ADDED
@@ -0,0 +1,95 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import pandas as pd
2
+ import zipfile
3
+ import re
4
+ from io import BytesIO
5
+
6
+
7
+ def detect_file_type(file_path):
8
+ type = file_path[-3:]
9
+ print(type)
10
+ if type in ["txt","zip"]:
11
+ return type
12
+ else:
13
+ return "unknown"
14
+
15
+ def preprocess_whatsapp_messages(file_path, file_type):
16
+ """
17
+ Preprocesses the Whatsapp messages zip file into a Pandas Dataframe, all messages in one day go
18
+ to a row and a timestamp is added.
19
+
20
+ Args:
21
+ file_path (str): Location of the file (zip or txt) of the conversation.
22
+
23
+ Returns:
24
+ str: Dataframe
25
+ """
26
+
27
+ # Load the zip file and extract text data
28
+ print(file_type)
29
+ if file_type == "zip":
30
+ with zipfile.ZipFile(file_path, 'r') as z:
31
+ file_name = z.namelist()[0]
32
+ with z.open(file_name) as file:
33
+ text_data = file.read().decode('utf-8')
34
+ else:
35
+ text_data = BytesIO(file_path.getvalue()).read().decode('utf-8')
36
+
37
+
38
+ # Split the text data into lines
39
+ lines = text_data.strip().split('\n')
40
+
41
+ # Create a DataFrame
42
+ df = pd.DataFrame(lines, columns=['message'])
43
+
44
+ # Process each line to separate timestamp and text
45
+ df[['timestamp', 'text']] = df['message'].str.split(']', n=1, expand=True)
46
+ df['timestamp'] = df['timestamp'].str.strip('[')
47
+
48
+ # Handle cases where the split might not work (e.g., missing ']' in a line)
49
+ df.dropna(subset=['timestamp', 'text'], inplace=True)
50
+
51
+ # Convert timestamp to datetime and remove the time, keeping only the date
52
+ df['timestamp'] = pd.to_datetime(df['timestamp'], format='%d/%m/%y, %H:%M:%S', errors='coerce').dt.date
53
+
54
+ # Drop rows where the timestamp conversion failed (which results in NaT)
55
+ df.dropna(subset=['timestamp'], inplace=True)
56
+
57
+ # Remove initial WhatsApp system messages in English and Spanish
58
+ filter_text_en = "Your messages and calls are end-to-end encrypted"
59
+ filter_text_es = "Los mensajes y las llamadas están cifrados de extremo a extremo"
60
+ df = df[~df['text'].str.contains(filter_text_en, na=False)]
61
+ df = df[~df['text'].str.contains(filter_text_es, na=False)]
62
+
63
+ # Additional preprocessing steps:
64
+ # Remove URLs and convert text to lowercase
65
+ df['text'] = df['text'].apply(lambda x: re.sub(r'https?:\/\/\S+', '', x)) # Remove URLs
66
+ df['text'] = df['text'].apply(lambda x: x.lower()) # Convert text to lowercase
67
+
68
+ # Remove emojis, images, stickers, documents while preserving colons after sender names
69
+ df['text'] = df['text'].apply(lambda x: re.sub(r'(?<!\w)(:\s|\s:\s|\s:)', '', x)) # Remove colons that are not part of sender's name
70
+ df['text'] = df['text'].apply(lambda x: re.sub(r'\[image omitted\]', '', x)) # Remove images
71
+ df['text'] = df['text'].apply(lambda x: re.sub(r'\[sticker omitted\]', '', x)) # Remove stickers
72
+ df['text'] = df['text'].apply(lambda x: re.sub(r'\[document omitted\]', '', x)) # Remove documents
73
+ df['text'] = df['text'].apply(lambda x: re.sub(r'<se editó este mensaje.>', '', x)) # Remove editing function (new Whatsapp addition) in Spanish
74
+ df['text'] = df['text'].apply(lambda x: re.sub(r'<this message was edited.>', '', x)) # Remove editing function (new Whatsapp addition) in English I AM GUESSING IDk
75
+
76
+ # Group by date and concatenate all messages from the same date
77
+ df = df.groupby('timestamp')['text'].apply(lambda x: '\n'.join(x)).reset_index()
78
+ df.columns = ['date', 'text']
79
+ df['date'] = pd.to_datetime(df['date'])
80
+ df['text'] = df['text'].astype(str)
81
+
82
+ return df
83
+
84
+ def get_dated_input(data, selected_date):
85
+ '''
86
+ The Pandas dataframe is processed and the text is extracted.
87
+ :param data:
88
+ :param selected_date:
89
+ :return:
90
+ '''
91
+ selected_date = pd.to_datetime(selected_date)
92
+ data_for_model = data[data['date'].dt.date == selected_date.date()]
93
+ data_for_model.loc[:, 'text'] = data_for_model['text']
94
+ first_row_text = data_for_model['text'].iloc[0]
95
+ return first_row_text
requirements.txt ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ torch==2.2.2
2
+ pandas==2.2.2
3
+ transformers==4.39.3
4
+ streamlit==1.33.0
5
+ git+https://github.com/huggingface/peft.git