<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
import json
import re

import torch

# Try import ing external module, if not, pip install the pacage and import 
try:
    from transformers import BertModel, BertTokenizer
except:
    !pip install transformers
    from transformers import BertModel, BertTokenizer

import sklearn
from sklearn_pandas import DataFrameMapper
import numpy as np
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction import DictVectorizer
from sklearn.feature_extraction.text import HashingVectorizer
# set the device to the used
device = 'cuda' if torch.cuda.is_available() else 'cpu'<jupyter_output><empty_output><jupyter_text>Dataset generationThis section can be skipped if you have the preprocessed data in your drive.  First, we read the CSV file into a Pandas DataFrame.<jupyter_code>df = pd.read_csv("https://fbpac-images.s3.amazonaws.com/fbpac/en-US.csv.gz")<jupyter_output><empty_output><jupyter_text>Some useful information of the Dataframe<jupyter_code>print(f"Total number of ads: {df.shape[0]}")
print(f"Number of political ads: {(df['political'] >= 1).sum()}")
print(f"Percentage of political ads: {(df['political'] >= 1).sum()/df.shape[0]}")


targets = []

def update_targets(target):
    '''
        This method parsees the target JSON file and appends any new target to the targets list.
    '''
    if pd.isna(target):
        return
    target = json.loads(target)
    for t in target:
        if t['target'] not in targets:
            targets.append(t['target'])


# apply the update_targets method to every row of the DataFrame.
df.apply(lambda row: update_targets(row['targets']), axis=1)
print(f"possible values of targets: \n{targets}")<jupyter_output>Total number of ads: 222168
Number of political ads: 44277
Percentage of political ads: 0.19929512801123475<jupyter_text>PreprocessWe want to parse the `target` column and use that information to construct additional feature columns ot every data based on the target information.<jupyter_code>def update_row_with_targets(row):
     if pd.isna(row['targets']):
         return row
     tgt = json.loads(row['targets'])
     for t in tgt:
         if 'segment' in t:
            row[t['target']] = t['segment']
     return row
 
 
 df = df.append(pd.DataFrame(columns=targets)).apply(lambda row: update_row_with_targets(row), axis=1)<jupyter_output><empty_output><jupyter_text>Preprocess dataset<jupyter_code>df.drop(columns=['Unnamed: 0','id','html', 'thumbnail', 'targeting', 'suppressed',
       'targets','thumbnail', 'created_at', 'updated_at', 'lang', 'images','listbuilding_fundraising_proba',
       'page_id', 'List','entities', 'page', 'lower_page','targetings', 'Mobile App'], inplace=True)
def cleanhtml(raw_html):
    cleanr = re.compile('<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});')
    cleantext = re.sub(cleanr, '', raw_html)
    return cleantext

df['message'] = df.apply(lambda row: cleanhtml(row['message']), axis=1)<jupyter_output><empty_output><jupyter_text>Save processed CSV file to drivePlease note that to do this, you will have to connect drive to this notebook. Then, mention the entire path (including file name) from the root of your drive. An example is given as the default. Use the form in the following cell to give the path.<jupyter_code>#@title File path if you want to save csv to Drive
path_in_drive = "UMass/datasets/processed_ad_dataset.csv" #@param {type:"string"}
full_path = f"/content/drive/My Drive/{path_in_drive}"
df.to_csv(full_path)<jupyter_output><empty_output><jupyter_text>Great! You now have the entire preprocessed Ad dataset saved to your drive!  [WIP]Modeling<jupyter_code># Run this ONLY if you're starting the notebook from here. Please set teh drive path in the previous cell
df = pd.read_csv(full_path)
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
sample_ids = tokenizer.encode('Hello this is Abdul')
print(f"Hello this is Abdul == {sample_ids}")
model = BertModel.from_pretrained('bert-base-uncased', output_hidden_states=True)
model.eval()
def get_embedding(sent):
    toks = tokenizer.encode(sent)
    with torch.no_grad():
        out = model(input_ids=torch.LongTensor(toks).unsqueeze(0))
    hidden_states = out[2]
    sent_embed = torch.mean(hidden_states[-1], dim=1).squeeze()
    return sent_embed<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1178226917776757, 'fraction_numerical': 0.011943040881947635, 'mean_word_length': 5.27521613832853, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 31, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '158652', 'license': 'MIT', 'n_tokens_mistral': 1437, 'n_tokens_neox': 1388, 'n_words': 397, 'path': 'Ad_dataset.ipynb', 'repo_name': 'hannanabdul55/neural-nets-research-playground', 'size': '4354'}