<jupyter_start><jupyter_text>CleaningCleaning of provided dataset and enhancement of features<jupyter_code>import numpy as np
import pandas as pd
import pickle

import matplotlib.pyplot as plt
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Input _Privacy restriction:_ Original (personal) dataset not in Repo. Provide self collected features and ratings about attended events in "../data/events.csv" (schema info in REFS.md)<jupyter_code>data_f = "../data/events.csv"
raw_data = pd.read_csv(data_f)

print(raw_data.shape)
print(raw_data.dtypes)
raw_data.head()<jupyter_output><empty_output><jupyter_text>Cleaning<jupyter_code>raw_data.date = pd.to_datetime(raw_data.date, format="%d.%m.%Y")
raw_data.start_time = pd.to_datetime(raw_data.start_time, format="%H:%M").dt.time
raw_data.duration = pd.to_datetime(raw_data.duration, format="%H:%M").dt.time
raw_data["weekday"] = raw_data.date.dt.weekday_name
raw_data.ticket_prize = raw_data.ticket_prize.replace("[\€,]", "", regex=True).astype(int)<jupyter_output><empty_output><jupyter_text>Enhancement __Buzzwords__<jupyter_code># Generate Buzzword List
buzzword_list = []

def parse_buzzword_file(file_path, buzzword_list):
    """
        parses textfile and appends word to buzzword list
    """
    with open(file_path, "r") as f:
        content = f.readlines()
        for buzzword in content:
            buzzword_list.append(buzzword.strip("\n").lower())
    return buzzword_list

buzzwords_wiki = parse_buzzword_file("../data/buzzwords_wiki.txt", buzzword_list)
buzzwords_de = parse_buzzword_file("../data/buzzwords_de.txt", buzzword_list)
buzzwords_personal = parse_buzzword_file("../data/buzzwords_personal.txt", buzzword_list)

print("Buzzword Count:", len(buzzword_list))
# Pickling Buzzword List for App
file_path = "../data/buzzword_list.pkl"
with open(file_path, "wb") as f:
    pickle.dump(buzzword_list, f)
# Create binary colunms (buzzwordy_title, buzzwordy_organizer)
buzzword_mask_title = raw_data.title.str.contains("|".join(buzzword_list), case=False)
buzzword_mask_organizer =  raw_data.organizer.str.contains("|".join(buzzword_list), case=False)

raw_data["buzzwordy_title"] = np.where(buzzword_mask_title, 1, 0)
raw_data["buzzwordy_organizer"] = np.where(buzzword_mask_organizer, 1, 0)<jupyter_output><empty_output><jupyter_text>Check DF and Output __Checking__<jupyter_code>print(raw_data.shape)
print(raw_data.dtypes)
raw_data.head()<jupyter_output><empty_output><jupyter_text>__Output__<jupyter_code># Pickling DF
file_path = "../data/events_df.pkl"
raw_data.to_pickle(file_path)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12730769230769232, 'fraction_numerical': 0.0015384615384615385, 'mean_word_length': 7.052631578947368, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '572544', 'license': 'Apache-2.0', 'n_tokens_mistral': 977, 'n_tokens_neox': 958, 'n_words': 151, 'path': 'EventDec/event_dec/notebook/0_Cleaning.ipynb', 'repo_name': 'datadonK23/pyData-Projects', 'size': '2600'}