<jupyter_start><jupyter_text>RegEx Workbook<jupyter_code># import modules
import json
import pandas as pd
import re
import regex
from datetime import datetime
pd.set_option("display.max_rows", 999)
pd.set_option("display.max_columns", 999)
# import twitter data
data_json = open('data/tweets/2017-12-01.json', mode='r').read()  
df = pd.read_json('data/tweets/2017-12-01.json')
df
df.dtypes<jupyter_output><empty_output><jupyter_text>Data Preprocessing<jupyter_code># extract all hashtags into new column
df['hashtags'] = df.text.apply(lambda x: re.findall(r'#[\S]+', str(x)))
df.hashtags.head()
df.text[4]
df.text[6]
# extract all callouts
df['callouts'] = df.text.apply(lambda x: re.findall(r'@[\S]+', str(x)))
df['callouts'].head()
charref = [x for x in df.text.apply(lambda x: re.findall(r'&[\S]+', str(x)))]
charref
flat_list = [item for sublist in charref for item in sublist]
len(flat_list)
# replace all hashtags, ampersands, and callouts with no space
df.text = df.text.apply(lambda x: re.sub(r'[#@&][\S]+', '', str(x)))
df.text.head()
# remove twitter RT and QT
df.text = df.text.apply(lambda x: re.sub(r'(\bRT\b|\bQT\b)', '', str(x)))
df.text.head()
# remove emoji
df.text = df.text.apply(lambda x: re.sub(r'\B[^\w\s]{2,}\B', ' ', str(x)))
df.text.head()
# remove html links
df.text = df.text.apply(lambda x:re.sub(r'(http|ftp|https)://([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?', '', str(x)))
df.text.head()
# remove punctuation
df.text = df.text.apply(lambda x: re.sub(r'[^\w\s]', r'', str(x)))
df.text.head()
# lower casing
df.text = df.text.apply(lambda x: " ".join(x.lower() for x in x.split()))
df.text.head()
# remove numbers or words with digits  
df.text = df.text.apply(lambda x: re.sub(r'\w*\d\w*', r'', str(x)))
df.text.head()
# remove any double or more whitespaces to single whitespaces
df.text = df.text.apply(lambda x: re.sub(r'\s\s+', ' ', str(x)))
df.text.head()
# remove any leading and trailing whitespace
df.text = df.text.apply(lambda x: re.sub(r'(\A\s+|\s+\Z)', '', str(x)))
df.text.head()
df.text.tail()<jupyter_output><empty_output><jupyter_text>Time Column<jupyter_code>df.time.head()
import regex
df['tdmatch'] = [x for x in df.time.apply(lambda x: regex.match(r'^(?P<year>\d{4})-(?P<month>\d{2})-(?P<day>\d{2})T(?P<time>\d{2}:\d{2}:\d{2})-\d{2}:\d{2}$', str(x)))]
df.tdmatch = df.tdmatch.apply(lambda x: x.groupdict())
df.tdmatch.head()
df['year'] = df.tdmatch.apply(lambda x: x['year'])
df['month'] = df.tdmatch.apply(lambda x: x['month'])
df['time_hms'] = df.tdmatch.apply(lambda x: x['time'])
df['day'] = df.tdmatch.apply(lambda x: x['day'])
pd.set_option("display.max_rows", 20)
pd.set_option("display.max_columns", 20)
df.head()
format = "%Y-%m-%dT%H:%M:%S%z"
df['datetime'] = df.time.apply(lambda x: datetime.strptime(x, format))
df.datetime.head()
df.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.24028021015761822, 'fraction_numerical': 0.012959719789842382, 'mean_word_length': 5.4324324324324325, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '134671', 'license': 'MIT', 'n_tokens_mistral': 1218, 'n_tokens_neox': 1184, 'n_words': 227, 'path': 'RegEx.ipynb', 'repo_name': 'examsherpa/Medium-Blogs', 'size': '2855'}