<jupyter_start><jupyter_text>Text Classification Using RNN (Qualitative or Quantitative)<jupyter_code>import numpy as np

import tensorflow_datasets as tfds
import tensorflow as tf

tfds.disable_progress_bar()
from google.colab import drive
drive.mount("/content/gdrive")
import pandas as pd
df = pd.read_csv('/content/gdrive/My Drive/Colab Notebooks/test_data/bertbaseuncased/cases_worknote.csv',encoding="latin1" )
df.head()
import re
import spacy
def regex_sub(text):
    ip_pattern = r'((((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))([^0-9]|$))|(((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?([^A-Za-z0-9]|$)))'
    yyyy_date_pattern = r'([1-2][0-9]{3}[-\.\/])((([1-9]|1[0-2]|0[1-9])[-\.\/]([1-9]|0[1-9]|1[0-9]|2[0-9]|3[0-1])[^0-9])|(([1-9]|0[1-9]|1[0-9]|2[0-9]|3[0-1])[-\.\/]([1-9]|0[1-9]|1[0-2])[^0-9]))'
    m_d_y_data_pattern = r'[^0-9]((([1-9]|0[1-9]|1[0-2])[-\.\/]([1-9]|0[1-9]|1[0-9]|2[0-9]|3[0-1]))|(([1-9]|0[1-9]|1[0-9]|2[0-9]|3[0-1])[-\.\/]([1-9]|1[0-2]|0[1-9])))[-\.\/]([1-2][0-9]{3}|[0-9]{2})'
    timestamp_pattern_1 = r'([1-2][0-9]{3}[-\.\/])((([1-9]|1[0-2]|0[1-9])[-\.\/]([1-9]|0[1-9]|1[0-9]|2[0-9]|3[0-1])[^0-9])|(([1-9]|0[1-9]|1[0-9]|2[0-9]|3[0-1])[-\.\/]([1-9]|0[1-9]|1[0-2])))[\s]([0-9]|[0-1][0-9]|2[0-3]):([0-9]|[0-5][0-9]):([0-9]|[0-5][0-9])[^0-9]'
    timestamp_pattern_2 = r'[^0-9]((([1-9]|0[1-9]|1[0-2])[-\.\/]([1-9]|0[1-9]|1[0-9]|2[0-9]|3[0-1]))|(([1-9]|0[1-9]|1[0-9]|2[0-9]|3[0-1])[-\.\/]([1-9]|1[0-2]|0[1-9])))[-\.\/]([1-2][0-9]{3}|[0-9]{2})[\s]([0-9]|[0-1][0-9]|2[0-3]):([0-9]|[0-5][0-9]):([0-9]|[0-5][0-9])[^0-9]'
    windows_file_path = r'[A-Za-z]:[\\][^\s]*'
    unix_file_path = r''
    url_pattern = r'http[s]?:[\/]{2}[^\s]+'
    email_pattern = r'[\S]+@[^\s\.]+\.[\S]+'
    html_tag = r'\<[^\>]*\>'
    text = re.sub(html_tag,' ',text)
    text = re.sub(windows_file_path,' filepath ',text)
    text = re.sub(url_pattern,' url ',text)
    text = re.sub(email_pattern,' email ',text)
    text = re.sub(ip_pattern,' ipaddress ',text)
    text = re.sub(timestamp_pattern_1,' timestamp ',text)
    text = re.sub(timestamp_pattern_2,' timestamp ',text)
    text = re.sub(yyyy_date_pattern,' timestamp ',text)
    text = re.sub(m_d_y_data_pattern,' timestamp ',text)
    text = text.lower()
    text = re.sub('[^a-z0-9\s]',' ',text)
#     text = nlp_oper(text)
    return text
df['note'] = df['note'].apply(lambda x:regex_sub(x))
df.head()
def user(string):
  pattern =r'[A-Za-z]+(?=\d{0,}@)'
  username= re.findall(pattern,string)
  #print('username is :',username)
  return username
df['user1'] = df['user'].apply(lambda x:user(x))
df.head()



def username(string):
  for (user1,item) in df.iteritems():
    if item.values == 'system':
      print("username is: ",item.values)
    else:
        print("username is : ",item.values)
  return string 
  
df['user2'] = df['user1'].apply(lambda x:username(x))
df['user2']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.42208774583963693, 'fraction_numerical': 0.14346949067070094, 'mean_word_length': 4.471724137931035, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '50312', 'license': 'MIT', 'n_tokens_mistral': 2471, 'n_tokens_neox': 2374, 'n_words': 144, 'path': 'test1.ipynb', 'repo_name': 'anitha67/100DaysofMLCode', 'size': '3966'}