<jupyter_start><jupyter_text>Character Set<jupyter_code>char_set = set()
df.apply(lambda x : [char_set.add(c) for c in x])
df = df.apply(lambda x : x.replace('"' , ""))
df[0]
len(char_set)
ord('0')
def isEnglish(s):
    try:
        s.encode(encoding='utf-8').decode('ascii')
    except UnicodeDecodeError:
        return False
    else:
        return True
english_chars = [chr(i) for i in range(ord('a') , ord('z')+1)]
#english_chars += [chr(i) for i in range(ord('0') , ord('9')+1)]
english_chars
df[df.map(lambda x : isEnglish(x))]
df[df.map(lambda x : any([c in english_chars for c in x]))]
sum([])
def count_chars(s , cs):
    res=0

    for c in s:
        res += 1 if c in cs else 0
    return res
df =df.sort_values(ascending=False,key=lambda x :x.apply(lambda xx : count_chars(xx , english_chars)))
df.head(20)
df[221855]<jupyter_output><empty_output><jupyter_text>Pre processed data examination<jupyter_code>pathpre = "../Data/PreProcessed/telegram_corona981_normalized_tokenized_20.csv"
dfpre = pd.read_csv(pathpre)

new_header = dfpre.iloc[0] #grab the first row for the header
dfpre = dfpre[1:] #take the data less the header row
dfpre.columns = new_header #set the header row as the df header
dfpre.head()

dfpre.iloc[20]['textField_nlp_normal']
dfpre.iloc[20]['textField_nlp_process']
char_set2 = set()
dfpre['textField_nlp_process'].apply(lambda x : [char_set2.add(c) for c in x])
len(char_set2)
len(char_set)
len(df)
len(dfpre)
char_set.difference(char_set2)
dfpre['tokens'].iloc[0]
dfpre['tokenlist'] = dfpre['tokens'].apply(lambda x : x.replace('"' , "").split('|'))
dfpre.iloc[0]['tokenlist'][3]
def no_word(c):
    if ""
words = set()
dfpre['tokenlist'].apply(lambda x : [words.add(c)  for c in x])
words
worddocument=[]<jupyter_output><empty_output><jupyter_text>work on raw data<jupyter_code>df.head()
char_set = set()
df['textField'].apply(lambda x : [char_set.add(c) for c in x])
len(char_set)
def print_field(df , field):
    df.apply(lambda x : print(type(x)))
from langdetect import detect_langs , detect
detect("dsadsada")
temp = df.iloc[200:300]
temp[temp['textField'].map(lambda x : detect(x)!='fa')]['textField'].iloc[3]
df.iloc[0:100]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16924476797088261, 'fraction_numerical': 0.021383075523202913, 'mean_word_length': 5.623493975903615, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '563405', 'license': 'MIT', 'n_tokens_mistral': 885, 'n_tokens_neox': 858, 'n_words': 186, 'path': 'CharacterWordManipulation.ipynb', 'repo_name': 'sajjadGG/CovidSocial2020', 'size': '2198'}