<jupyter_start><jupyter_text>Actions performed Prototype how to load the wall street journal data.  Evaluation methods to compare trees comparison with google word vectors and convert words which are not there to UNK or some other symbol Filters sentences which are more than MAX_WORDS_COUNT words. Made the sentence equal to length MAX_WORDS_COUNT by adding &lt; /fill &gt; word  Load training and test dataset  Use BracketParse corpus reader to load the wall street journal data functions subtrees(), label(), leaves() are important checkout load-penn-treebank-dataset.py for protoypes<jupyter_code>dirname = os.getcwd()
dirname = os.path.dirname(dirname)
dataset_path = os.path.join(dirname, 'datasets/')
print(dataset_path)
## https://www.nltk.org/_modules/nltk/tree.html
## http://www.nltk.org/howto/tree.html
## above link contains the API and also some tutorials

#reader = nltk.corpus.BracketParseCorpusReader('.','SWB-all-sentences-original-with-punctuation.MRG')
reader = nltk.corpus.BracketParseCorpusReader(dataset_path,'WSJ.txt')
print(reader.fileids())
print(type(reader))

## reads the file and converts each line into a tree
all_trees = reader.parsed_sents()
print('No. of trees: ', len(all_trees))
print(type(all_trees[0]))
def get_frequency(trees):
    frequency_dict = {}
    for tree in trees:
        words = tree.leaves()
        words = [word.lower() for word in words]
        for word in words:
            if word not in frequency_dict.keys():
                frequency_dict[word] = 0
            frequency_dict[word] = frequency_dict[word] + 1
    return frequency_dict

def filter_trees(trees, threshold_value, frequency_dict):
    filtered_trees = []
    
    if(frequency_dict is None):
        frequency_dict = get_frequency(trees)
    
    print('No.of words: ',len(frequency_dict))
    keys = list(frequency_dict.keys())
    for key in keys:
        if(frequency_dict[key] <= threshold_value) :
            frequency_dict.pop(key, None)
    print('No. of words whose frequency is more than ',threshold_value,' is:', len(frequency_dict))
    
    print('No. of trees:',len(trees))
    filtered_trees = []
    for tree in trees:
        words = tree.leaves()
        words = [word.lower() for word in words]
        isFrequent = True
        for word in words:
            if word not in frequency_dict.keys():
                isFrequent = False
                break
        if(isFrequent):
            filtered_trees.append(tree)
    print('No. of filtered trees:',len(filtered_trees))
    
    return filtered_trees

FREQUENCY_THRESHOLD_VALUE = 2
frequency_dict = get_frequency(all_trees)
trees = filter_trees(all_trees, FREQUENCY_THRESHOLD_VALUE, frequency_dict)<jupyter_output>No.of words:  44377
No. of words whose frequency is more than  2  is: 18198
No. of trees: 49208
No. of filtered trees: 28837<jupyter_text>Evaluation methods to compare trees equality operator (==) is good enough to compare trees. When comparing both the tree and tree label is taken into account Initial methods will not have categorization classification technique. So, it is important to replace all the values with say UNK symbol of google corpus It is created in such a way that subcategories of punctuations are named by itself  Preprocess Tree to chomsky normal form, collapse unary and make it UNLabelled<jupyter_code>NO_LABEL = 'NO_LABEL'

def set_no_label(tree):
    if(type(tree[0]) == type('a string')):
        # we have reached the bottom most parent
        tree.set_label(NO_LABEL)
        return
   
    # accesses the left child
    set_no_label(tree[0])
    
    # accesses the right child
    set_no_label(tree[1])

    tree.set_label(NO_LABEL)
    

def preprocess_tree(tree):
    
    # set chomsky normal form
    tree.chomsky_normal_form()
    # merge the unary branches
    tree.collapse_unary(collapsePOS=True, collapseRoot=True)
    # set the label with 'NO_LABEL'
    set_no_label(tree)
    
#temp_trees = [preprocess_tree(tree) for tree in trees]
#temp_trees[0]
tree = trees[10]
preprocess_tree(tree)
tree
preprocessed_trees = []
for i in range(len(trees)):
    tree = trees[i]
    preprocess_tree(tree)
    preprocessed_trees.append(tree)
    
preprocessed_trees[1000]<jupyter_output><empty_output><jupyter_text>Convert the words which does not appear in Google word embeddings with UNK or other symbolAs suggested in 'check-embeddings-wallstreet-coverage' google word embeddings are better than glove embeddings<jupyter_code>UNK = '</s>'
FILLER = '</fill>'

outfile = dataset_path +'google_word_corpus.pic'

with open(outfile, 'rb') as pickle_file:    
    googleCorpus, google_corpus_word_to_int, google_corpus_int_to_word = pickle.load(pickle_file)
#googleSet = pd.read_csv(dataset_path+'GoogleNews-vectors-negative10.txt', sep=' ', header=None)

#print(googleSet.shape)
#print(googleSet.head())

# TO BE DONE - Appropriate value should be added for FILLER. Right now it is a range from 1 to STATE_SIZE
#fill_data = {0:FILLER}
#for i in range(1, googleSet.shape[1]):
#    fill_data[i] = i
#googleSet = googleSet.append(fill_data, ignore_index=True)

#print(googleSet.shape)
#print(googleSet.head())

#temp = list(googleCorpus)
#temp.append(FILLER)
#googleCorpus = set(temp)
#corpus_length = len(google_corpus_word_to_int)
#google_corpus_word_to_int[FILLER] = corpus_length
#google_corpus_int_to_word[corpus_length] = FILLER


#googleWords = googleSet.iloc[:,0:1]
#googleVectors = googleSet.iloc[:,1:]
UNK = '</s>' # this is the symbol for UNK in google corpus

def set_UNK(tree):
    if(type(tree[0]) == type('a string')):
        # we have reached the bottom most parent
        tree[0] = tree[0].lower()
        if(tree[0] not in googleCorpus):
            tree[0] = UNK
        return
   
    # accesses the left child
    set_UNK(tree[0])
    
    # accesses the right child
    set_UNK(tree[1])
    
#temp_trees = [preprocess_tree(tree) for tree in trees]
#temp_trees[0]
tree = preprocessed_trees[0]
set_UNK(tree)
tree
preprocessed_unk_trees = []
for i in range(len(preprocessed_trees)):
    tree = preprocessed_trees[i]
    set_UNK(tree)
    preprocessed_unk_trees.append(tree)
    
preprocessed_unk_trees[1000]
def UNK_count_analysis(tree):
    words = tree.leaves()
    words_count = len(words)
    unk_count = 0
    for word in words:
        if word == UNK:
            unk_count = unk_count + 1
    #print(unk_count, words_count)
    return unk_count / words_count

display(preprocessed_unk_trees[0])
UNK_count_analysis(preprocessed_unk_trees[0])
THRESHOLD_PERCENTAGE = 0.25
preprocessed_unk_lt25per_trees = []
print(len(preprocessed_unk_trees))
for i in range(len(preprocessed_unk_trees)):
    tree = preprocessed_unk_trees[i]
    unk_percentage = UNK_count_analysis(tree)
    if(unk_percentage <= THRESHOLD_PERCENTAGE):
        preprocessed_unk_lt25per_trees.append(tree)
print(len(preprocessed_unk_lt25per_trees))<jupyter_output>28837
23292<jupyter_text>Only sentences with <= MAX_WORDS_COUNT words (<MAX_WORDS_COUNT words suffix additional words and make it MAX_WORDS_COUNT)<jupyter_code>FILLER = '</fill>'
MAX_WORDS_COUNT = 10
preprocessed_unk_max_words_trees = []
for i in range(len(preprocessed_unk_lt25per_trees)):
    tree = preprocessed_unk_lt25per_trees[i]
    words_count = len(tree.leaves())
    if(words_count == MAX_WORDS_COUNT): # it has been changed strictly from <= to ==
        for i in range(MAX_WORDS_COUNT - words_count):
            tree = nltk.Tree(NO_LABEL, [tree, nltk.Tree(NO_LABEL, [FILLER])])
        preprocessed_unk_max_words_trees.append(tree)

print(len(preprocessed_unk_lt25per_trees))
print(len(preprocessed_unk_max_words_trees))
preprocessed_unk_max_words_trees[60] #123
def convert_golden_tree(tree, leaves, input_list):
    if(type(tree[0]) == type('some string')):
        #print(tree[0])
        leaveIndex = leaves.index(tree[0])
        leaves[leaveIndex]=''
        return leaveIndex
    
    leftIndex = convert_golden_tree(tree[0], leaves, input_list)
    rightIndex = convert_golden_tree(tree[1], leaves, input_list)
    #print(leftIndex, rightIndex)
    #print(leftIndex)
    input_list.append(leftIndex)
    #return len(leaves) + len(input_list)
    return leftIndex

tree_list =[]
tree_leaves = preprocessed_unk_max_words_trees[0].leaves()
convert_golden_tree(preprocessed_unk_max_words_trees[0], tree_leaves, tree_list)
tree_list
golden_trees_combination = []
for i in range(len(preprocessed_unk_max_words_trees)):
    tree = preprocessed_unk_max_words_trees[i]
    temp_combination = []
    tree_leaves = tree.leaves()
    convert_golden_tree(tree, tree_leaves, temp_combination)
    golden_trees_combination.append(temp_combination)
    
print(len(golden_trees_combination))
# Serialize it as CSV

treesDict = [{'sentence': ' '.join(tree.leaves()), 'tree':tree} for tree in preprocessed_unk_max_words_trees]
treeDataframe = pd.DataFrame(data=treesDict, columns=['sentence', 'tree'])
treeDataframe.head()
treeDataframe.to_csv(dataset_path+'constituency-parsing-data-'+str(MAX_WORDS_COUNT)+'-UNK-NOLABEL.csv', sep=' ', index=False, header=False)
load_treeData = pd.read_csv(dataset_path+'constituency-parsing-data-'+str(MAX_WORDS_COUNT)+'-UNK-NOLABEL.csv', sep=' ', header=None, )
load_treeData.columns =['sentence', 'tree']
load_treeData.head()
print(load_treeData.iat[0,1])
print(type(load_treeData.iat[0,1]))
load_treeData['tree'] = load_treeData['tree'].apply(nltk.Tree.fromstring)
print(load_treeData.iat[0,1])
print(type(load_treeData.iat[0,1]))
load_treeData['length'] = load_treeData['sentence'].apply(lambda x: len(x.split()))
load_treeData.sort_values(by='length',ascending=True, inplace=True)
load_treeData.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10630369526964083, 'fraction_numerical': 0.011178966980643825, 'mean_word_length': 4.637106184364061, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 19, 'https://': 1, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '65020', 'license': 'Apache-2.0', 'n_tokens_mistral': 3361, 'n_tokens_neox': 3212, 'n_words': 740, 'path': 'pre-processing/wallstreet-journal-data-preprocessing.ipynb', 'repo_name': 'Kandy16/constituency-parsing', 'size': '9661'}