<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Parse XML Data2&nbsp;&nbsp;Tokenize Text3&nbsp;&nbsp;Create Word Embeddings with Wiki-Extvec4&nbsp;&nbsp;Create Contextual String Embeddings with Flair5&nbsp;&nbsp;Create BERT Embeddings with flair6&nbsp;&nbsp;Create ELMo Embeddings with flair7&nbsp;&nbsp;Build Dataset7.1&nbsp;&nbsp;Tagging7.2&nbsp;&nbsp;Split Dataset8&nbsp;&nbsp;Data compress  Parse XML Data<jupyter_code>from xml.dom.minidom import parse
import xml.dom.minidom
from collections import Counter


def parse(n):
    """
    Parse XML corpus
    """
    sentence, label = [], []
    DOMTree = xml.dom.minidom.parse(n)
    items = DOMTree.documentElement.getElementsByTagName('item')
    for item in items:
        label.append(item.getAttribute('label'))
        sent = item.getElementsByTagName('sentence')
        sentence.append(sent[0].childNodes[0].data)
    return sentence, label


trainSent, trainLabel = parse('corpus/train-corpus.xml')
testSent, testLabel = parse('corpus/test-corpus.xml')<jupyter_output><empty_output><jupyter_text>Tokenize Text<jupyter_code>from keras.preprocessing.text import text_to_word_sequence
import numpy as np
import pickle
import re
MAX_WLEN = 58
MAX_CLEN = 23


def delete(s):
    """
    Delete parentheses
    """
    return re.sub('\((.*?)\)', '', s)


def cut(s):
    """
    Word segmentation
    """
    ws = text_to_word_sequence(s,
                               filters='!#$&*+.%/<=>?@[\\]^_`{|}~\t\n',
                               lower=False,
                               split=' ')
    for s in [',', ':', ';', '"']:
        sep_pm(ws, s)
    return ws


def sep_pm(ws, s):
    """
    Separate punctuation mark: , : ; " 
    """
    if s in [',', ':', ';', '"']:
        for i in range(len(ws)):
            if ws[i].endswith(s) and ws[i] != s:
                ws[i] = ws[i][:-1]
                ws.insert(i+1, s)
        for i in range(len(ws)):
            if ws[i].endswith(s) and ws[i] != s:
                ws[i] = ws[i][:-1]
                ws.insert(i+1, s)
    if s in ['"']:
        for i in range(len(ws)):
            if ws[i].startswith(s) and ws[i] != s:
                ws[i] = ws[i][1:]
                ws.insert(i, s)
        for i in range(len(ws)):
            if ws[i].startswith(s) and ws[i] != s:
                ws[i] = ws[i][1:]
                ws.insert(i, s)


def find_cp_idx(ws):
    """
    Find index of causality phrases
    """
    E = []
    for n in ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11']:
        e = []
        for i in range(len(ws)):
            if ws[i] == 'e'+n:
                e.append(i-2*(int(n)-1))
                for j in range(i+1, len(ws)):
                    if ws[j] == 'e'+n and j-2 != i:
                        e.append(j-2*int(n))
                break
        if e != []:
            E.append([i for i in range(e[0], e[-1]+1)])
    return E


def format_sentence(ws):
    """
    Remove entity tags
    """
    return [i for i in ws if i not in ['e1', 'e2', 'e3', 'e4', 'e5', 'e6', 'e7', 'e8', 'e9', 'e10', 'e11']]


train_index = []
trainSent = [delete(i) for i in trainSent]
trainWords = [cut(i) for i in trainSent]
for w in trainWords:
    train_index.append(find_cp_idx(w))
trainWords = [format_sentence(i) for i in trainWords]
trainChars = [[list(w) for w in s] for s in trainWords]

test_index = []
testSent = [delete(i) for i in testSent]
testWords = [cut(i) for i in testSent]
for w in testWords:
    test_index.append(find_cp_idx(w))
testWords = [format_sentence(i) for i in testWords]
testChars = [[list(w) for w in s] for s in testWords]

counts = Counter()
for sw in trainWords+testWords:
    counts.update(sw)

vocab = sorted(counts, key=counts.get, reverse=True)
word2index = {w: i for i, w in enumerate(vocab, 1)}
index2word = {i: w for w, i in word2index.items()}

counts = Counter()
for sc in trainChars+testChars:
    counts.update(sum(sc, []))

c_vocab = sorted(counts, key=counts.get, reverse=True)
char2index = {w: i for i, w in enumerate(c_vocab, 1)}
index2char = {i: w for w, i in char2index.items()}

w_filePath = 'data/index/index_w.pkl'
with open(w_filePath, 'wb') as fp:
    pickle.dump((word2index, index2word), fp, -1)

c_filePath = 'data/index/index_c.pkl'
with open(c_filePath, 'wb') as fp:
    pickle.dump((char2index, index2char), fp, -1)
causalLabel = [l for l in testLabel if l != 'Non-Causal']
print('\n------ Causal Sentence Analysis ------\n')
print('{:<40}{:>30}\n'.format('', 'number of causality triplet'))
for l in ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12']:
    print('{:<40}{:>30d}'.format(
        l+':', Counter([Counter(i[12:])['e']/2 for i in causalLabel])[int(l)]))
print('{:<40}{:>30d}'.format('total', int(
    sum([Counter(i[12:])['e']/2 for i in causalLabel]))))<jupyter_output>------ Causal Sentence Analysis ------

                                           number of causality triplet

1:                                                                 126
2:                                                                  47
3:                                                                   7
4:                                                                   8
5:                                                                   1
6:                                                                   1
7:                                                                   0
8:                                                                   0
9:                                                                   0
10:                                                                  0
11:                                                                  0
12:                                                                  1
total                              [...]<jupyter_text>Create Word Embeddings with Wiki-Extvec<jupyter_code>from pynlp import StanfordCoreNLP
from nltk import WordNetLemmatizer, PorterStemmer, LancasterStemmer
import spacy
import math
import h5py
annotators = 'lemma'
core_nlp = StanfordCoreNLP(annotators=annotators)
nlp = spacy.load('en_3')
wnl = WordNetLemmatizer()
porter = PorterStemmer()
lancaster = LancasterStemmer()

VOCAB_SIZE = len(word2index)+1
EXTVEC_DIM = 300
def replace(w, d):
    """
    Replace the words that are not in the dictionary
    """
    r = d.get(w)
    if r is None:
        nw = w.lower()
        r = d.get(nw)
    if r is None:
        nw = [i[0].lemma for i in core_nlp(w)][0]
        r = d.get(nw)
    if r is None:
        nw = [i.lemma_ for i in nlp(w)][0]
        r = d.get(nw)
    if r is None:
        nw = wnl.lemmatize(w)
        r = d.get(nw)
    if r is None:
        nw = porter.stem(w)
        r = d.get(nw)
    if r is None:
        nw = lancaster.stem(w)
        r = d.get(nw)
    if r is None:
        nw = w[:-1]
        r = d.get(nw)
    if r is None:
        nw = w[:-2]
        r = d.get(nw)
    return r
extvec_n_symbols = 1476022
extvec_index_dict = {}
extvec_embedding_weights = np.empty((extvec_n_symbols, EXTVEC_DIM))
filePath = 'your path to /dependency-based_word_embeddings/wiki_extvec'
with open(filePath, encoding='utf-8') as fp:
    index = 0
    for l in fp:
        l = l.split(' ')
        word = l[0]
        extvec_index_dict[word] = index
        extvec_embedding_weights[index, :] = np.asarray(l[1:], dtype='float32')
        index += 1
# Generate random embedding with same scale as extvec
SEED = 666
np.random.seed(SEED)
shape = (VOCAB_SIZE, EXTVEC_DIM)
scale = math.sqrt(3.0 / EXTVEC_DIM)
extvec_embedding = np.random.uniform(low=-scale, high=scale, size=shape)
# Copy from extvec weights of words that appear in index2word
count = 0
for i in range(1, VOCAB_SIZE):
    w = index2word[i]
    g = extvec_index_dict.get(w)
    if g is None:
        g = replace(w, extvec_index_dict)
    if g is not None:
        extvec_embedding[i, :] = extvec_embedding_weights[g, :]
        count += 1
print('{num_tokens}-{per:.3f}% tokens in vocab found in Wiki-Extvec and copied to embedding.'.format(
    num_tokens=count, per=count/float(VOCAB_SIZE)*100))
filePath = 'data/embedding/extvec_embedding.npy'
np.save(open(filePath, 'wb'), extvec_embedding)<jupyter_output><empty_output><jupyter_text>Create Contextual String Embeddings with Flair<jupyter_code>from flair.embeddings import FlairEmbeddings, StackedEmbeddings
from tqdm import tqdm
from flair.data import Sentence
FLAIR_DIM = 4096


def flair_cse(sw):
    """
    Convert sentence to contextual string embeddings with flair
    """
    charlm_embedding_forward = FlairEmbeddings('news-forward')
    charlm_embedding_backward = FlairEmbeddings('your path to /news-backward-0.4.1.pt')
    stacked_embeddings = StackedEmbeddings(
        embeddings=[charlm_embedding_forward, charlm_embedding_backward])
    result = []
    nsw = [Sentence(' '.join(i)) for i in sw]
    for s in tqdm(nsw):
        stacked_embeddings.embed(s)
        result.append(np.concatenate((np.array([np.array(
            token.embedding) for token in s]), np.zeros((MAX_WLEN-len(s), FLAIR_DIM))), axis=0))
    return np.array(result)
trainFlair = flair_cse(trainWords)
testFlair = flair_cse(testWords)<jupyter_output>100%|██████████| 786/786 [07:30<00:00,  1.33it/s]<jupyter_text>Create BERT Embeddings with flair<jupyter_code>from flair.embeddings import BertEmbeddings
from tqdm import tqdm
from flair.data import Sentence
BERT_DIM = 4096


def flair_bert(sw):
    """
    Convert sentence to bert embeddings with flair
    """
    bert_embedding = BertEmbeddings('bert-large-cased')
    result = []
    nsw = [Sentence(' '.join(i)) for i in sw]
    for s in tqdm(nsw):
        bert_embedding.embed(s)
        result.append(np.concatenate((np.array([np.array(
            token.embedding) for token in s]), np.zeros((MAX_WLEN-len(s), BERT_DIM))), axis=0))
    return np.array(result)
trainBERT = flair_bert(trainWords)
testBERT = flair_bert(testWords)<jupyter_output>100%|██████████| 786/786 [04:18<00:00,  2.72it/s]<jupyter_text>Create ELMo Embeddings with flair<jupyter_code>from flair.embeddings import ELMoEmbeddings
from tqdm import tqdm
from flair.data import Sentence
ELMO_DIM = 3072


def flair_elmo(sw):
    """
    Convert sentence to ELMo embeddings with flair
    """
    elmo_embedding = ELMoEmbeddings('original')
    result = []
    nsw = [Sentence(' '.join(i)) for i in sw]
    for s in tqdm(nsw):
        elmo_embedding.embed(s)
        result.append(np.concatenate((np.array([np.array(
            token.embedding) for token in s]), np.zeros((MAX_WLEN-len(s), ELMO_DIM))), axis=0))
    return np.array(result)
trainELMo = flair_elmo(trainWords)
testELMo = flair_elmo(testWords)<jupyter_output>100%|██████████| 786/786 [04:34<00:00,  2.14it/s]<jupyter_text>Build Dataset  Tagging<jupyter_code>def find_cp(l):
    """
    Find index of cause and effect from label
    """
    p = '\((e.*?)\)'
    t = re.findall(p, l)
    t = [i.split(',') for i in t]
    c = [i[0] for i in t]
    e = [i[-1] for i in t]
    return c, e


def tagging(s, n, w):
    """
    Tagging
    """
    for i in range(len(w)):
        if i == 0:
            s[w[i]] = n
        else:
            s[w[i]] = n+1
    return s


def word2label(s, idx, l):
    """
    Convert sentence to label sequence
    :Tagging Scheme
    ---7 Tag---
    0: O = Other
    1: B-C = Cause Begin
    2: I-C = Cause Inside
    3: B-E = Effect Begin
    4: I-E = Effect Inside
    5: B-CE = Cause|Effect Begin
    6: I-CE = Cause|Effect Inside
    """
    r = [0]*len(s)
    if l == 'Non-Causal':
        return r
    else:
        c, e = find_cp(l)
        element = set(c+e)
        cause, cause_and_effect, effect = [], [], []
        for i in element:
            if i not in e:
                cause.append(i)
            if i in c and i in e:
                cause_and_effect.append(i)
            if i not in c and i in e:
                effect.append(i)
        for e in element:
            if e in cause:
                r = tagging(r, 1, idx[int(e[1:])-1])
            if e in effect:
                r = tagging(r, 3, idx[int(e[1:])-1])
            if e in cause_and_effect:
                r = tagging(r, 5, idx[int(e[1:])-1])
    return r


train_labelSeq = [word2label(
    trainWords[i], train_index[i], trainLabel[i]) for i in range(len(trainWords))]
test_labelSeq = [word2label(testWords[i], test_index[i], testLabel[i])
                 for i in range(len(testWords))]
labelSeq = sum(train_labelSeq+test_labelSeq, [])
tagAll = ['O', 'B-C', 'I-C', 'B-E', 'I-E', 'B-CE', 'I-CE']
print('\n------ Corpous Tag Analysis ------\n')
print('{:<20}{:>30}{:>30}\n'.format('', 'number of tag', 'percentage'))
for i in range(len(tagAll)):
    print('{:<20}{:>30d}{:>30}'.format(tagAll[i]+':', Counter(labelSeq)[i], str(Counter(
        labelSeq)[i]/sum(Counter(labelSeq)[i] for i in range(len(tagAll)))*100)[:5]+'%'))
print('{:<20}{:>30d}{:>30d}'.format('total', sum(
    Counter(labelSeq)[i] for i in range(len(tagAll))), 1))
labelSeq = sum(train_labelSeq, [])
tagAll = ['O', 'B-C', 'I-C', 'B-E', 'I-E', 'B-CE', 'I-CE']
print('\n------ Corpous Tag Analysis ------\n')
print('{:<20}{:>30}{:>30}\n'.format('', 'number of tag', 'percentage'))
for i in range(len(tagAll)):
    print('{:<20}{:>30d}{:>30}'.format(tagAll[i]+':', Counter(labelSeq)[i], str(Counter(
        labelSeq)[i]/sum(Counter(labelSeq)[i] for i in range(len(tagAll)))*100)[:5]+'%'))
print('{:<20}{:>30d}{:>30d}'.format('total', sum(
    Counter(labelSeq)[i] for i in range(len(tagAll))), 1))
labelSeq = sum(test_labelSeq, [])
tagAll = ['O', 'B-C', 'I-C', 'B-E', 'I-E', 'B-CE', 'I-CE']
print('\n------ Corpous Tag Analysis ------\n')
print('{:<20}{:>30}{:>30}\n'.format('', 'number of tag', 'percentage'))
for i in range(len(tagAll)):
    print('{:<20}{:>30d}{:>30}'.format(tagAll[i]+':', Counter(labelSeq)[i], str(Counter(
        labelSeq)[i]/sum(Counter(labelSeq)[i] for i in range(len(tagAll)))*100)[:5]+'%'))
print('{:<20}{:>30d}{:>30d}'.format('total', sum(
    Counter(labelSeq)[i] for i in range(len(tagAll))), 1))<jupyter_output>------ Corpous Tag Analysis ------

                                     number of tag                    percentage

O:                                           11826                        92.50%
B-C:                                           236                        1.846%
I-C:                                           229                        1.791%
B-E:                                           238                        1.861%
I-E:                                           230                        1.799%
B-CE:                                            9                        0.070%
I-CE:                                           16                        0.125%
total                                        12784                             1<jupyter_text>Split Dataset<jupyter_code>from keras.preprocessing.sequence import pad_sequences
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
import h5py
SEED = 666


def char_data(cw):
    result = []
    for s in cw:
        result.append(np.concatenate((pad_sequences([[char2index[c] for c in w] for w in s], maxlen=MAX_CLEN, padding='post', truncating='post'),
                                      np.zeros((MAX_WLEN-len(s), MAX_CLEN))), axis=0))
    return np.array(result)


trainCrray = char_data(trainChars)

testCrray = char_data(testChars)

trainSeq = [[word2index[cw] for cw in s] for s in trainWords]
trainWrray = pad_sequences(trainSeq, maxlen=MAX_WLEN,
                           padding='post', truncating='post')

testSeq = [[word2index[cw] for cw in s] for s in testWords]
testWrray = pad_sequences(testSeq, maxlen=MAX_WLEN,
                          padding='post', truncating='post')

train_labelSeq = pad_sequences(
    train_labelSeq, maxlen=MAX_WLEN, padding='post', truncating='post')
train_labelData = train_labelSeq.reshape(len(trainWords), MAX_WLEN, 1)

test_labelSeq = pad_sequences(
    test_labelSeq, maxlen=MAX_WLEN, padding='post', truncating='post')
test_labelData = test_labelSeq.reshape(len(testWords), MAX_WLEN, 1)

xTrain, _, yTrain, _ = train_test_split(
    trainWrray, train_labelData, test_size=0., random_state=SEED)
xTest, _, yTest, _ = train_test_split(
    testWrray, test_labelData, test_size=0., random_state=SEED)
xTrain_c = train_test_split(trainCrray, test_size=0., random_state=SEED)[0]
xTest_c = train_test_split(testCrray, test_size=0., random_state=SEED)[0]
xTrain_flair = train_test_split(trainFlair, test_size=0., random_state=SEED)[0]
xTest_flair = train_test_split(testFlair, test_size=0., random_state=SEED)[0]
xTrain_bert = train_test_split(trainBERT, test_size=0., random_state=SEED)[0]
xTest_bert = train_test_split(testBERT, test_size=0., random_state=SEED)[0]
xTrain_elmo = train_test_split(trainELMo, test_size=0., random_state=SEED)[0]
xTest_elmo = train_test_split(testELMo, test_size=0., random_state=SEED)[0]
xTrain.shape, yTrain.shape, xTest.shape, yTest.shape, xTrain_c.shape, xTest_c.shape, xTrain_flair.shape, xTest_flair.shape
h5f = h5py.File('data/train/train.h5', 'w')
h5f.create_dataset('xTrain', data=xTrain)
h5f.create_dataset('xTrain_c', data=xTrain_c)
h5f.create_dataset('yTrain', data=yTrain)
h5f.close()

h5f = h5py.File('data/test/test.h5', 'w')
h5f.create_dataset('xTest', data=xTest)
h5f.create_dataset('xTest_c', data=xTest_c)
h5f.create_dataset('yTest', data=yTest)
h5f.close()

h5f = h5py.File('data/embedding/flair.h5', 'w')
h5f.create_dataset('xTrain_flair', data=xTrain_flair)
h5f.create_dataset('xTest_flair', data=xTest_flair)
h5f.close()
h5f = h5py.File('data/embedding/bert.h5', 'w')
h5f.create_dataset('xTrain_bert', data=xTrain_bert)
h5f.create_dataset('xTest_bert', data=xTest_bert)
h5f.close()
h5f = h5py.File('data/embedding/elmo.h5', 'w')
h5f.create_dataset('xTrain_elmo', data=xTrain_elmo)
h5f.create_dataset('xTest_elmo', data=xTest_elmo)
h5f.close()<jupyter_output><empty_output><jupyter_text>Data compress<jupyter_code>import zlib

def compress(infile, dst, level=9):
    infile = open(infile, 'rb')
    dst = open(dst, 'wb')
    compress = zlib.compressobj(level)
    data = infile.read(1024)
    while data:
        dst.write(compress.compress(data))
        data = infile.read(1024)
    dst.write(compress.flush())


def decompress(infile, dst):
    infile = open(infile, 'rb')
    dst = open(dst, 'wb')
    decompress = zlib.decompressobj()
    data = infile.read(1024)
    while data:
        dst.write(decompress.decompress(data))
        data = infile.read(1024)
    dst.write(decompress.flush())
    
#compress('data/embedding/bert.h5',
#         'data/embedding/c_bert.h5')
decompress('data/embedding/c_bert.h5', 'data/embedding/bert.h5')
decompress('data/embedding/c_elmo.h5', 'data/embedding/elmo.h5')
decompress('data/embedding/c_flair.h5', 'data/embedding/flair.h5')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1540132450331126, 'fraction_numerical': 0.025748344370860928, 'mean_word_length': 2.4685777287761854, 'pattern_counts': {'":': 0, '<': 48, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 5}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 45, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '333709', 'license': 'Apache-2.0', 'n_tokens_mistral': 7124, 'n_tokens_neox': 6675, 'n_words': 1477, 'path': 'data_prep.ipynb', 'repo_name': 'Das-Boot/causality_extraction', 'size': '18875'}