<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-

"""
Extract relation (Hyponyms, Hypernyms, Synonyms) from nltk only for English omw
"""

__author__ = '<NAME>, <NAME>'
__date__ = '29-03-2014'


from directoryparser import parseDirectory
from directoryparser import getDico
from simplefileparser import getValue
from nltk.corpus import wordnet

out = None


def createWordList(data):
    wordlist = []

    if type(data) is dict:
        for key in data.keys():
            values = data[key]
            for value in values:
                wordlist.append((key, str(value)))

    return wordlist


def getSynsets(word):
    return wordnet.synsets(word)


def getHyponyms(syn):
    return syn.hyponyms()


def getHypernyms(syn):
    return syn.hypernyms()


def getId(syn):
    i = ""
    padding = ""
    offset = str(syn.offset())

    for i in range(8 - len(offset)):
        padding = padding + "0"

    offset = padding + offset
    offset = offset + "-" + syn.pos()

    return offset

def getName(syns):
    syno = []
    for s in syns:
        for l in s.lemma_names():
            name = s.name().split('.')[0]
            syno.append((getId(s), l))
    return syno



def relations(w1, w2, rtype):
    global out
    out.write(str(w1[0]) + "\t" + rtype + "\t" + str(w2[0]) + "\n")

def getWord(t):
    return getValue(t)


def extractRelation(directory):
    """Extract the relation for English Omw
    """
    global out

    parseDirectory(directory)
    data = getDico()
    wordlist = createWordList(data)

    out = open('rels.csv', 'a')
    out.write('w1\ttype\tw2\n')

    for t in wordlist:
        word = getWord(t)
        syns = getSynsets(word)
        syno = getName(syns)
        for s in syno:
            relations(t, s, 'SYNO')

        for syn in syns:
            hypos = getHyponyms(syn)
            hypolemma = getName(hypos)
            for h in hypolemma:
                relations(t, h, 'HYPO')


        for syn in syns:
            hyper = getHypernyms(syn)
            hyperlemma = getName(hyper)
            for h in hyperlemma:
                relations(t, h, 'HYPER')



def main():<commit_msg>Modify header of csv file<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-

"""
Extract relation (Hyponyms, Hypernyms, Synonyms) from nltk only for English omw
"""

__author__ = '<NAME>, <NAME>'
__date__ = '29-03-2014'


from directoryparser import parseDirectory
from directoryparser import getDico
from simplefileparser import getValue
from nltk.corpus import wordnet

out = None


def createWordList(data):
    wordlist = []

    if type(data) is dict:
        for key in data.keys():
            values = data[key]
            for value in values:
                wordlist.append((key, str(value)))

    return wordlist


def getSynsets(word):
    return wordnet.synsets(word)


def getHyponyms(syn):
    return syn.hyponyms()


def getHypernyms(syn):
    return syn.hypernyms()


def getId(syn):
    i = ""
    padding = ""
    offset = str(syn.offset())

    for i in range(8 - len(offset)):
        padding = padding + "0"

    offset = padding + offset
    offset = offset + "-" + syn.pos()

    return offset

def getName(syns):
    syno = []
    for s in syns:
        for l in s.lemma_names():
            name = s.name().split('.')[0]
            syno.append((getId(s), l))
    return syno



def relations(w1, w2, rtype):
    global out
    out.write(str(w1[0]) + "\t" + rtype + "\t" + str(w2[0]) + "\n")

def getWord(t):
    return getValue(t)


def extractRelation(directory):
    """Extract the relation for English Omw
    """
    global out

    parseDirectory(directory)
    data = getDico()
    wordlist = createWordList(data)

    out = open('rels.csv', 'a')
    out.write('s1\ttype\ts2\n')

    for t in wordlist:
        word = getWord(t)
        syns = getSynsets(word)
        syno = getName(syns)
        for s in syno:
            relations(t, s, 'SYNO')

        for syn in syns:
            hypos = getHyponyms(syn)
            hypolemma = getName(hypos)
            for h in hypolemma:
                relations(t, h, 'HYPO')


        for syn in syns:
            hyper = getHypernyms(syn)
            hyperlemma = getName(hyper)
            for h in hyperlemma:
                relations(t, h, 'HYPER')



def main():
{'fraction_non_alphanumeric': 0.125, 'fraction_numerical': 0.00929368029739777, 'mean_word_length': 2.200743494423792, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1664188', 'n_tokens_mistral': 1495, 'n_tokens_neox': 1417, 'n_words': 391}