<commit_before>#!/usr/bin/env python

import os
import sys
import tempfile
import shutil


from KafNafParserPy import *
from subprocess import Popen,PIPE
from lxml import etree
from convert_penn_to_kaf import convert_penn_to_knaf_with_numtokens
from alpino_dependency import Calpino_dependency

last_modified='05sept2014'
version="0.1"
this_name = 'Morphosyntactic parser based on Alpino'

###############################################

def set_up_alpino():
    ##Uncomment next line and point it to your local path to Alpino if you dont want to set the environment variable ALPINO_HOME
    #os.environ['ALPINO_HOME'] = '/home/izquierdo/tools/Alpino'
    
    if 'ALPINO_HOME' not in os.environ:
        print>>sys.stderr,'ALPINO_HOME variable not set. Set this variable to point to your local path to Alpino. For instance:'
        print>>sys.stderr,'export ALPINO_HOME=/home/your_user/your_tools/Alpino'
        sys.exit(-1)
    os.environ['SP_CSETLEN']='212'
    os.environ['SP_CTYPE']='utf8'

def load_sentences(in_obj):
    previous_sent = None
    current_sent = []
    sentences = []
    for token_obj in in_obj.get_tokens():
        token = token_obj.get_text()
        sent = token_obj.get_sent()
        token_id = token_obj.get_id()
        if sent != previous_sent and previous_sent!=None:
            sentences.append(current_sent)
            current_sent = [(token,token_id)]
        else:
            current_sent.append((token,token_id))
        previous_sent = sent
    
    if len(current_sent) !=0:
        sentences.append(current_sent)
    return sentences

def node_to_penn(node,map_token_begin_node):
    children = node.getchildren()
    if len(children) == 0:
        word = node.get('word',None)
        if word is not None:
            #The attribute begin gives you the number of the token
            word = word.replace('(','-LRB')
            word = word.replace(')','-RRB-')
    
            num_token = node.get('begin')
            map_token_begin_node[num_token] = node
            word = num_token+'#'+word
            if node.get('rel') == 'hd': 
                head = '=H'
            else: 
                head = ''
            return '('+node.get('pos')+head+' '+word.encode('utf-8')+')'
        else:
            return ''
    else:
        str = '('+node.get('cat')+' '
        for n in children:
            str+=node_to_penn(n,map_token_begin_node)
        str+=')'
        return str

def xml_to_penn(filename):
    '''
    Converts the xml from Alpino into penntreebank format
    '''
    ## Under certain condition, there is know bug of Alpino, it sets the encoding in the XML
    ## to iso-8859-1, but the real encoding is UTF-8. So we need to force to use this encoding
  
    parser = etree.XMLParser(encoding='UTF-8')
    tree = etree.parse(filename,parser)
    ##This is a mapping for the token begin (0,1,2,...) to the <node element
    map_token_begin_node = {}
    
    str = node_to_penn(tree.find('node'),map_token_begin_node)
    return str,map_token_begin_node
  
  
def process_alpino_xml(xml_file,sentence,count_terms,knaf_obj,cnt_t,cnt_nt,cnt_edge):
    #sentence is a list of pairs [
    print>>sys.stderr,'Processing file',xml_file
    
    penn_tree_str, map_token_begin_node = xml_to_penn(xml_file)
    
    ##########################################
    #Create the term layer    
    ##########################################

    term_ids = []
    lemma_for_termid = {}
    print>>sys.stderr,'  Creating the term layer...'
    for num_token, (token,token_id) in enumerate(sentence):
        new_term_id = 't_'+str(count_terms)
        count_terms+=1
        term_ids.append(new_term_id)
        alpino_node = map_token_begin_node[str(num_token)]
        term_obj = Cterm(type=knaf_obj.get_type())
        term_obj.set_id(new_term_id)
        new_span = Cspan()
        new_span.create_from_ids([token_id])
        term_obj.set_span(new_span)
        term_obj.set_lemma(alpino_node.get('lemma','unknown'))
        lemma_for_termid[new_term_id] = alpino_node.get('lemma','unknown')
        term_obj.set_pos(alpino_node.get('pos','unknown')+'#'+alpino_node.get('postag','unknown'))
        term_obj.set_morphofeat(alpino_node.get('frame','unknown'))
        knaf_obj.add_term(term_obj)
    ##########################################
        
    ##########################################
    ##Constituency layer
    print>>sys.stderr,'  Creating the constituency layer...'
    tree_obj,cnt_t,cnt_nt,cnt_edge = convert_penn_to_knaf_with_numtokens(penn_tree_str,term_ids,lemma_for_termid,cnt_t,cnt_nt,cnt_edge)
    knaf_obj.add_constituency_tree(tree_obj)
    ##########################################
    

    ##########################################
    # Dependency part
    ##########################################
    
    # Call to Alpino with the --treebank option to get the dependencies out of the XML
    print>>sys.stderr,'  Creating the dependency layer...'
    alpino_bin = os.path.join(os.environ['ALPINO_HOME'],'bin','Alpino')
    cmd = alpino_bin+' -treebank_triples '+xml_file
    alpino_pro = Popen(cmd,stdout=PIPE,stdin=PIPE,stderr=PIPE,shell=True)
    alpino_pro.wait()
    
    for line in alpino_pro.stdout:
        line = line.strip().decode('utf-8')
        my_dep = Calpino_dependency(line)
        if my_dep.is_ok():
            deps = my_dep.generate_dependencies(term_ids)
            for d in deps:
               knaf_obj.add_dependency(d)
    ##########################################
<commit_msg>changes to morph_syn: (1) add whether term is open or closed. (2) simplify pos and move more advanced info to morphfeat<commit_after>#!/usr/bin/env python

import os
import sys
import tempfile
import shutil


from KafNafParserPy import *
from subprocess import Popen,PIPE
from lxml import etree
from convert_penn_to_kaf import convert_penn_to_knaf_with_numtokens
from alpino_dependency import Calpino_dependency

last_modified='05sept2014'
version="0.1"
this_name = 'Morphosyntactic parser based on Alpino'

###############################################

def set_up_alpino():
    ##Uncomment next line and point it to your local path to Alpino if you dont want to set the environment variable ALPINO_HOME
    #os.environ['ALPINO_HOME'] = '/home/izquierdo/tools/Alpino'
    
    if 'ALPINO_HOME' not in os.environ:
        print>>sys.stderr,'ALPINO_HOME variable not set. Set this variable to point to your local path to Alpino. For instance:'
        print>>sys.stderr,'export ALPINO_HOME=/home/your_user/your_tools/Alpino'
        sys.exit(-1)
    os.environ['SP_CSETLEN']='212'
    os.environ['SP_CTYPE']='utf8'

def load_sentences(in_obj):
    previous_sent = None
    current_sent = []
    sentences = []
    for token_obj in in_obj.get_tokens():
        token = token_obj.get_text()
        sent = token_obj.get_sent()
        token_id = token_obj.get_id()
        if sent != previous_sent and previous_sent!=None:
            sentences.append(current_sent)
            current_sent = [(token,token_id)]
        else:
            current_sent.append((token,token_id))
        previous_sent = sent
    
    if len(current_sent) !=0:
        sentences.append(current_sent)
    return sentences

def get_term_type(pos):
    if pos in ['det','pron','prep','vg','conj' ]:
        return 'close'
    else:
        return 'open'

def node_to_penn(node,map_token_begin_node):
    children = node.getchildren()
    if len(children) == 0:
        word = node.get('word',None)
        if word is not None:
            #The attribute begin gives you the number of the token
            word = word.replace('(','-LRB')
            word = word.replace(')','-RRB-')
    
            num_token = node.get('begin')
            map_token_begin_node[num_token] = node
            word = num_token+'#'+word
            if node.get('rel') == 'hd': 
                head = '=H'
            else: 
                head = ''
            return '('+node.get('pos')+head+' '+word.encode('utf-8')+')'
        else:
            return ''
    else:
        str = '('+node.get('cat')+' '
        for n in children:
            str+=node_to_penn(n,map_token_begin_node)
        str+=')'
        return str

def xml_to_penn(filename):
    '''
    Converts the xml from Alpino into penntreebank format
    '''
    ## Under certain condition, there is know bug of Alpino, it sets the encoding in the XML
    ## to iso-8859-1, but the real encoding is UTF-8. So we need to force to use this encoding
  
    parser = etree.XMLParser(encoding='UTF-8')
    tree = etree.parse(filename,parser)
    ##This is a mapping for the token begin (0,1,2,...) to the <node element
    map_token_begin_node = {}
    
    str = node_to_penn(tree.find('node'),map_token_begin_node)
    return str,map_token_begin_node
  
  
def process_alpino_xml(xml_file,sentence,count_terms,knaf_obj,cnt_t,cnt_nt,cnt_edge):
    #sentence is a list of pairs [
    print>>sys.stderr,'Processing file',xml_file
    
    penn_tree_str, map_token_begin_node = xml_to_penn(xml_file)
    
    ##########################################
    #Create the term layer    
    ##########################################

    term_ids = []
    lemma_for_termid = {}
    print>>sys.stderr,'  Creating the term layer...'
    for num_token, (token,token_id) in enumerate(sentence):
        new_term_id = 't_'+str(count_terms)
        count_terms+=1
        term_ids.append(new_term_id)
        alpino_node = map_token_begin_node[str(num_token)]
        term_obj = Cterm(type=knaf_obj.get_type())
        term_obj.set_id(new_term_id)
        new_span = Cspan()
        new_span.create_from_ids([token_id])
        term_obj.set_span(new_span)
        term_obj.set_lemma(alpino_node.get('lemma','unknown'))
        lemma_for_termid[new_term_id] = alpino_node.get('lemma','unknown')

        alppos = alpino_node.get('pos','unknown')
        term_obj.set_pos(alppos)
        term_obj.set_morphofeat(alpino_node.get('postag','unknown'))
        termtype = get_term_type(alppos)
        term_obj.set_type(termtype)
        knaf_obj.add_term(term_obj)

        knaf_obj.add_term(term_obj)
    ##########################################
        
    ##########################################
    ##Constituency layer
    print>>sys.stderr,'  Creating the constituency layer...'
    tree_obj,cnt_t,cnt_nt,cnt_edge = convert_penn_to_knaf_with_numtokens(penn_tree_str,term_ids,lemma_for_termid,cnt_t,cnt_nt,cnt_edge)
    knaf_obj.add_constituency_tree(tree_obj)
    ##########################################
    

    ##########################################
    # Dependency part
    ##########################################
    
    # Call to Alpino with the --treebank option to get the dependencies out of the XML
    print>>sys.stderr,'  Creating the dependency layer...'
    alpino_bin = os.path.join(os.environ['ALPINO_HOME'],'bin','Alpino')
    cmd = alpino_bin+' -treebank_triples '+xml_file
    alpino_pro = Popen(cmd,stdout=PIPE,stdin=PIPE,stderr=PIPE,shell=True)
    alpino_pro.wait()
    
    for line in alpino_pro.stdout:
        line = line.strip().decode('utf-8')
        my_dep = Calpino_dependency(line)
        if my_dep.is_ok():
            deps = my_dep.generate_dependencies(term_ids)
            for d in deps:
               knaf_obj.add_dependency(d)
    ##########################################

{'fraction_non_alphanumeric': 0.17004405286343613, 'fraction_numerical': 0.005110132158590308, 'mean_word_length': 3.139679066374909, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6780543', 'n_tokens_mistral': 3762, 'n_tokens_neox': 3666, 'n_words': 741}