<commit_before>#!/usr/bin/env python3
#-*- coding: utf-8 -*-

from lxml.etree import ElementTree
import subprocess
import io
import pickle
import os
from os.path import join
import csv
import sys
import locale
import re
from functools import cmp_to_key
import itertools
from operator import itemgetter
from collections import defaultdict

locale.setlocale(locale.LC_ALL, 'fr_FR.UTF-8')

FVN_PATH = 'loadmapping'
FORGET_LIST = ['?', '*', '']


def get_members(tree):
    return [member.get('name') for member in tree.findall(".//MEMBER")]

with open(join(FVN_PATH, 'data/LADL_to_verbes'), 'rb') as f:
    ladl_dict = pickle.load(f)
with open(join(FVN_PATH, 'data/LVF+1_to_verbs'), 'rb') as f:
    lvf_dict = pickle.load(f)
with open(join(FVN_PATH, 'data/DICOVALENCE_VERBS'), 'rb') as f:
    dicovalence_verbs = pickle.load(f)
with open(join(FVN_PATH, 'data/verb_dictionary.pickle'), 'rb') as f:
    verb_dict = pickle.load(f)


# We want to allow multiple classes and various writings that make sense for
# humans
def parse_ladl(raw):
    ladl = raw.replace("-", "")

    if " ou " in ladl:
        operation = 'ou'
        ladl = ladl.split(" ou ")
    elif " et " in ladl:
        operation = 'et'
        ladl = ladl.split(" et ")
    else:
        operation = None
        ladl = [ladl]

    for i, ladl_class in enumerate(ladl):
        if ladl_class.endswith("source") or ladl_class.endswith("dest"):
            ladl[i] = ladl_class.split()[0]

    return operation, ladl


def parse_lvf(raw):
    lvf = raw

    if " ou " in lvf:
        operation = "ou"
        lvf = lvf.split(" ou ")
    elif " et " in lvf:
        operation = "et"
        lvf = lvf.split(" et ")
    else:
        operation = None
        lvf = [lvf]

    return operation, lvf


def parse_path(specific_class):
    path = []
    mode = 'new'
    for c in specific_class:
        if c == '[':
            path.append('')
            mode = 'same'
        elif c == ']':
            mode = 'new'
        elif c == '.':
            continue
        else:
            if mode == 'same':
                path[-1] += c
            elif mode == 'new':
                path.append(c)
            else:
                raise Exception('Unknown mode {}'.format(mode))

    return path


def get_one_verb_class(specific_class, resource):
    if resource == 'LADL':
        return get_one_verb_class_ladl(specific_class)
    elif resource == 'LVF':
        path = parse_path(specific_class)
        return get_one_verb_class_lvf(lvf_dict, path)


def get_one_verb_class_ladl(ladl_class):
    """Generalized version of specific_dict[specific_class] which allows
    regexes"""
    selected_verbs = set()
    for one_class in ladl_dict:
        # if regex is invalid, exit
        stripped = ladl_class.strip()
        if re.match(ladl_class, one_class):
            selected_verbs |= set(ladl_dict[one_class])

    return list(selected_verbs)


def everything_from_dict(d):
    if type(d) == list:
        return d
    elif type(d) == dict:
        l = []
        for k in d:
            l.extend(everything_from_dict(d[k]))
        return l
    else:
        raise Exception("Unknown type {}!".format(type(d)))


def get_one_verb_class_lvf(lvf_dict, lvf_path):
    """Recursively return all verbs for @lvf_class (eg. L3b, N1c.[12])"""
    if not lvf_path:
        if type(lvf_dict) == list:
            return set(lvf_dict)
        elif type(lvf_dict) == dict:
            return everything_from_dict(lvf_dict)
        else:
            raise Exception("Unknown type {}!".format(type(d)))
    else:
        prefix = lvf_path[0]
        suffix = lvf_path[1:]
        if len(prefix) > 1:
            keys = [c for c in prefix]
        else:
            keys = [prefix]

        verb_list = []
        for k in keys:
            new_stuff = get_one_verb_class_lvf(lvf_dict[k], suffix)
            verb_list.extend(new_stuff)

        return verb_list


def get_verbs_for_class_list(operation_and_list, resource):
    """Given a pre-processed list (by parse_*), return corresponding verbs"""
    operation, class_list = operation_and_list
    verbs = set(get_one_verb_class(class_list[0], resource))

    if operation is None:
        assert(len(class_list) == 1)
    else:
        assert(len(class_list) > 1)
        for specific_class in class_list[1:]:
            new_verbs = set(get_one_verb_class(specific_class, resource))
            if operation == 'ou':
                verbs |= new_verbs
            elif operation == 'et':
                verbs &= new_verbs

    return verbs


def translations_for_class(verbs, ladl, lvf):
    ladl_classes = parse_ladl(ladl)
    lvf_classes = parse_ladl(lvf)

    candidates = defaultdict(set)
    lvf, ladl = set(), set()

    for v in verbs:
        for c in verb_dict[v]:
            candidates[c].add(v)

    if not ladl_classes[1][0] in FORGET_LIST:
        ladl = get_verbs_for_class_list(ladl_classes, 'LADL')
        if not ladl:
            print("Warning, unknown class {}".format(ladl_classes))

    if not lvf_classes[1][0] in FORGET_LIST:
        lvf = get_verbs_for_class_list(lvf_classes, 'LVF')
        if not lvf:
            print("Warning, unknown class {}".format(lvs_classes))

    final = []
    for c in candidates:
        color = 'none'
        if c in ladl and c in lvf:
            color, id_color = 'both', 0
        elif c in lvf:
            color, id_color = 'lvf', 2
        elif c in ladl:
            color, id_color = 'ladl', 1
        elif c in dicovalence_verbs:
            color, id_color = 'dicovalence', 3<commit_msg>Remove parse_lvf: parse_ladl was enough<commit_after>#!/usr/bin/env python3
#-*- coding: utf-8 -*-

from lxml.etree import ElementTree
import subprocess
import io
import pickle
import os
from os.path import join
import csv
import sys
import locale
import re
from functools import cmp_to_key
import itertools
from operator import itemgetter
from collections import defaultdict

locale.setlocale(locale.LC_ALL, 'fr_FR.UTF-8')

FVN_PATH = 'loadmapping'
FORGET_LIST = ['?', '*', '']


def get_members(tree):
    return [member.get('name') for member in tree.findall(".//MEMBER")]

with open(join(FVN_PATH, 'data/LADL_to_verbes'), 'rb') as f:
    ladl_dict = pickle.load(f)
with open(join(FVN_PATH, 'data/LVF+1_to_verbs'), 'rb') as f:
    lvf_dict = pickle.load(f)
with open(join(FVN_PATH, 'data/DICOVALENCE_VERBS'), 'rb') as f:
    dicovalence_verbs = pickle.load(f)
with open(join(FVN_PATH, 'data/verb_dictionary.pickle'), 'rb') as f:
    verb_dict = pickle.load(f)


# We want to allow multiple classes and various writings that make sense for
# humans
def parse_complex_lvfladl(raw):
    ladl = raw.replace("-", "")

    if " ou " in ladl:
        operation = 'ou'
        ladl = ladl.split(" ou ")
    elif " et " in ladl:
        operation = 'et'
        ladl = ladl.split(" et ")
    else:
        operation = None
        ladl = [ladl]

    for i, ladl_class in enumerate(ladl):
        if ladl_class.endswith("source") or ladl_class.endswith("dest"):
            ladl[i] = ladl_class.split()[0]

    return operation, ladl


def parse_path(specific_class):
    path = []
    mode = 'new'
    for c in specific_class:
        if c == '[':
            path.append('')
            mode = 'same'
        elif c == ']':
            mode = 'new'
        elif c == '.':
            continue
        else:
            if mode == 'same':
                path[-1] += c
            elif mode == 'new':
                path.append(c)
            else:
                raise Exception('Unknown mode {}'.format(mode))

    return path


def get_one_verb_class(specific_class, resource):
    if resource == 'LADL':
        return get_one_verb_class_ladl(specific_class)
    elif resource == 'LVF':
        path = parse_path(specific_class)
        return get_one_verb_class_lvf(lvf_dict, path)


def get_one_verb_class_ladl(ladl_class):
    """Generalized version of specific_dict[specific_class] which allows
    regexes"""
    selected_verbs = set()
    for one_class in ladl_dict:
        # if regex is invalid, exit
        stripped = ladl_class.strip()
        if re.match(ladl_class, one_class):
            selected_verbs |= set(ladl_dict[one_class])

    return list(selected_verbs)


def everything_from_dict(d):
    if type(d) == list:
        return d
    elif type(d) == dict:
        l = []
        for k in d:
            l.extend(everything_from_dict(d[k]))
        return l
    else:
        raise Exception("Unknown type {}!".format(type(d)))


def get_one_verb_class_lvf(lvf_dict, lvf_path):
    """Recursively return all verbs for @lvf_class (eg. L3b, N1c.[12])"""
    if not lvf_path:
        if type(lvf_dict) == list:
            return set(lvf_dict)
        elif type(lvf_dict) == dict:
            return everything_from_dict(lvf_dict)
        else:
            raise Exception("Unknown type {}!".format(type(d)))
    else:
        prefix = lvf_path[0]
        suffix = lvf_path[1:]
        if len(prefix) > 1:
            keys = [c for c in prefix]
        else:
            keys = [prefix]

        verb_list = []
        for k in keys:
            new_stuff = get_one_verb_class_lvf(lvf_dict[k], suffix)
            verb_list.extend(new_stuff)

        return verb_list


def get_verbs_for_class_list(operation_and_list, resource):
    """Given a pre-processed list (by parse_*), return corresponding verbs"""
    operation, class_list = operation_and_list
    verbs = set(get_one_verb_class(class_list[0], resource))

    if operation is None:
        assert(len(class_list) == 1)
    else:
        assert(len(class_list) > 1)
        for specific_class in class_list[1:]:
            new_verbs = set(get_one_verb_class(specific_class, resource))
            if operation == 'ou':
                verbs |= new_verbs
            elif operation == 'et':
                verbs &= new_verbs

    return verbs


def translations_for_class(verbs, ladl, lvf):
    ladl_classes = parse_complex_lvfladl(ladl)
    lvf_classes = parse_complex_lvfladl(lvf)

    candidates = defaultdict(set)
    lvf, ladl = set(), set()

    for v in verbs:
        for c in verb_dict[v]:
            candidates[c].add(v)

    if not ladl_classes[1][0] in FORGET_LIST:
        ladl = get_verbs_for_class_list(ladl_classes, 'LADL')
        if not ladl:
            print("Warning, unknown class {}".format(ladl_classes))

    if not lvf_classes[1][0] in FORGET_LIST:
        lvf = get_verbs_for_class_list(lvf_classes, 'LVF')
        if not lvf:
            print("Warning, unknown class {}".format(lvs_classes))

    final = []
    for c in candidates:
        color = 'none'
        if c in ladl and c in lvf:
            color, id_color = 'both', 0
        elif c in lvf:
            color, id_color = 'lvf', 2
        elif c in ladl:
            color, id_color = 'ladl', 1
        elif c in dicovalence_verbs:
            color, id_color = 'dicovalence', 3
{'fraction_non_alphanumeric': 0.11348232276973805, 'fraction_numerical': 0.0045795933321121085, 'mean_word_length': 2.376314162028448, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4221450', 'n_tokens_mistral': 3742, 'n_tokens_neox': 3632, 'n_words': 929}