<commit_before># -*- coding: utf-8 -*-
from itertools import chain, groupby
from collections import defaultdict
import pickle


def print_verbose(verbose, text):
    if verbose:
        print text


def RESTrequest(*args, **kwargs):
    """ritorna e salva il blob di dati che viene restituito
    da KEGG senza badare alla formattazione"""
    verbose = kwargs.get('verbose', False)
    force_download = kwargs.get('force', False)
    save = kwargs.get('force', True)

    # so you can copy paste from kegg
    args = list(chain.from_iterable(a.split('/') for a in args))
    args = [a for a in args if a]
    request = 'http://rest.kegg.jp/' + "/".join(args)
    print_verbose(verbose, "richiedo la pagina: " + request)
    filename = "./KEGG_" + "_".join(args)
    try:
        if force_download:
            raise IOError()
        print_verbose(verbose, "loading the cached file " + filename)
        with open(filename, 'r') as f:
            data = pickle.load(f)
    except IOError:
        print_verbose(verbose, "downloading the library,it may take some time")
        import urllib2
        try:
            req = urllib2.urlopen(request)
            data = req.read()
            if save:
                with open(filename, 'w') as f:
                    print_verbose(verbose, "saving the file to " + filename)
                    pickle.dump(data, f)
        # clean the error stacktrace
        except urllib2.HTTPError as e:
            raise e
    return data


def KEGGlink(db1, db2, **kwargs):
    data = RESTrequest('link', db1, db2, **kwargs)
    data = [tuple(d.split('\t')) for d in data.split('\n')][:-1]
    rel_dir = defaultdict(list)
    rel_inv = defaultdict(list)
    for element_1, element_2 in data:
        rel_inv[element_1].append(element_2)
        rel_dir[element_2].append(element_1)
    return rel_dir, rel_inv


def KEGGconv(db1, db2, **kwargs):
    data = RESTrequest('conv', db1, db2, **kwargs)
    data = [tuple(d.split('\t')) for d in data.split('\n')][:-1]
    rel_dir = defaultdict(list)
    rel_inv = defaultdict(list)
    for element_1, element_2 in data:
        rel_inv[element_1].append(element_2)
        rel_dir[element_2].append(element_1)
    return rel_dir, rel_inv


def KEGGlist(db, organism='', **kwargs):
    data = RESTrequest('list', db, organism, **kwargs)
    data = [tuple(d.split('\t')) for d in data.split('\n')][:-1]
    return dict(data)


def KEGGget(element, option='', **kwargs):
    # options = aaseq | ntseq | mol | kcf | image | kgml
    data = RESTrequest('get', element, option, **kwargs)
    data = data.split('\n')
    grouped = list(l.split(' ', 1) for l in data)
    grouped = [l for l in grouped if len(l) > 1]
    result = defaultdict(list)
    last_key = None
    for key, value in grouped:
        if key.strip():
            last_key = key.strip()
        result[last_key].append(value.strip())
    return result


def KEGGfind(database, description, option='', **kwargs):
    # options = aaseq | ntseq | mol | kcf | image | kgml
    data = RESTrequest('get', element, option, **kwargs)
    data = data.split('\n')
    grouped = list(l.split(' ', 1) for l in data)
    grouped = [l for l in grouped if len(l) > 1]
    result = defaultdict(list)
    last_key = None
    for key, value in grouped:
        if key.strip():
            last_key = key.strip()
        result[last_key].append(value.strip())<commit_msg>removed the keggfind function as not correct at all<commit_after># -*- coding: utf-8 -*-
from itertools import chain, groupby
from collections import defaultdict
import pickle


def print_verbose(verbose, text):
    if verbose:
        print text


def RESTrequest(*args, **kwargs):
    """ritorna e salva il blob di dati che viene restituito
    da KEGG senza badare alla formattazione"""
    verbose = kwargs.get('verbose', False)
    force_download = kwargs.get('force', False)
    save = kwargs.get('force', True)

    # so you can copy paste from kegg
    args = list(chain.from_iterable(a.split('/') for a in args))
    args = [a for a in args if a]
    request = 'http://rest.kegg.jp/' + "/".join(args)
    print_verbose(verbose, "richiedo la pagina: " + request)
    filename = "./KEGG_" + "_".join(args)
    try:
        if force_download:
            raise IOError()
        print_verbose(verbose, "loading the cached file " + filename)
        with open(filename, 'r') as f:
            data = pickle.load(f)
    except IOError:
        print_verbose(verbose, "downloading the library,it may take some time")
        import urllib2
        try:
            req = urllib2.urlopen(request)
            data = req.read()
            if save:
                with open(filename, 'w') as f:
                    print_verbose(verbose, "saving the file to " + filename)
                    pickle.dump(data, f)
        # clean the error stacktrace
        except urllib2.HTTPError as e:
            raise e
    return data


def KEGGlink(db1, db2, **kwargs):
    data = RESTrequest('link', db1, db2, **kwargs)
    data = [tuple(d.split('\t')) for d in data.split('\n')][:-1]
    rel_dir = defaultdict(list)
    rel_inv = defaultdict(list)
    for element_1, element_2 in data:
        rel_inv[element_1].append(element_2)
        rel_dir[element_2].append(element_1)
    return rel_dir, rel_inv


def KEGGconv(db1, db2, **kwargs):
    data = RESTrequest('conv', db1, db2, **kwargs)
    data = [tuple(d.split('\t')) for d in data.split('\n')][:-1]
    rel_dir = defaultdict(list)
    rel_inv = defaultdict(list)
    for element_1, element_2 in data:
        rel_inv[element_1].append(element_2)
        rel_dir[element_2].append(element_1)
    return rel_dir, rel_inv


def KEGGlist(db, organism='', **kwargs):
    data = RESTrequest('list', db, organism, **kwargs)
    data = [tuple(d.split('\t')) for d in data.split('\n')][:-1]
    return dict(data)


def KEGGget(element, option='', **kwargs):
    # options = aaseq | ntseq | mol | kcf | image | kgml
    data = RESTrequest('get', element, option, **kwargs)
    data = data.split('\n')
    grouped = list(l.split(' ', 1) for l in data)
    grouped = [l for l in grouped if len(l) > 1]
    result = defaultdict(list)
    last_key = None
    for key, value in grouped:
        if key.strip():
            last_key = key.strip()
        result[last_key].append(value.strip())
{'fraction_non_alphanumeric': 0.13339655444918602, 'fraction_numerical': 0.00948316737790422, 'mean_word_length': 2.659919028340081, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3247080', 'n_tokens_mistral': 2093, 'n_tokens_neox': 2054, 'n_words': 590}