<commit_before># -*- coding: utf-8 -*-
import os
import csv
import datetime
import pandas as pd
from itertools import chain
from functools import partial

from anacode import codes


def backup(root, files):
    """Backs up `files` from `root` directory and return list of backed up
    file names. Backed up files will have datetime suffix appended to original
    file name.

    :param root: Absolute path to folder where files to backup are located
    :type root: str
    :param files: Names of files that needs backing up
    :type files: str
    :return: list -- List of backed up file names
    """
    backed_up = []
    join = os.path.join
    root_contents = os.listdir(root)
    dt_str = datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S')
    for file_name in files:
        if file_name not in root_contents:
            continue
        new_name = file_name + '_' + dt_str
        os.rename(join(root, file_name), join(root, new_name))
        backed_up.append(new_name)
    return backed_up


HEADERS = {
    'categories': [u'doc_id', u'text_order', u'category', u'probability'],
    'concepts': [u'doc_id', u'text_order', u'concept', u'freq',
                 u'relevance_score', u'concept_type'],
    'concepts_surface_strings': [u'doc_id', u'text_order', u'concept',
                                 u'surface_string', u'text_span'],
    'sentiments': [u'doc_id', u'text_order', u'sentiment_value'],
    'absa_entities': [u'doc_id', u'text_order', u'entity_name', u'entity_type',
                      u'surface_string', u'text_span'],
    'absa_normalized_texts': [u'doc_id', u'text_order', u'normalized_text'],
    'absa_relations': [u'doc_id', u'text_order', u'relation_id',
                       u'opinion_holder', u'restriction', u'sentiment_value',
                       u'is_external', u'surface_string', u'text_span'],
    'absa_relations_entities': [u'doc_id', u'text_order', u'relation_id',
                                u'entity_type', u'entity_name'],
    'absa_evaluations': [u'doc_id', u'text_order', u'evaluation_id',
                         u'sentiment_value', u'surface_string', u'text_span'],
    'absa_evaluations_entities': [u'doc_id', u'text_order', u'evaluation_id',
                                  u'entity_type', u'entity_name'],
}


# `anacode.agg.aggregations.ApiDataset.from_path` depends
# on ordering of files defined in values here
CSV_FILES = {
    'categories': ['categories.csv'],
    'concepts': ['concepts.csv', 'concepts_surface_strings.csv'],
    'sentiments': ['sentiments.csv'],
    'absa': [
        'absa_entities.csv', 'absa_normalized_texts.csv',
        'absa_relations.csv', 'absa_relations_entities.csv',
        'absa_evaluations.csv', 'absa_evaluations_entities.csv'
    ]
}


def categories_to_list(doc_id, analyzed, single_document=False):
    """Converts categories response to flat list with doc_id included.

    :param doc_id: Will be inserted to each row as first element
    :param analyzed: Response json from anacode api for categories call
    :type analyzed: list
    :param single_document: Is analysis describing just one document
    :type single_document: bool
    :return: dict -- Dictionary with one key 'categories' pointing to flat list
     of categories
    """
    cat_list = []
    for order, text_analyzed in enumerate(analyzed):
        for result_dict in text_analyzed:
            row = [doc_id, 0, result_dict.get('label'),
                   result_dict.get('probability')]
            if single_document:
                row[1] += order
            else:
                row[0] += order
            cat_list.append(row)
    return {'categories': cat_list}


def concepts_to_list(doc_id, analyzed, single_document=False):
    """Converts concepts response to flat lists with doc_id included

    :param doc_id: Will be inserted to each row as first element
    :param analyzed: Response json from anacode api for concepts call
    :type analyzed: list
    :param single_document: Is analysis describing just one document
    :type single_document: bool
    :return: dict -- Dictionary with two keys: 'concepts' pointing to flat list
     of found concepts and their metadata and 'concepts_surface_strings'
     pointing to flat list of strings realizing found concepts
    """
    con_list, exp_list = [], []
    for order, text_analyzed in enumerate(analyzed):
        for concept in text_analyzed:
            row = [doc_id, 0, concept.get('concept'),
                   concept.get('freq'), concept.get('relevance_score'),
                   concept.get('type')]
            if single_document:
                row[1] += order
            else:
                row[0] += order
            con_list.append(row)
            for string in concept.get('surface', []):
                surface_str, span = string['surface_string'], string['span']
                exp_list.append([row[0], row[1], concept.get('concept'),
                                 surface_str, '-'.join(map(str, span))])
    return {'concepts': con_list, 'concepts_surface_strings': exp_list}


def sentiments_to_list(doc_id, analyzed, single_document=False):
    """Converts sentiments response to flat lists with doc_id included

    :param doc_id: Will be inserted to each row as first element
    :param analyzed: Response json from anacode api for sentiment call
    :type analyzed: list
    :param single_document: Is analysis describing just one document
    :type single_document: bool
    :return: dict -- Dictionary with one key 'sentiments' pointing to flat list
     of sentiment probabilities
    """
    sen_list = []<commit_msg>Making concept dumping slightly more robust<commit_after># -*- coding: utf-8 -*-
import os
import csv
import datetime
import pandas as pd
from itertools import chain
from functools import partial

from anacode import codes


def backup(root, files):
    """Backs up `files` from `root` directory and return list of backed up
    file names. Backed up files will have datetime suffix appended to original
    file name.

    :param root: Absolute path to folder where files to backup are located
    :type root: str
    :param files: Names of files that needs backing up
    :type files: str
    :return: list -- List of backed up file names
    """
    backed_up = []
    join = os.path.join
    root_contents = os.listdir(root)
    dt_str = datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S')
    for file_name in files:
        if file_name not in root_contents:
            continue
        new_name = file_name + '_' + dt_str
        os.rename(join(root, file_name), join(root, new_name))
        backed_up.append(new_name)
    return backed_up


HEADERS = {
    'categories': [u'doc_id', u'text_order', u'category', u'probability'],
    'concepts': [u'doc_id', u'text_order', u'concept', u'freq',
                 u'relevance_score', u'concept_type'],
    'concepts_surface_strings': [u'doc_id', u'text_order', u'concept',
                                 u'surface_string', u'text_span'],
    'sentiments': [u'doc_id', u'text_order', u'sentiment_value'],
    'absa_entities': [u'doc_id', u'text_order', u'entity_name', u'entity_type',
                      u'surface_string', u'text_span'],
    'absa_normalized_texts': [u'doc_id', u'text_order', u'normalized_text'],
    'absa_relations': [u'doc_id', u'text_order', u'relation_id',
                       u'opinion_holder', u'restriction', u'sentiment_value',
                       u'is_external', u'surface_string', u'text_span'],
    'absa_relations_entities': [u'doc_id', u'text_order', u'relation_id',
                                u'entity_type', u'entity_name'],
    'absa_evaluations': [u'doc_id', u'text_order', u'evaluation_id',
                         u'sentiment_value', u'surface_string', u'text_span'],
    'absa_evaluations_entities': [u'doc_id', u'text_order', u'evaluation_id',
                                  u'entity_type', u'entity_name'],
}


# `anacode.agg.aggregations.ApiDataset.from_path` depends
# on ordering of files defined in values here
CSV_FILES = {
    'categories': ['categories.csv'],
    'concepts': ['concepts.csv', 'concepts_surface_strings.csv'],
    'sentiments': ['sentiments.csv'],
    'absa': [
        'absa_entities.csv', 'absa_normalized_texts.csv',
        'absa_relations.csv', 'absa_relations_entities.csv',
        'absa_evaluations.csv', 'absa_evaluations_entities.csv'
    ]
}


def categories_to_list(doc_id, analyzed, single_document=False):
    """Converts categories response to flat list with doc_id included.

    :param doc_id: Will be inserted to each row as first element
    :param analyzed: Response json from anacode api for categories call
    :type analyzed: list
    :param single_document: Is analysis describing just one document
    :type single_document: bool
    :return: dict -- Dictionary with one key 'categories' pointing to flat list
     of categories
    """
    cat_list = []
    for order, text_analyzed in enumerate(analyzed):
        for result_dict in text_analyzed:
            row = [doc_id, 0, result_dict.get('label'),
                   result_dict.get('probability')]
            if single_document:
                row[1] += order
            else:
                row[0] += order
            cat_list.append(row)
    return {'categories': cat_list}


def concepts_to_list(doc_id, analyzed, single_document=False):
    """Converts concepts response to flat lists with doc_id included

    :param doc_id: Will be inserted to each row as first element
    :param analyzed: Response json from anacode api for concepts call
    :type analyzed: list
    :param single_document: Is analysis describing just one document
    :type single_document: bool
    :return: dict -- Dictionary with two keys: 'concepts' pointing to flat list
     of found concepts and their metadata and 'concepts_surface_strings'
     pointing to flat list of strings realizing found concepts
    """
    con_list, exp_list = [], []
    for order, text_analyzed in enumerate(analyzed):
        for concept in text_analyzed or []:
            row = [doc_id, 0, concept.get('concept'),
                   concept.get('freq'), concept.get('relevance_score'),
                   concept.get('type')]
            if single_document:
                row[1] += order
            else:
                row[0] += order
            con_list.append(row)
            for string in concept.get('surface', []):
                surface_str, span = string['surface_string'], string['span']
                exp_list.append([row[0], row[1], concept.get('concept'),
                                 surface_str, '-'.join(map(str, span))])
    return {'concepts': con_list, 'concepts_surface_strings': exp_list}


def sentiments_to_list(doc_id, analyzed, single_document=False):
    """Converts sentiments response to flat lists with doc_id included

    :param doc_id: Will be inserted to each row as first element
    :param analyzed: Response json from anacode api for sentiment call
    :type analyzed: list
    :param single_document: Is analysis describing just one document
    :type single_document: bool
    :return: dict -- Dictionary with one key 'sentiments' pointing to flat list
     of sentiment probabilities
    """
    sen_list = []
{'fraction_non_alphanumeric': 0.10791622293219737, 'fraction_numerical': 0.001597444089456869, 'mean_word_length': 2.9101318528799447, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1964425', 'n_tokens_mistral': 3372, 'n_tokens_neox': 3265, 'n_words': 956}