<commit_before>
import pandas as pd
import json
import xmltodict
import warnings
from quantipy.core.tools.dp.prep import start_meta, condense_dichotomous_set


def quantipy_from_ascribe(path_xml, path_txt, text_key='main'):
 
    # Read the AScribe data (tab-delimited)
    meta_ascribe = xmltodict.parse(open(path_xml))
    data_ascribe = pd.DataFrame.from_csv(
        path_txt, 
        sep='\t', 
        header=0, 
        encoding='utf-16'
    )
    data_ascribe[data_ascribe.index.name] = data_ascribe.index
     
    # Start a Quantipy meta document
    meta = start_meta(text_key=text_key)
    meta['columns']['responseid'] = {
        'type': 'int',
        'text': {text_key: 'responseid'}
    }
    meta['sets']['data file']['items'] = ['columns@responseid']
    
    MultiForm = meta_ascribe['CodedQuestions']['MultiForm']
    if not isinstance(MultiForm, list):
        meta_ascribe['CodedQuestions']['MultiForm'] = [MultiForm]

    # Container to record the names, in order, of the resulting
    # coded columns
    coded_names = []
     
    for var in meta_ascribe['CodedQuestions']['MultiForm']:
        name = var['Name']
        coded_names.append(name)
        coded_from = var['FormTexts']['FormText']['Title']
        var_text = var['FormTexts']['FormText']['Text'].replace('\n', ' ')
        if var_text is None: var_text = 'Label not provided'
        var_text = {text_key: var_text}
        columns = []
        values = []
        for val in var['Answers']['Answer']:
            value = int(val['@Precode'])
            if value==0:
                msg = (
                    "The value 0 has been assigned to a code for the "
                    "variable '%s'."
                ) % (name)
                warnings.warn(msg)
            val_text = val['Texts']['Text']['#text'].replace('\n', ' ')
            if val_text is None: val_text = 'Label not provided'<commit_msg>Fixed problem related to missing variable/category text during conversion from Ascribe.<commit_after>
import pandas as pd
import json
import xmltodict
import warnings
from quantipy.core.tools.dp.prep import start_meta, condense_dichotomous_set


def quantipy_from_ascribe(path_xml, path_txt, text_key='main'):
 
    # Read the AScribe data (tab-delimited)
    meta_ascribe = xmltodict.parse(open(path_xml))
    data_ascribe = pd.DataFrame.from_csv(
        path_txt, 
        sep='\t', 
        header=0, 
        encoding='utf-16'
    )
    data_ascribe[data_ascribe.index.name] = data_ascribe.index
     
    # Start a Quantipy meta document
    meta = start_meta(text_key=text_key)
    meta['columns']['responseid'] = {
        'type': 'int',
        'text': {text_key: 'responseid'}
    }
    meta['sets']['data file']['items'] = ['columns@responseid']
    
    MultiForm = meta_ascribe['CodedQuestions']['MultiForm']
    if not isinstance(MultiForm, list):
        meta_ascribe['CodedQuestions']['MultiForm'] = [MultiForm]

    # Container to record the names, in order, of the resulting
    # coded columns
    coded_names = []
     
    for var in meta_ascribe['CodedQuestions']['MultiForm']:
        name = var['Name']
        coded_names.append(name)
        coded_from = var['FormTexts']['FormText']['Title']
        var_text = var['FormTexts']['FormText']['Text']
        var_text = '' if var_text is None else var_text.replace('\n', ' ')
        if var_text is None: var_text = 'Label not provided'
        var_text = {text_key: var_text}
        columns = []
        values = []
        for val in var['Answers']['Answer']:
            value = int(val['@Precode'])
            if value==0:
                msg = (
                    "The value 0 has been assigned to a code for the "
                    "variable '%s'."
                ) % (name)
                warnings.warn(msg)
            val_text = val['Texts']['Text']['#text']
            val_text = '' if val_text is None else val_text.replace('\n', ' ')
            if val_text is None: val_text = 'Label not provided'
{'fraction_non_alphanumeric': 0.13113113113113112, 'fraction_numerical': 0.0025025025025025025, 'mean_word_length': 2.5340406719717063, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2339270', 'n_tokens_mistral': 1256, 'n_tokens_neox': 1205, 'n_words': 283}