<commit_before>"""
Classes for reading data files (or dictionaries)
and converting them to DataContainer objects.

:author: <NAME> (<EMAIL>)
:author: <NAME> (<EMAIL>)
:author: <NAME> (<EMAIL>)

:date: 10/25/2017
:organization: ETS
"""

import warnings
import pandas as pd

from functools import partial
from os.path import (abspath,
                     exists,
                     join,
                     splitext)

from rsmtool.container import DataContainer



def read_jsonlines(filename, converters=None):
    """ Read jsonlines from a file. 
    Normalize nested jsons with up to one level of nesting

    Parameters
    ----------
    filename: str
        Name of file to read
    converters : dict or None, optional
        A dictionary specifying how the types of the columns
        in the file should be converted. Specified in the same
        format as for `pd.read_csv()`.

    Returns
    -------
    df : pandas DataFrame
         Data frame containing the data in the given file.
    """

    df = pd.read_json(filename,
                      orient='records',
                      lines=True,
                      dtype=converters)
        
    # let's check if we have nested columns
    nested_cols = [c for c in df.columns if isinstance(df[c][0],dict)]

    if len(nested_cols) > 0:
        dfs = []
        for col in nested_cols:<commit_msg>Update rsmtool/reader.py<commit_after>"""
Classes for reading data files (or dictionaries)
and converting them to DataContainer objects.

:author: <NAME> (<EMAIL>)
:author: <NAME> (<EMAIL>)
:author: <NAME> (<EMAIL>)

:date: 10/25/2017
:organization: ETS
"""

import warnings
import pandas as pd

from functools import partial
from os.path import (abspath,
                     exists,
                     join,
                     splitext)

from rsmtool.container import DataContainer



def read_jsonlines(filename, converters=None):
    """
    Read jsonlines from a file. 
    Normalize nested jsons with up to one level of nesting

    Parameters
    ----------
    filename: str
        Name of file to read
    converters : dict or None, optional
        A dictionary specifying how the types of the columns
        in the file should be converted. Specified in the same
        format as for `pd.read_csv()`.

    Returns
    -------
    df : pandas DataFrame
         Data frame containing the data in the given file.
    """

    df = pd.read_json(filename,
                      orient='records',
                      lines=True,
                      dtype=converters)
        
    # let's check if we have nested columns
    nested_cols = [c for c in df.columns if isinstance(df[c][0],dict)]

    if len(nested_cols) > 0:
        dfs = []
        for col in nested_cols:
{'fraction_non_alphanumeric': 0.08995284729778745, 'fraction_numerical': 0.007254261878853826, 'mean_word_length': 2.0508849557522124, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2834487', 'n_tokens_mistral': 800, 'n_tokens_neox': 741, 'n_words': 286}