<commit_before>
import random
from collections import defaultdict
import json
import os
import zipfile
import sys
if sys.version_info[0]==2:
    from urllib import urlretrieve
else:
    from urllib.request import urlretrieve

import pandas as pd
import numpy as np

from shorttext.utils.deprecation import deprecated


def retrieve_csvdata_as_dict(filepath):
    """ Retrieve the training data in a CSV file.

    Retrieve the training data in a CSV file, with the first column being the
    class labels, and second column the text data. It returns a dictionary with
    the class labels as keys, and a list of short texts as the value for each key.

    :param filepath: path of the training data (CSV)
    :return: a dictionary with class labels as keys, and lists of short texts
    :type filepath: str
    :rtype: dict
    """
    df = pd.read_csv(filepath)
    category_col, descp_col = df.columns.values.tolist()
    shorttextdict = defaultdict(lambda : [])
    for category, descp in zip(df[category_col], df[descp_col]):
        if type(descp)==str:
            shorttextdict[category] += [descp]
    return dict(shorttextdict)


# deprecated, kept for backward compatibility
@deprecated
def retrieve_data_as_dict(filepath):
    """ Retrieve the training data in a CSV file.

    This calls :func:`~retrieve_csvdata_as_dict` for backward compatibility.

    :param filepath: path of the training data (CSV)
    :return: a dictionary with class labels as keys, and lists of short texts
    :type filepath: str
    :rtype: dict
    """
    return retrieve_csvdata_as_dict(filepath)


def retrieve_jsondata_as_dict(filepath):
    """ Retrieve the training data in a JSON file.

    Retrieve the training data in a JSON file, with
    the class labels as keys, and a list of short texts as the value for each key.
    It returns the corresponding dictionary.

    :param filepath: path of the training data (JSON)
    :return: a dictionary with class labels as keys, and lists of short texts<commit_msg>rewritten load_csvdata function to save memory by without loading all data into a pandas data frame<commit_after>
import random
from collections import defaultdict
import json
import os
import zipfile
import sys
import csv
if sys.version_info[0]==2:
    from urllib import urlretrieve
else:
    from urllib.request import urlretrieve

import pandas as pd
import numpy as np

from shorttext.utils.deprecation import deprecated


def retrieve_csvdata_as_dict(filepath):
    """ Retrieve the training data in a CSV file.

    Retrieve the training data in a CSV file, with the first column being the
    class labels, and second column the text data. It returns a dictionary with
    the class labels as keys, and a list of short texts as the value for each key.

    :param filepath: path of the training data (CSV)
    :return: a dictionary with class labels as keys, and lists of short texts
    :type filepath: str
    :rtype: dict
    """
    datafile = open(filepath, 'r')
    reader = csv.reader(datafile)
    headerread = False
    shorttextdict = defaultdict(lambda: [])
    for label, content in reader:
        if headerread:
            if type(content) == str:
                shorttextdict[label] += [content]
        else:
            category_col, descp_col = label, content
            headerread = True
    # df = pd.read_csv(filepath)
    # category_col, descp_col = df.columns.values.tolist()
    # shorttextdict = defaultdict(lambda : [])
    # for category, descp in zip(df[category_col], df[descp_col]):
    #     if type(descp)==str:
    #         shorttextdict[category] += [descp]
    return dict(shorttextdict)


# deprecated, kept for backward compatibility
@deprecated
def retrieve_data_as_dict(filepath):
    """ Retrieve the training data in a CSV file.

    This calls :func:`~retrieve_csvdata_as_dict` for backward compatibility.

    :param filepath: path of the training data (CSV)
    :return: a dictionary with class labels as keys, and lists of short texts
    :type filepath: str
    :rtype: dict
    """
    return retrieve_csvdata_as_dict(filepath)


def retrieve_jsondata_as_dict(filepath):
    """ Retrieve the training data in a JSON file.

    Retrieve the training data in a JSON file, with
    the class labels as keys, and a list of short texts as the value for each key.
    It returns the corresponding dictionary.

    :param filepath: path of the training data (JSON)
    :return: a dictionary with class labels as keys, and lists of short texts
{'fraction_non_alphanumeric': 0.06619280319857841, 'fraction_numerical': 0.000888494002665482, 'mean_word_length': 3.4584158415841584, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '950514', 'n_tokens_mistral': 1292, 'n_tokens_neox': 1258, 'n_words': 542}