<commit_before>import pandas as pd
from StringIO import StringIO

try:
    from pandas.io.common import ZipFile
except ImportError:
    from zipfile import ZipFile

def get_movielens_data(local_file=None, get_ratings=True, get_genres=False,
                        split_genres=True, mdb_mapping=False):
    '''Downloads movielens data and stores it in pandas dataframe.
    '''
    if not local_file:
        # downloading data
        from requests import get
        zip_file_url = 'http://files.grouplens.org/datasets/movielens/ml-1m.zip'
        zip_response = get(zip_file_url)
        zip_contents = StringIO(zip_response.content)
    else:
        zip_contents = local_file

    ml_data = ml_genres = mapping = None
    # loading data into memory
    with ZipFile(zip_contents) as zfile:
        zip_files = pd.Series(zfile.namelist())
        zip_file = zip_files[zip_files.str.contains('ratings')].iat[0]
        is_latest = 'latest' in zip_file
        header = 0 if is_latest else None
        if get_ratings:
            zdata = zfile.read(zip_file)
            delimiter = ','
            zdata = zdata.replace('::', delimiter) # makes data compatible with pandas c-engine
            ml_data = pd.read_csv(StringIO(zdata), sep=delimiter, header=header, engine='c',
                                    names=['userid', 'movieid', 'rating', 'timestamp'],
                                    usecols=['userid', 'movieid', 'rating'])

        if get_genres:
            zip_file = zip_files[zip_files.str.contains('movies')].iat[0]
            with zfile.open(zip_file) as zdata:
                delimiter = ',' if is_latest else '::'
                genres_data = pd.read_csv(zdata, sep=delimiter, header=header, engine='python',
                                            names=['movieid', 'movienm', 'genres'])

            ml_genres = get_split_genres(genres_data) if split_genres else genres_data

        if is_latest and mdb_mapping:
            # imdb and tmdb mapping - exists only in ml-latest datasets
            zip_file = zip_files[zip_files.str.contains('links')].iat[0]
            with zfile.open(zip_file) as zdata:
                mapping = pd.read_csv(zdata, sep=',', header=0, engine='c',
                                        names=['movieid', 'imdbid', 'tmdbid'])
<commit_msg>handle byte-string interoperability<commit_after>from io import BytesIO
import pandas as pd

try:
    from pandas.io.common import ZipFile
except ImportError:
    from zipfile import ZipFile

def get_movielens_data(local_file=None, get_ratings=True, get_genres=False,
                        split_genres=True, mdb_mapping=False):
    '''Downloads movielens data and stores it in pandas dataframe.
    '''
    if not local_file:
        # downloading data
        from requests import get
        zip_file_url = 'http://files.grouplens.org/datasets/movielens/ml-1m.zip'
        zip_response = get(zip_file_url)
        zip_contents = BytesIO(zip_response.content)
    else:
        zip_contents = local_file

    ml_data = ml_genres = mapping = None
    # loading data into memory
    with ZipFile(zip_contents) as zfile:
        zip_files = pd.Series(zfile.namelist())
        zip_file = zip_files[zip_files.str.contains('ratings')].iat[0]
        is_latest = 'latest' in zip_file
        header = 0 if is_latest else None
        if get_ratings:
            zdata = zfile.read(zip_file)
            delimiter = ','
            zdata = zdata.replace(b'::', delimiter.encode()) # makes data compatible with pandas c-engine
            ml_data = pd.read_csv(BytesIO(zdata), sep=delimiter, header=header, engine='c',
                                    names=['userid', 'movieid', 'rating', 'timestamp'],
                                    usecols=['userid', 'movieid', 'rating'])

        if get_genres:
            zip_file = zip_files[zip_files.str.contains('movies')].iat[0]
            with zfile.open(zip_file) as zdata:
                delimiter = ',' if is_latest else '::'
                genres_data = pd.read_csv(zdata, sep=delimiter, header=header, engine='python',
                                            names=['movieid', 'movienm', 'genres'])

            ml_genres = get_split_genres(genres_data) if split_genres else genres_data

        if is_latest and mdb_mapping:
            # imdb and tmdb mapping - exists only in ml-latest datasets
            zip_file = zip_files[zip_files.str.contains('links')].iat[0]
            with zfile.open(zip_file) as zdata:
                mapping = pd.read_csv(zdata, sep=',', header=0, engine='c',
                                        names=['movieid', 'imdbid', 'tmdbid'])

{'fraction_non_alphanumeric': 0.10410841041084104, 'fraction_numerical': 0.002581200258120026, 'mean_word_length': 2.240418118466899, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3801995', 'n_tokens_mistral': 1426, 'n_tokens_neox': 1377, 'n_words': 316}