<commit_before>import pandas as pd
import numpy as np
from datashape import *

def unit_to_dtype(ds):
    """

    >>> unit_to_dtype('int32')
    dtype('int32')
    >>> unit_to_dtype('float64')
    dtype('float64')
    >>> unit_to_dtype('?int64')
    dtype('float64')
    >>> unit_to_dtype('string')
    dtype('O')
    """
    if isinstance(ds, str):
        ds = dshape(ds)[0]
    if isinstance(ds, Option) and isnumeric(ds):
        return unit_to_dtype(str(ds).replace('int', 'float').replace('?', ''))
    if isinstance(ds, Option) and ds.ty in (date_, datetime_, string):
        ds = ds.ty
    return to_numpy_dtype(ds)


def dshape_to_pandas(ds):
    """

    >>> dshape_to_pandas('{a: int32}')
    ({'a': dtype('int32')}, [])

    >>> dshape_to_pandas('{a: int32, when: datetime}')
    ({'a': dtype('int32')}, ['when'])

    >>> dshape_to_pandas('{a: ?int64}')
    ({'a': dtype('float64')}, [])
    """
    if isinstance(ds, str):
        ds = dshape(ds)
    if isinstance(ds, DataShape) and len(ds) == 1:
        ds = ds[0]

    dtypes = {name: unit_to_dtype(typ) for name, typ in ds.dict.items()
                if not 'date' in str(typ)}

    datetimes = [name for name, typ in ds.dict.items()
                    if 'date' in str(typ)]

    return dtypes, datetimes


def isnumeric(ds):
    """
    >>> isnumeric('int32')
    True
    >>> isnumeric('string')
    False
    >>> isnumeric('var * {amount: ?int32}')
    False
    """
    if isinstance(ds, str):
            ds = dshape(ds)<commit_msg>Get the measure as it might be DataShape which has not dict attr<commit_after>import numpy as np
from datashape import *

def unit_to_dtype(ds):
    """

    >>> unit_to_dtype('int32')
    dtype('int32')
    >>> unit_to_dtype('float64')
    dtype('float64')
    >>> unit_to_dtype('?int64')
    dtype('float64')
    >>> unit_to_dtype('string')
    dtype('O')
    """
    if isinstance(ds, str):
        ds = dshape(ds)[0]
    if isinstance(ds, Option) and isnumeric(ds):
        return unit_to_dtype(str(ds).replace('int', 'float').replace('?', ''))
    if isinstance(ds, Option) and ds.ty in (date_, datetime_, string):
        ds = ds.ty
    return to_numpy_dtype(ds)


def dshape_to_pandas(ds):
    """

    >>> dshape_to_pandas('{a: int32}')
    ({'a': dtype('int32')}, [])

    >>> dshape_to_pandas('{a: int32, when: datetime}')
    ({'a': dtype('int32')}, ['when'])

    >>> dshape_to_pandas('{a: ?int64}')
    ({'a': dtype('float64')}, [])
    """
    if isinstance(ds, str):
        ds = dshape(ds)
    if isinstance(ds, DataShape) and len(ds) == 1:
        ds = ds[0]

    dtypes = {name: unit_to_dtype(typ) for name, typ in ds.measure.dict.items()
                if not 'date' in str(typ)}

    datetimes = [name for name, typ in ds.measure.dict.items()
                    if 'date' in str(typ)]

    return dtypes, datetimes


def isnumeric(ds):
    """
    >>> isnumeric('int32')
    True
    >>> isnumeric('string')
    False
    >>> isnumeric('var * {amount: ?int32}')
    False
    """
    if isinstance(ds, str):
            ds = dshape(ds)
{'fraction_non_alphanumeric': 0.18389305510270623, 'fraction_numerical': 0.020215194000652102, 'mean_word_length': 2.792336217552534, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6783828', 'n_tokens_mistral': 1142, 'n_tokens_neox': 1117, 'n_words': 245}