<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright © 2015 jaidev <<EMAIL>>
#
# Distributed under terms of the MIT license.

"""
Misecellaneous bells and whistles.
"""

import json


class TypeEncoder(json.JSONEncoder):

    def default(self, obj):
        if isinstance(obj, type):
            return str(obj)
        elif isinstance(obj, set):
            return list(obj)
        elif callable(obj):
            return ".".join((obj.__module__, obj.__name__))
        else:
            return json.JSONEncoder.default(self, obj)

<commit_msg>ENH: Generate a questionnaire for the client<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright © 2015 jaidev <<EMAIL>>
#
# Distributed under terms of the MIT license.

"""
Misecellaneous bells and whistles.
"""

import json
import pandas as pd
import datetime

DATA_TYPES = {'String': str, 'Date/Time': datetime.date, 'Float': float,
              'Integer': int}


class TypeEncoder(json.JSONEncoder):

    def default(self, obj):
        if isinstance(obj, type):
            return str(obj)
        elif isinstance(obj, set):
            return list(obj)
        elif callable(obj):
            return ".".join((obj.__module__, obj.__name__))
        else:
            return json.JSONEncoder.default(self, obj)


def generate_questionnaire(filepath):
    """Generate a questionnaire for data at `filepath`.

    This questionnaire will be presented to the client, which helps us
    automatically generate the schema.

    :param filepath: Path to the file that needs to be ingested.
    :type filepath: str
    :return: A dictionary of questions and their possible answers. The format
    of the dictionary is such that every key is a question to be put to the
    client, and its value is a list of possible answers. The first item in the
    list is the default value.
    :rtype: dict
    """
    qdict = {}
    if filepath.endswith(".tsv"):
        dataframe = pd.read_table(filepath)
    else:
        dataframe = pd.read_csv(filepath)
    for col in dataframe.columns:
        qstring = "What is the data type of {}?".format(col)
        if "float" in str(dataframe[col].dtype).lower():
            defaultType = "Float"
        elif "object" in str(dataframe[col].dtype).lower():
            defaultType = "String"
        elif "int" in str(dataframe[col].dtype).lower():
            defaultType = "Integer"
        typeslist = DATA_TYPES.keys()
        typeslist.remove(defaultType)
        typeslist = [defaultType] + typeslist
        qdict[qstring] = typeslist
    return qdict


{'fraction_non_alphanumeric': 0.11060259344012205, 'fraction_numerical': 0.004576659038901602, 'mean_word_length': 2.6229281767955803, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5588745', 'n_tokens_mistral': 783, 'n_tokens_neox': 760, 'n_words': 252}