<commit_before>import pandas as pd
import numpy as np

import pandas_profiling


def test_example(tmpdir):
    df = pd.read_csv(
        "https://data.nasa.gov/api/views/gh4g-9sfh/rows.csv?accessType=DOWNLOAD"
    )
    # Note: Pandas does not support dates before 1880, so we ignore these for this analysis
    df["year"] = pd.to_datetime(df["year"], errors="coerce")

    # Example: Constant variable
    df["source"] = "NASA"

    # Example: Boolean variable
    df["boolean"] = np.random.choice([True, False], df.shape[0])

    # Example: Mixed with base types
    df["mixed"] = np.random.choice([1, "A"], df.shape[0])

    # Example: Highly correlated variables
    df["reclat_city"] = df["reclat"] + np.random.normal(scale=5, size=(len(df)))

    # Example: Duplicate observations
    duplicates_to_add = pd.DataFrame(df.iloc[0:10])
    duplicates_to_add["name"] = duplicates_to_add["name"] + " copy"

    df = df.append(duplicates_to_add, ignore_index=True)

    output_file = tmpdir / "profile.html"
    profile = df.profile_report(
        title="NASA Meteorites",
        samples={"head": 5, "tail": 5},
        correlations={"cramers": False},
        sort="ascending",
    )
    profile.to_file(output_file=output_file)
    assert (tmpdir / "profile.html").exists(), "Output file does not exist"
    assert (
        type(profile.get_description()) == dict
        and len(profile.get_description().items()) == 5
    ), "Unexpected result"
    assert set(profile.get_rejected_variables(0.9)) == {
        "reclat_city"
    }, "rejected variables not correct"
    assert [*profile.get_description()["variables"]] == [
        "boolean",
        "Counties",
        "fall",<commit_msg>Python3.5 compatibility<commit_after>import pandas as pd
import numpy as np

import pandas_profiling


def test_example(tmpdir):
    df = pd.read_csv(
        "https://data.nasa.gov/api/views/gh4g-9sfh/rows.csv?accessType=DOWNLOAD"
    )
    # Note: Pandas does not support dates before 1880, so we ignore these for this analysis
    df["year"] = pd.to_datetime(df["year"], errors="coerce")

    # Example: Constant variable
    df["source"] = "NASA"

    # Example: Boolean variable
    df["boolean"] = np.random.choice([True, False], df.shape[0])

    # Example: Mixed with base types
    df["mixed"] = np.random.choice([1, "A"], df.shape[0])

    # Example: Highly correlated variables
    df["reclat_city"] = df["reclat"] + np.random.normal(scale=5, size=(len(df)))

    # Example: Duplicate observations
    duplicates_to_add = pd.DataFrame(df.iloc[0:10])
    duplicates_to_add["name"] = duplicates_to_add["name"] + " copy"

    df = df.append(duplicates_to_add, ignore_index=True)

    output_file = tmpdir / "profile.html"
    profile = df.profile_report(
        title="NASA Meteorites",
        samples={"head": 5, "tail": 5},
        correlations={"cramers": False},
        sort="ascending",
    )
    profile.to_file(output_file=output_file)
    assert (tmpdir / "profile.html").exists(), "Output file does not exist"
    assert (
        type(profile.get_description()) == dict
        and len(profile.get_description().items()) == 5
    ), "Unexpected result"
    assert set(profile.get_rejected_variables(0.9)) == {
        "reclat_city"
    }, "rejected variables not correct"
    assert list(profile.get_description()["variables"].keys()) == [
        "boolean",
        "Counties",
        "fall",
{'fraction_non_alphanumeric': 0.14868035190615836, 'fraction_numerical': 0.011143695014662757, 'mean_word_length': 3.395618556701031, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6162612', 'n_tokens_mistral': 1151, 'n_tokens_neox': 1092, 'n_words': 232}