<commit_before>import os
import subprocess
import hashlib
import urllib.request
from urllib.request import urlretrieve

import pandas as pd


def get_sha256(fn):
    # calculate sha256 of downloaded file
    sha256 = hashlib.sha256()
    with open(fn, 'rb') as fp:
        while True:
            chunk = fp.read(sha256.block_size)
            if not chunk:
                break
            sha256.update(chunk)

    return sha256.hexdigest()


def test_download_mimic_demo(mimic_demo_path, mimic_demo_url, mimic_tables):
    """
    Download the MIMIC demo to a local folder.
    """
    # download the SHA256 sums
    r = urllib.request.urlopen(f'{mimic_demo_url}SHA256SUMS.txt')
    sha_values = r.read().decode('utf-8').rstrip('\n')
    sha_values = [x.split(' ') for x in sha_values.split('\n')]

    sha_fn = [x[1] for x in sha_values]
    sha_values = [x[0] for x in sha_values]

    # download each table
    for table in mimic_tables:
        # ensure we have a reference SHA-256 sum
        assert f'{table}.csv' in sha_fn
        idx = sha_fn.index(f'{table}.csv')
        sha_ref = sha_values[idx]

        fn = os.path.join(mimic_demo_path, f'{table}.csv')

        # don't download the file if it already exists
        if os.path.exists(fn):
            fn_sha = get_sha256(fn)
            if fn_sha == sha_ref:
                # no need to download again!
                continue

        # download the file
        urlretrieve(f'{mimic_demo_url}{table}.csv', fn)

        # check we downloaded the file properly
        fn_sha = get_sha256(fn)
        assert fn_sha == sha_ref


def test_build_mimic_demo(mimic_demo_path, mimic_db_params, create_mimic_db):
    """
    Try to build MIMIC-III demo using the make file and the downloaded data.
    """
    # call make files to create MIMIC user and build database
    build_path = os.path.join(os.getcwd(), 'buildmimic', 'postgres/')

    dbname = mimic_db_params['name']
    dbpass = mimic_db_params['password']
    dbuser = mimic_db_params['user']
    dbschema = mimic_db_params['schema']
    dbhost = mimic_db_params['host']

    # Create mimic user
    # make_user = f'make create-user DBNAME={dbname}'
    # subprocess.call(make_user, shell=True, cwd=build_path)

    # Build MIMIC demo
    make_mimic = (
        f'make mimic datadir={mimic_demo_path} '
        f'DBNAME={dbname} DBUSER={dbuser} DBPASS={dbpass} '
        f'DBSCHEMA={dbschema} DBHOST={dbhost}'
    )
    subprocess.check_output(make_mimic, shell=True, cwd=build_path)


# The MIMIC test db has been created by this point
# Add unit tests below


def test_db_con(mimic_con):<commit_msg>update path to build script. fix test.<commit_after>import os
import subprocess
import hashlib
import urllib.request
from urllib.request import urlretrieve

import pandas as pd


def get_sha256(fn):
    # calculate sha256 of downloaded file
    sha256 = hashlib.sha256()
    with open(fn, 'rb') as fp:
        while True:
            chunk = fp.read(sha256.block_size)
            if not chunk:
                break
            sha256.update(chunk)

    return sha256.hexdigest()


def test_download_mimic_demo(mimic_demo_path, mimic_demo_url, mimic_tables):
    """
    Download the MIMIC demo to a local folder.
    """
    # download the SHA256 sums
    r = urllib.request.urlopen(f'{mimic_demo_url}SHA256SUMS.txt')
    sha_values = r.read().decode('utf-8').rstrip('\n')
    sha_values = [x.split(' ') for x in sha_values.split('\n')]

    sha_fn = [x[1] for x in sha_values]
    sha_values = [x[0] for x in sha_values]

    # download each table
    for table in mimic_tables:
        # ensure we have a reference SHA-256 sum
        assert f'{table}.csv' in sha_fn
        idx = sha_fn.index(f'{table}.csv')
        sha_ref = sha_values[idx]

        fn = os.path.join(mimic_demo_path, f'{table}.csv')

        # don't download the file if it already exists
        if os.path.exists(fn):
            fn_sha = get_sha256(fn)
            if fn_sha == sha_ref:
                # no need to download again!
                continue

        # download the file
        urlretrieve(f'{mimic_demo_url}{table}.csv', fn)

        # check we downloaded the file properly
        fn_sha = get_sha256(fn)
        assert fn_sha == sha_ref


def test_build_mimic_demo(mimic_demo_path, mimic_db_params, create_mimic_db):
    """
    Try to build MIMIC-III demo using the make file and the downloaded data.
    """
    # call make files to create MIMIC user and build database
    build_path = os.path.join(os.getcwd(), 'mimic-iii', 'buildmimic',
                              'postgres/')

    dbname = mimic_db_params['name']
    dbpass = mimic_db_params['password']
    dbuser = mimic_db_params['user']
    dbschema = mimic_db_params['schema']
    dbhost = mimic_db_params['host']

    # Create mimic user
    # make_user = f'make create-user DBNAME={dbname}'
    # subprocess.call(make_user, shell=True, cwd=build_path)

    # Build MIMIC demo
    make_mimic = (
        f'make mimic datadir={mimic_demo_path} '
        f'DBNAME={dbname} DBUSER={dbuser} DBPASS={<PASSWORD>} '
        f'DBSCHEMA={dbschema} DBHOST={dbhost}'
    )
    subprocess.check_output(make_mimic, shell=True, cwd=build_path)


# The MIMIC test db has been created by this point
# Add unit tests below


def test_db_con(mimic_con):
{'fraction_non_alphanumeric': 0.10293012772351616, 'fraction_numerical': 0.014650638617580767, 'mean_word_length': 2.742094167252284, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2256995', 'n_tokens_mistral': 1900, 'n_tokens_neox': 1779, 'n_words': 452}