<commit_before>import contextlib
import csv
import StringIO
import unittest

import preproc

results_input = """
"NAME","COMPONENT_ID","ORD_VALUE","REFERENCE_LOW","REFERENCE_HIGH","REFERENCE_UNIT","SPECIMN_TAKEN_TIME","RESULT_DATE","STUDY_ID"
"PanelA_Test1","11","7.2","4.0","10.0","thou/cu mm","2012-06-22 13:30:00","2012-06-24 00:00:00","301"
"PanelA_Test1","11","7.2","4.0","10.0","thou/cu mm","2012-06-22 13:30:00","2012-06-24 00:00:00","302"
"PanelA_Test1","11","7.2","4.0","10.0","thou/cu mm","2012-06-22 13:30:00","2012-06-24 00:00:00","303"
"PanelA_Test1","11","7.2","4.0","10.0","thou/cu mm","2012-06-21 13:30:00","2012-06-23 00:00:00","303"
"PanelA_Test1","11","7.2","4.0","10.0","thou/cu mm","2012-06-21 13:30:00","2012-06-23 00:00:00","302"
"PanelB_Test1","21","123.1","4.0","10.0","IU/mL","2012-06-20 13:30:00","2012-06-22 00:00:00","303"
"PanelA_Test1","11","7.2","4.0","10.0","thou/cu mm","2012-06-20 13:30:00","2012-06-22 00:00:00","303"
"""

expected = """
"NAME","COMPONENT_ID","ORD_VALUE","REFERENCE_LOW","REFERENCE_HIGH","REFERENCE_UNIT","SPECIMN_TAKEN_TIME","RESULT_DATE","STUDY_ID"
"PanelA_Test1","11","7.2","4.0","10.0","thou/cu mm","2012-06-22 13:30:00","2012-06-24 00:00:00","301"
"PanelA_Test1","11","7.2","4.0","10.0","thou/cu mm","2012-06-22 13:30:00","2012-06-24 00:00:00","302"
"PanelA_Test1","11","7.2","4.0","10.0","thou/cu mm","2012-06-22 13:30:00","2012-06-24 00:00:00","303"
"PanelA_Test1","11","7.2","4.0","10.0","thou/cu mm","2012-06-21 13:30:00","2012-06-23 00:00:00","303"
"PanelA_Test1","11","7.2","4.0","10.0","thou/cu mm","2012-06-21 13:30:00","2012-06-23 00:00:00","302"
"PanelB_Test1","21","123.1","4.0","10.0","IU/mL","2012-06-20 13:30:00","2012-06-22 00:00:00","303"
"""

STUDY_ID = 'STUDY_ID'
COMPONENT_ID = 'COMPONENT_ID'


class PreprocessingTests(unittest.TestCase):
    def test_group_rows_by_panel(self):
        panels = {
            'rna': [1230],
            'cbc': [600, 712, 372]
        }

        rows = [
            {STUDY_ID: '300', COMPONENT_ID: '1230', 'data': '123'},
            {STUDY_ID: '301', COMPONENT_ID: '1230', 'data': '456'},
            {STUDY_ID: '300', COMPONENT_ID: '600', 'data': '10'},
            {STUDY_ID: '301', COMPONENT_ID: '600', 'data': '11'},
            {STUDY_ID: '300', COMPONENT_ID: '000', 'data': 'Yes'}
        ]

        expected = {
            'rna': [rows[0], rows[1]],
            'cbc': [rows[2], rows[3]]
        }

        grouped = preproc.group_rows_by_panel(panels, rows)

        self.assertDictEqual(expected, grouped)
<commit_msg>Add 'NONE' to expected output, group_rows_by_panel<commit_after>import contextlib
import csv
import StringIO
import unittest

import preproc

results_input = """
"NAME","COMPONENT_ID","ORD_VALUE","REFERENCE_LOW","REFERENCE_HIGH","REFERENCE_UNIT","SPECIMN_TAKEN_TIME","RESULT_DATE","STUDY_ID"
"PanelA_Test1","11","7.2","4.0","10.0","thou/cu mm","2012-06-22 13:30:00","2012-06-24 00:00:00","301"
"PanelA_Test1","11","7.2","4.0","10.0","thou/cu mm","2012-06-22 13:30:00","2012-06-24 00:00:00","302"
"PanelA_Test1","11","7.2","4.0","10.0","thou/cu mm","2012-06-22 13:30:00","2012-06-24 00:00:00","303"
"PanelA_Test1","11","7.2","4.0","10.0","thou/cu mm","2012-06-21 13:30:00","2012-06-23 00:00:00","303"
"PanelA_Test1","11","7.2","4.0","10.0","thou/cu mm","2012-06-21 13:30:00","2012-06-23 00:00:00","302"
"PanelB_Test1","21","123.1","4.0","10.0","IU/mL","2012-06-20 13:30:00","2012-06-22 00:00:00","303"
"PanelA_Test1","11","7.2","4.0","10.0","thou/cu mm","2012-06-20 13:30:00","2012-06-22 00:00:00","303"
"""

expected = """
"NAME","COMPONENT_ID","ORD_VALUE","REFERENCE_LOW","REFERENCE_HIGH","REFERENCE_UNIT","SPECIMN_TAKEN_TIME","RESULT_DATE","STUDY_ID"
"PanelA_Test1","11","7.2","4.0","10.0","thou/cu mm","2012-06-22 13:30:00","2012-06-24 00:00:00","301"
"PanelA_Test1","11","7.2","4.0","10.0","thou/cu mm","2012-06-22 13:30:00","2012-06-24 00:00:00","302"
"PanelA_Test1","11","7.2","4.0","10.0","thou/cu mm","2012-06-22 13:30:00","2012-06-24 00:00:00","303"
"PanelA_Test1","11","7.2","4.0","10.0","thou/cu mm","2012-06-21 13:30:00","2012-06-23 00:00:00","303"
"PanelA_Test1","11","7.2","4.0","10.0","thou/cu mm","2012-06-21 13:30:00","2012-06-23 00:00:00","302"
"PanelB_Test1","21","123.1","4.0","10.0","IU/mL","2012-06-20 13:30:00","2012-06-22 00:00:00","303"
"""

STUDY_ID = 'STUDY_ID'
COMPONENT_ID = 'COMPONENT_ID'


class PreprocessingTests(unittest.TestCase):
    def test_group_rows_by_panel(self):
        panels = {
            'rna': [1230],
            'cbc': [600, 712, 372]
        }

        rows = [
            {STUDY_ID: '300', COMPONENT_ID: '1230', 'data': '123'},
            {STUDY_ID: '301', COMPONENT_ID: '1230', 'data': '456'},
            {STUDY_ID: '300', COMPONENT_ID: '600', 'data': '10'},
            {STUDY_ID: '301', COMPONENT_ID: '600', 'data': '11'},
            {STUDY_ID: '300', COMPONENT_ID: '000', 'data': 'Yes'}
        ]

        expected = {
            'rna': [rows[0], rows[1]],
            'cbc': [rows[2], rows[3]],
            'NONE': [rows[4]]
        }

        grouped = preproc.group_rows_by_panel(panels, rows)

        self.assertDictEqual(expected, grouped)

{'fraction_non_alphanumeric': 0.2766990291262136, 'fraction_numerical': 0.2316504854368932, 'mean_word_length': 4.813769751693003, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1981770', 'n_tokens_mistral': 2945, 'n_tokens_neox': 2274, 'n_words': 251}