<commit_before>import os, os.path

class FileDescriptor(object):
    def __init__(self, token, mode, directory_name, filename, verify_exists = True):
        self.filename = filename
        self.complete_pathname = os.path.join(directory_name, filename)
        self.token = token
        self.mode = mode
        self.fd = None
        self._records_read = 0
        self._records_written = 0

        if verify_exists:
            if not os.path.exists(self.complete_pathname):
                print '** File not found !! ', self.complete_pathname
                raise Exception()

    def __str__(self):
        return 'token={0:25}, mode={1:10}\n\t filename={2:50} \n\t complete_pathname={3:50} \n\t fd={4}\n\t '.format(self.token, self.mode, self.filename, self.complete_pathname, self.fd)

    def get_patient_records(self, DESYNPUF_ID, record_list):
        ## assumes records are in DESYNPUF_ID order
        ## we will table all the records for an ID
        ## when that ID is requested, return them and table the next set
        ## if requested ID is less than buffer-ID, return empty list

        # print 'get_patient_records->',DESYNPUF_ID
        done = False
        skip_count = 0
        if self.fd is None:
            self.fd = open(self.complete_pathname,'r')
            # ignore header
            rec = self.fd.readline()

        while not done:
            try:
                rec = self.fd.readline()
                # print 'in->', rec
                if rec[0:len(DESYNPUF_ID)] == DESYNPUF_ID:
                    # print '\t ** keep'
                    # store array of fields instead of raw rec
                    record_list.append((rec[:-1]).split(','))
                elif rec[0:len(DESYNPUF_ID)] < DESYNPUF_ID:
                    skip_count += 1
                else:
                    # print '\t ** break; backup'
                    self.fd.seek(-len(rec),1)
                    done = True

            except IOError:
                print 'IO error'
                done = True

        # print self.token, ' len records->', len(record_list), ' skip_count->', skip_count
        # return records

    def open(self):
        if self.fd is None:
            open_mode = 'r'
            if self.mode == 'write': open_mode = 'w'
            self.fd = open(self.complete_pathname, open_mode)
        return self.fd

    def write(self, record):
        if self.mode == 'read':
            print '** Attempt to write to read-only file'
            raise Exception()
        if self.fd is None:
            open_mode = 'a'
            if self.mode == 'write': open_mode = 'w'
            self.fd = open(self.complete_pathname, open_mode)
        self.fd.write(record)

    def increment_recs_read(self, count=1):
        self._records_read += count

    def increment_recs_written(self, count=1):
        self._records_written += count

    def flush(self):
        if self.mode == 'read' or self.fd is None: return
        self.fd.flush()

    def close(self):
        if self.fd is None: return
        self.fd.close()

    @property
    def records_read(self):
        return self._records_read

    @property
    def records_written(self):
        return self._records_written


class FileControl(object):
    def __init__(self, base_synpuf_input_directory, base_output_directory, synpuf_dir_format, sample_number, verify_exists = True):
        self.base_directory = base_synpuf_input_directory
        self.base_output_directory = base_output_directory

        input_directory = os.path.join(base_synpuf_input_directory, synpuf_dir_format.format(sample_number))

        self.files = {}

        #-- input files
        self.files['beneficiary'] = FileDescriptor('beneficiary', 'read', input_directory,
                                                "DE1_0_comb_Beneficiary_Summary_File_Sample_" + str(sample_number) + ".csv.srt")
        self.files['inpatient'] = FileDescriptor('inpatient', 'read', input_directory,
                                                "DE1_0_2008_to_2010_Inpatient_Claims_Sample_" + str(sample_number) + ".csv.srt")
        self.files['outpatient'] = FileDescriptor('outpatient', 'read', input_directory,
                                                "DE1_0_2008_to_2010_Outpatient_Claims_Sample_" + str(sample_number) + ".csv.srt")
        self.files['carrier_A'] = FileDescriptor('carrier_A', 'read', input_directory,
                                                "DE1_0_2008_to_2010_Carrier_Claims_Sample_" + str(sample_number) + "A.csv.srt")
        self.files['carrier_B'] = FileDescriptor('carrier_B', 'read', input_directory,
                                                "DE1_0_2008_to_2010_Carrier_Claims_Sample_" + str(sample_number) + "B.csv.srt")
        self.files['prescription'] = FileDescriptor('prescription', 'read', input_directory,
                                                "DE1_0_2008_to_2010_Prescription_Drug_Events_Sample_" + str(sample_number) + ".csv.srt")

        # output files
        output_files = [
                'person',
                'observation',
                'observation_period',
                'specimen',
                'death',
                'visit_occurrence',
                'visit_cost',
                'condition_occurrence',
                'procedure_occurrence',
                'procedure_cost',
                'drug_exposure',
                'drug_cost',
                'device_exposure',
                'device_cost',
                'measurement_occurrence',
                'location',
                'care_site',
                'provider',
                'payer_plan_period']

        for token in output_files:
            self.files[token] = FileDescriptor(token, 'append',
                                              self.base_output_directory,
                                              '{0}_{1}.csv'.format(token, sample_number),
                                              verify_exists = False)<commit_msg>Sort CSV files if necessary<commit_after>import os, os.path

class FileDescriptor(object):
    def __init__(self, token, mode, directory_name, filename, verify_exists = True):
        self.filename = filename
        self.directory_name = directory_name
        self.complete_pathname = os.path.join(directory_name, filename)
        self.token = token
        self.mode = mode
        self.verify_exists = verify_exists
        self.fd = None
        self._records_read = 0
        self._records_written = 0

        if verify_exists:
            if not os.path.exists(self.complete_pathname):
                print '** File not found !! ', self.complete_pathname
                raise Exception()

    def __str__(self):
        return 'token={0:25}, mode={1:10}\n\t filename={2:50} \n\t complete_pathname={3:50} \n\t fd={4}\n\t '.format(self.token, self.mode, self.filename, self.complete_pathname, self.fd)

    def get_patient_records(self, DESYNPUF_ID, record_list):
        ## assumes records are in DESYNPUF_ID order
        ## we will table all the records for an ID
        ## when that ID is requested, return them and table the next set
        ## if requested ID is less than buffer-ID, return empty list

        # print 'get_patient_records->',DESYNPUF_ID
        done = False
        skip_count = 0
        if self.fd is None:
            self.fd = open(self.complete_pathname,'r')
            # ignore header
            rec = self.fd.readline()

        while not done:
            try:
                rec = self.fd.readline()
                # print 'in->', rec
                if rec[0:len(DESYNPUF_ID)] == DESYNPUF_ID:
                    # print '\t ** keep'
                    # store array of fields instead of raw rec
                    record_list.append((rec[:-1]).split(','))
                elif rec[0:len(DESYNPUF_ID)] < DESYNPUF_ID:
                    skip_count += 1
                else:
                    # print '\t ** break; backup'
                    self.fd.seek(-len(rec),1)
                    done = True

            except IOError:
                print 'IO error'
                done = True

        # print self.token, ' len records->', len(record_list), ' skip_count->', skip_count
        # return records

    def open(self):
        if self.fd is None:
            open_mode = 'r'
            if self.mode == 'write': open_mode = 'w'
            self.fd = open(self.complete_pathname, open_mode)
        return self.fd

    def write(self, record):
        if self.mode == 'read':
            print '** Attempt to write to read-only file'
            raise Exception()
        if self.fd is None:
            open_mode = 'a'
            if self.mode == 'write': open_mode = 'w'
            self.fd = open(self.complete_pathname, open_mode)
        self.fd.write(record)

    def increment_recs_read(self, count=1):
        self._records_read += count

    def increment_recs_written(self, count=1):
        self._records_written += count

    def flush(self):
        if self.mode == 'read' or self.fd is None: return
        self.fd.flush()

    def close(self):
        if self.fd is None: return
        self.fd.close()

    @property
    def records_read(self):
        return self._records_read

    @property
    def records_written(self):
        return self._records_written

    def sorted(self):
        sorted_path = self.complete_pathname + '.srt'
        if not (os.path.exists(sorted_path) and os.path.getsize(sorted_path) > 0):
            print("Sorting {0}".format(self.complete_pathname))
            with open(sorted_path, 'w') as sorted_file:
                with open(self.complete_pathname) as source:
                    contents = sorted(source.readlines())
                    sorted_file.writelines(contents)
                    sorted_file.flush()
        return FileDescriptor(self.token, self.mode, self.directory_name, self.filename + '.srt', self.verify_exists)


class FileControl(object):
    def __init__(self, base_synpuf_input_directory, base_output_directory, synpuf_dir_format, sample_number, verify_exists = True):
        self.base_directory = base_synpuf_input_directory
        self.base_output_directory = base_output_directory

        input_directory = os.path.join(base_synpuf_input_directory, synpuf_dir_format.format(sample_number))

        self.files = {}

        #-- input files
        self.files['beneficiary'] = FileDescriptor('beneficiary', 'read', input_directory,
                                                "DE1_0_comb_Beneficiary_Summary_File_Sample_" + str(sample_number) + ".csv").sorted()
        self.files['inpatient'] = FileDescriptor('inpatient', 'read', input_directory,
                                                "DE1_0_2008_to_2010_Inpatient_Claims_Sample_" + str(sample_number) + ".csv").sorted()
        self.files['outpatient'] = FileDescriptor('outpatient', 'read', input_directory,
                                                "DE1_0_2008_to_2010_Outpatient_Claims_Sample_" + str(sample_number) + ".csv").sorted()
        self.files['carrier_A'] = FileDescriptor('carrier_A', 'read', input_directory,
                                                "DE1_0_2008_to_2010_Carrier_Claims_Sample_" + str(sample_number) + "A.csv").sorted()
        self.files['carrier_B'] = FileDescriptor('carrier_B', 'read', input_directory,
                                                "DE1_0_2008_to_2010_Carrier_Claims_Sample_" + str(sample_number) + "B.csv").sorted()
        self.files['prescription'] = FileDescriptor('prescription', 'read', input_directory,
                                                "DE1_0_2008_to_2010_Prescription_Drug_Events_Sample_" + str(sample_number) + ".csv").sorted()

        # output files
        output_files = [
                'person',
                'observation',
                'observation_period',
                'specimen',
                'death',
                'visit_occurrence',
                'visit_cost',
                'condition_occurrence',
                'procedure_occurrence',
                'procedure_cost',
                'drug_exposure',
                'drug_cost',
                'device_exposure',
                'device_cost',
                'measurement_occurrence',
                'location',
                'care_site',
                'provider',
                'payer_plan_period']

        for token in output_files:
            self.files[token] = FileDescriptor(token, 'append',
                                              self.base_output_directory,
                                              '{0}_{1}.csv'.format(token, sample_number),
                                              verify_exists = False)
{'fraction_non_alphanumeric': 0.10246738959610246, 'fraction_numerical': 0.012258368694012258, 'mean_word_length': 1.7836832895888013, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1042908', 'n_tokens_mistral': 3674, 'n_tokens_neox': 3424, 'n_words': 736}