<commit_before># core for reading XDS input and log files - first IDXREF

def rj_parse_idxref_xds_inp(xds_inp_lines):
    general_records = []
    images = None
    phi = None

    for record in xds_inp_lines:

        if 'SPOT_RANGE=' in record:
            images = tuple(map(int, record.split('=')[-1].split()))
            continue

        if 'UNIT_CELL_CONSTANTS' in record:
            continue

        if 'SPACE_GROUP_NUMBER' in record:
            continue
        
        if 'OSCILLATION_RANGE' in record:
            phi = float(record.split('=')[-1].strip())

        general_records.append(record.strip())

    return images, phi, general_records

def rj_parse_idxref_lp(xds_lp_lines):
    cell = None

    for record in xds_lp_lines:
        if 'UNIT CELL PARAMETERS' in record:
            cell = tuple(map(float, record.split()[-6:]))

    return cell<commit_msg>Got this chapper producing a worthwhile XDS.INP file for correct from INTEGRATE.LP<commit_after># core for reading XDS input and log files - first IDXREF

def rj_parse_idxref_xds_inp(xds_inp_lines):
    general_records = []
    images = None
    phi = None

    for record in xds_inp_lines:

        if 'SPOT_RANGE=' in record:
            images = tuple(map(int, record.split('=')[-1].split()))
            continue

        if 'UNIT_CELL_CONSTANTS' in record:
            continue

        if 'SPACE_GROUP_NUMBER' in record:
            continue
        
        if 'OSCILLATION_RANGE' in record:
            phi = float(record.split('=')[-1].strip())

        general_records.append(record.strip())

    return images, phi, general_records

def rj_parse_idxref_lp(xds_lp_lines):
    cell = None

    for record in xds_lp_lines:
        if 'UNIT CELL PARAMETERS' in record:
            cell = tuple(map(float, record.split()[-6:]))

    return cell

def rj_parse_integrate_lp(integrate_lp_lines):
    general_records = []
    images = None
    phi = None
    cell = None

    ignore = [
        'MINIMUM_VALID_PIXEL_VALUE',
        'BACKGROUND_PIXEL',
        'MAXIMUM_ERROR_OF_SPOT_POSITION',
        'MINPK'
        ]
        

    for record in integrate_lp_lines:
        if '*********' in record:
            break

        if not '=' in record:
            continue

        if record.split('=')[0].strip() in ignore:
            continue

        if 'DATA_RANGE' in record:
            images = tuple(map(int, record.split('=')[-1].split()))
            continue

        if 'OSCILLATION_RANGE' in record:
            phi = float(record.replace('DEGREES', '').split()[-1])
            continue

        if 'UNIT_CELL_CONSTANTS' in record:
            cell = tuple(map(float, record.split('=')[-1].split()))
            continue

        if 'SPACE_GROUP_NUMBER' in record:
            continue

        if 'STARTING_ANGLE' in record:
            continue

        if 'RESOLUTION RANGE' in record:
            continue

        if 'NUMBER OF TRUSTED' in record:
            continue

        if 'MEAN CONTENTS' in record:
            continue

        if 'MEAN VALUE' in record:
            continue

        if 'NUMBER OF' in record:
            continue

        if 'PIXEL VALUE' in record:
            continue

        if 'X-RAY_WAVELENGTH' in record:
            general_records.append(record.replace('ANGSTROM', '').strip())
            continue

        if record.strip():
            general_records.append(record.replace('-0', ' -0').strip())

    return images, phi, cell, general_records

if __name__ == '__main__':
    import sys

    images, phi, cell, junk = rj_parse_integrate_lp(open(sys.argv[1]).readlines())

    print 'DATA_RANGE= %d %d' % images
    print 'OSCILLATION_RANGE= %.2f' % phi
    print 'UNIT_CELL_CONSTANTS= %.2f %.2f %.2f %.2f %.2f %.2f' % cell
    print 'SPACE_GROUP_NUMBER=1'

    for record in junk:
        print record
        
    standard = [
        'JOB=CORRECT',
        'MAXIMUM_NUMBER_OF_PROCESSORS=4',
        'CORRECTIONS=!',
        'REFINE(CORRECT)=CELL',
        'OVERLOAD=65000',
        'DIRECTION_OF_DETECTOR_X-AXIS=1.0 0.0 0.0',
        'DIRECTION_OF_DETECTOR_Y-AXIS=0.0 1.0 0.0',
        'TRUSTED_REGION=0.0 1.41'
        
        ]

    for record in standard:
        print record
        
        
{'fraction_non_alphanumeric': 0.10794249351873675, 'fraction_numerical': 0.010370021211407024, 'mean_word_length': 2.0314285714285716, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2133375', 'n_tokens_mistral': 1410, 'n_tokens_neox': 1358, 'n_words': 325}