<commit_before>'''
Created on 2 Dec 2014

@author: jmht
'''

import logging
import os
import shutil
import sys

from iotbx import reflection_file_reader

import ample_util # Avoid circular dependencies
import exit_util
import cif_parser # Avoid circular dependencies

_logger = logging.getLogger()
_logger.setLevel(logging.DEBUG)

def del_column(file_name, column, overwrite=True):
    """Delete a column from an mtz file and return a path to the file"""
    mtzDel = ample_util.filename_append(file_name, "d{0}".format(column) )
    cmd = [ "mtzutils", "hklin1", file_name, "hklout", mtzDel ]
    stdin = "EXCLUDE 1 {0}".format( column )
    logfile = os.path.join( os.getcwd(), "mtzutils.log" )
    retcode = ample_util.run_command(cmd, stdin=stdin, logfile=logfile)
    if retcode != 0:
        msg = "Error running mtzutils. Check the logfile: {0}".format(logfile)
        _logger.critical(msg)
        raise RuntimeError, msg
    
    if overwrite:
        shutil.move(mtzDel,file_name)
        return file_name
    else:
        return mtzDel

def add_rfree(file_name,directory=None,overwrite=True):
    """Run uniqueify on mtz file to generate RFREE data column"""
    mtzUnique = ample_util.filename_append(file_name, "uniqueify", directory=directory)

    cmd = ['uniqueify', file_name, mtzUnique]
    logfile = os.path.join( os.getcwd(), "uniqueify.log" )
    retcode = ample_util.run_command(cmd, logfile=logfile)
    if retcode != 0:
        msg = "Error running command: {0}. Check the logfile: {1}".format(" ".join(cmd),logfile)
        _logger.critical(msg)
        raise RuntimeError, msg

    if overwrite:
        shutil.move(mtzUnique,file_name)
        return file_name
    else:
        return mtzUnique

def get_labels(file_name):
    """Return the F, FP and FREE column labels"""
    
    reflection_file = reflection_file_reader.any_reflection_file(file_name=file_name)
    if not reflection_file.file_type()=="ccp4_mtz":
        msg="File is not of type ccp4_mtz: {0}".format(file_name)
        logging.critical(msg)
        raise RuntimeError,msg
    
    content=reflection_file.file_content()
    ctypes=content.column_types()
    clabels=content.column_labels()
    ftype='F'
    if not ftype in ctypes:
        raise RuntimeError,"Cannot find any structure amplitudes in: {0}".format(file_name)
    F=clabels[ctypes.index(ftype)]
    
    # FP derived from F
    FP='SIG'+F
    if not FP in clabels:
        raise RuntimeError,"Cannot find label {0} in file: {1}".format(FP,file_name)
    
    FREE=_get_rfree(content)
    return F,FP,FREE

def get_rfree(file_name):
    """Return the Rfree label"""

    reflection_file = reflection_file_reader.any_reflection_file(file_name=file_name)
    if not reflection_file.file_type()=="ccp4_mtz":
        msg="File is not of type ccp4_mtz: {0}".format(file_name)
        logging.critical(msg)
        raise RuntimeError,msg
    
    return _get_rfree(reflection_file.file_content())

# def get_resolution(file_name):
#     if int(CCP4_VERSION[0]) >= 7:
#         hkl_info=clipper.HKL_info()
#         mtz_file=clipper.CCP4MTZfile()
#         mtz_file.open_read(file_name)
#         mtz_file.import_hkl_info(hkl_info)
# 
#         resolution =  "%.2lf" % hkl_info.resolution().limit()
#     else:
#         resolution = None
#     return resolution

def _get_rfree(content):
    rfree_label = None
    #print "GOT ",content.column_labels()
    for label in content.column_labels():
        if 'free' in label.lower():
            column = content.get_column(label=label)
            selection_valid = column.selection_valid()
            flags = column.extract_values()
            sel_0 = (flags == 0)
            # extract number of work/test reflections
            n0=( sel_0 & selection_valid).count(True)
            n1=(~sel_0 & selection_valid).count(True)
            #print "Number of 0 (work):",n0
            #print "Number of 1 (test):",n1
            #print float(n0)/float(n1)*100
            if n0>0 and n1>0:
                if rfree_label:
                    _logger.warning("FOUND >1 RFREE label in file!")<commit_msg>Fix  documentation and checks on getting F and SIGF now that I understand things better<commit_after>'''
Created on 2 Dec 2014

@author: jmht
'''

import logging
import os
import shutil
import sys

from iotbx import reflection_file_reader

import ample_util # Avoid circular dependencies
import exit_util
import cif_parser # Avoid circular dependencies

_logger = logging.getLogger()
_logger.setLevel(logging.DEBUG)

COLTYPE_F = 'F'
COLTYPE_SIGF = 'Q'

def del_column(file_name, column, overwrite=True):
    """Delete a column from an mtz file and return a path to the file"""
    mtzDel = ample_util.filename_append(file_name, "d{0}".format(column) )
    cmd = [ "mtzutils", "hklin1", file_name, "hklout", mtzDel ]
    stdin = "EXCLUDE 1 {0}".format( column )
    logfile = os.path.join( os.getcwd(), "mtzutils.log" )
    retcode = ample_util.run_command(cmd, stdin=stdin, logfile=logfile)
    if retcode != 0:
        msg = "Error running mtzutils. Check the logfile: {0}".format(logfile)
        _logger.critical(msg)
        raise RuntimeError, msg
    
    if overwrite:
        shutil.move(mtzDel,file_name)
        return file_name
    else:
        return mtzDel

def add_rfree(file_name,directory=None,overwrite=True):
    """Run uniqueify on mtz file to generate RFREE data column"""
    mtzUnique = ample_util.filename_append(file_name, "uniqueify", directory=directory)

    cmd = ['uniqueify', file_name, mtzUnique]
    logfile = os.path.join( os.getcwd(), "uniqueify.log" )
    retcode = ample_util.run_command(cmd, logfile=logfile)
    if retcode != 0:
        msg = "Error running command: {0}. Check the logfile: {1}".format(" ".join(cmd),logfile)
        _logger.critical(msg)
        raise RuntimeError, msg

    if overwrite:
        shutil.move(mtzUnique,file_name)
        return file_name
    else:
        return mtzUnique

def get_labels(file_name):
    """Return the F, SIGF and FREE column labels"""
    
    reflection_file = reflection_file_reader.any_reflection_file(file_name=file_name)
    if not reflection_file.file_type()=="ccp4_mtz":
        msg="File is not of type ccp4_mtz: {0}".format(file_name)
        logging.critical(msg)
        raise RuntimeError,msg
    
    content=reflection_file.file_content()
    ctypes=content.column_types()
    clabels=content.column_labels()
    if not COLTYPE_F in ctypes:
        raise RuntimeError,"Cannot find any structure amplitudes in: {0}".format(file_name)
    F = clabels[ctypes.index(COLTYPE_F)]
    
    # SIGF derived from F
    SIGF = 'SIG' + F
    if not SIGF in clabels:
        raise RuntimeError,"Cannot find label {0} in file: {1}".format(SIGF)
    i = clabels.index(SIGF)
    if ctypes[i] != COLTYPE_SIGF:
        raise RuntimeError,"SIGF label {0} is not of type: {1}".format(SIGF, )
    
    FREE=_get_rfree(content)
    return F,SIGF,FREE

def get_rfree(file_name):
    """Return the Rfree label"""

    reflection_file = reflection_file_reader.any_reflection_file(file_name=file_name)
    if not reflection_file.file_type()=="ccp4_mtz":
        msg="File is not of type ccp4_mtz: {0}".format(file_name)
        logging.critical(msg)
        raise RuntimeError,msg
    
    return _get_rfree(reflection_file.file_content())

# def get_resolution(file_name):
#     if int(CCP4_VERSION[0]) >= 7:
#         hkl_info=clipper.HKL_info()
#         mtz_file=clipper.CCP4MTZfile()
#         mtz_file.open_read(file_name)
#         mtz_file.import_hkl_info(hkl_info)
# 
#         resolution =  "%.2lf" % hkl_info.resolution().limit()
#     else:
#         resolution = None
#     return resolution

def _get_rfree(content):
    rfree_label = None
    #print "GOT ",content.column_labels()
    for label in content.column_labels():
        if 'free' in label.lower():
            column = content.get_column(label=label)
            selection_valid = column.selection_valid()
            flags = column.extract_values()
            sel_0 = (flags == 0)
            # extract number of work/test reflections
            n0=( sel_0 & selection_valid).count(True)
            n1=(~sel_0 & selection_valid).count(True)
            #print "Number of 0 (work):",n0
            #print "Number of 1 (test):",n1
            #print float(n0)/float(n1)*100
            if n0>0 and n1>0:
                if rfree_label: _logger.warning("FOUND >1 RFREE label in file!")
{'fraction_non_alphanumeric': 0.1104011359602414, 'fraction_numerical': 0.011596260797538753, 'mean_word_length': 3.089017900338655, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3061931', 'n_tokens_mistral': 2860, 'n_tokens_neox': 2785, 'n_words': 657}