<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Module that deals with sequence properties within SCT.
Loads in yaml files containing scattering parameters and residue volumes
"""

# Copyright 2014 University College London

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#    http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
import collections
import yaml
import os
from sct.six.moves import zip

share_dir = os.path.join(__file__.rsplit(os.sep, 1)[0], 'share')
"""Path to the input data files"""
# Lists of the residues which are handled by SCT programs
polar = ['ARG', 'ASN', 'ASP', 'GLN', 'GLU', 'HIS', 'LYS', 'SER', 'THR']
"""List of polar amino acids (aa) - 3 letter codes"""
non_polar = [
    'ALA',
    'CYS',
    'GLY',
    'ILE',
    'LEU',
    'MET',
    'PHE',
    'PRO',
    'TRP',
    'TYR',
    'VAL']
"""List of non-polar aa - 3 letter codes"""
amino_acids = polar + non_polar
"""List of all aa - 3 letter codes"""
monosaccharides = ['FUC', 'GAL', 'GLC', 'MAN', 'NAG', 'NGA', 'SIA']
"""list of monosaccharides - 3 letter codes"""
all_residues = amino_acids + monosaccharides
"""List combining amino_acids and monosaccharides - 3 letter codes"""

# Create dictionary to convert one letter to three letter amino acid codes
# Trick taken from the pymol wiki
aa1 = list("ARNDCQEGHILKMFPSTWYV")
"""List of amino acid 1 letter codes"""
aa1to3 = dict(list(zip(aa1, sorted(amino_acids))))
"""Dictionary providing 1 to 3 letter aa code conversion"""

# Load parameters into module global variables

# Load scattering and mass parameters:
# bH, bD, mass, no_electron, no_exchange_H, no_exchange_peptide_H,
# solvent -[BDDO, BHHO, EHHO], constants -[avagadro]
param_file = open(os.path.join(share_dir, 'sluv_parameters.yml'), 'r')
"""Path to file containing scattering and mass parameters"""
params = yaml.load(param_file)
"""Dictionary of scattering parameters read in from file"""

# Load the different volume datasets
vol_file = open(os.path.join(share_dir, 'aa_volumes.yml'), 'r')
"""Path to file containing volumes parameters"""
res_vols = yaml.load(vol_file)
"""Dictionary of residue volumes read in from file"""

# Difference in scattering length from heavy to light water
bDH_diff = params['solvent']['BOD'] - params['solvent']['BOH']
"""Difference in scattering length for heavy and light water"""


def residue_freq_dict():
    """
    Create dictionary holding the frequencies of all residues

    @rtype:    dictionary
    @return:   Initialized dictionary for all residue types recognized by SCT
    (three letter residue codes are used as the key). Values for each residue
    type are set to 0.
    """

    res_freq = {}
    for res_name in all_residues:
        res_freq[res_name] = 0

    return res_freq


def parse_fasta(filename):
    """
    Parse fasta files. Adapted from:
    http://www.petercollingridge.co.uk/python-bioinformatics-tools/fasta-parser

    @type  filename: string
    @param filename: Path to fasta file to be read.
    @rtype:   list, dictionary
    @return:  1. Sequence identifiers found in order they appear in the file

              2. Dictionary where keys are sequence identifiers and values are
              sequences as strings (single letter amino acid codes used).
    """

    # We are going to store ordered sequence names and the sequences themselves
    order = []
    sequences = {}

    with open(filename) as f:
        for line in f:
            # Title line for sequences in fasta files start with >
            if line.startswith('>'):
                name = line[1:].rstrip('\n')
                name = name.replace('_', ' ')
                order.append(name)
                sequences[name] = ''
            elif len(order) > 0:
                # If we have seen a title but not in this line
                # add the contents of the line to the currently named sequence
                # Note: * = chain ending character in fasta so is removed
                sequences[name] += line.rstrip('\n').rstrip('*')

    return order, sequences


def fasta_res_freq(filename):
    """
    Read fasta file and return dictionary of residue frequencies.

    @type  filename: string
    @param filename: Path to fasta file to be read.
    @rtype:   dictionary
    @return:  Dictionary for all residue types recognized by SCT
              (three letter residue codes are used as the key). Values are the
              frequency with which each residue is found in input fasta file.
    """

    # Initialize a dictionary of all recognized residues (frequency = 0)
    res_freq = residue_freq_dict()

    seq_names, sequences = parse_fasta(filename)

    # Loop through all sequences found in the fasta file
    for seq_name in seq_names:
        freq_list = collections.Counter(sequences[seq_name])
        for aa in freq_list:
            # Convert one to three letter amino acid code
            res_id = aa1to3[aa]
            # Add sequence totals to overall tally for each residue
            res_freq[res_id] += freq_list[aa]
<commit_msg>Fix endline processing for Windows formatted files<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Module that deals with sequence properties within SCT.
Loads in yaml files containing scattering parameters and residue volumes
"""

# Copyright 2014 University College London

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#    http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
import collections
import yaml
import os
from sct.six.moves import zip

share_dir = os.path.join(__file__.rsplit(os.sep, 1)[0], 'share')
"""Path to the input data files"""
# Lists of the residues which are handled by SCT programs
polar = ['ARG', 'ASN', 'ASP', 'GLN', 'GLU', 'HIS', 'LYS', 'SER', 'THR']
"""List of polar amino acids (aa) - 3 letter codes"""
non_polar = [
    'ALA',
    'CYS',
    'GLY',
    'ILE',
    'LEU',
    'MET',
    'PHE',
    'PRO',
    'TRP',
    'TYR',
    'VAL']
"""List of non-polar aa - 3 letter codes"""
amino_acids = polar + non_polar
"""List of all aa - 3 letter codes"""
monosaccharides = ['FUC', 'GAL', 'GLC', 'MAN', 'NAG', 'NGA', 'SIA']
"""list of monosaccharides - 3 letter codes"""
all_residues = amino_acids + monosaccharides
"""List combining amino_acids and monosaccharides - 3 letter codes"""

# Create dictionary to convert one letter to three letter amino acid codes
# Trick taken from the pymol wiki
aa1 = list("ARNDCQEGHILKMFPSTWYV")
"""List of amino acid 1 letter codes"""
aa1to3 = dict(list(zip(aa1, sorted(amino_acids))))
"""Dictionary providing 1 to 3 letter aa code conversion"""

# Load parameters into module global variables

# Load scattering and mass parameters:
# bH, bD, mass, no_electron, no_exchange_H, no_exchange_peptide_H,
# solvent -[BDDO, BHHO, EHHO], constants -[avagadro]
param_file = open(os.path.join(share_dir, 'sluv_parameters.yml'), 'r')
"""Path to file containing scattering and mass parameters"""
params = yaml.load(param_file)
"""Dictionary of scattering parameters read in from file"""

# Load the different volume datasets
vol_file = open(os.path.join(share_dir, 'aa_volumes.yml'), 'r')
"""Path to file containing volumes parameters"""
res_vols = yaml.load(vol_file)
"""Dictionary of residue volumes read in from file"""

# Difference in scattering length from heavy to light water
bDH_diff = params['solvent']['BOD'] - params['solvent']['BOH']
"""Difference in scattering length for heavy and light water"""


def residue_freq_dict():
    """
    Create dictionary holding the frequencies of all residues

    @rtype:    dictionary
    @return:   Initialized dictionary for all residue types recognized by SCT
    (three letter residue codes are used as the key). Values for each residue
    type are set to 0.
    """

    res_freq = {}
    for res_name in all_residues:
        res_freq[res_name] = 0

    return res_freq


def parse_fasta(filename):
    """
    Parse fasta files. Adapted from:
    http://www.petercollingridge.co.uk/python-bioinformatics-tools/fasta-parser

    @type  filename: string
    @param filename: Path to fasta file to be read.
    @rtype:   list, dictionary
    @return:  1. Sequence identifiers found in order they appear in the file

              2. Dictionary where keys are sequence identifiers and values are
              sequences as strings (single letter amino acid codes used).
    """

    # We are going to store ordered sequence names and the sequences themselves
    order = []
    sequences = {}

    with open(filename) as f:
        for line in f:
            # Title line for sequences in fasta files start with >
            if line.startswith('>'):
                name = line[1:].rstrip('\r\n')
                name = name.replace('_', ' ')
                order.append(name)
                sequences[name] = ''
            elif len(order) > 0:
                # If we have seen a title but not in this line
                # add the contents of the line to the currently named sequence
                # Note: * = chain ending character in fasta so is removed
                sequences[name] += line.rstrip('\r\n').rstrip('*')

    return order, sequences


def fasta_res_freq(filename):
    """
    Read fasta file and return dictionary of residue frequencies.

    @type  filename: string
    @param filename: Path to fasta file to be read.
    @rtype:   dictionary
    @return:  Dictionary for all residue types recognized by SCT
              (three letter residue codes are used as the key). Values are the
              frequency with which each residue is found in input fasta file.
    """

    # Initialize a dictionary of all recognized residues (frequency = 0)
    res_freq = residue_freq_dict()

    seq_names, sequences = parse_fasta(filename)

    # Loop through all sequences found in the fasta file
    for seq_name in seq_names:
        freq_list = collections.Counter(sequences[seq_name])
        for aa in freq_list:
            # Convert one to three letter amino acid code
            res_id = aa1to3[aa]
            # Add sequence totals to overall tally for each residue
            res_freq[res_id] += freq_list[aa]

{'fraction_non_alphanumeric': 0.09700479594606823, 'fraction_numerical': 0.0057913311012578045, 'mean_word_length': 3.165849981153411, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7415521', 'n_tokens_mistral': 3293, 'n_tokens_neox': 3065, 'n_words': 1274}