<commit_before>/**
 * Copyright (c) 2009-2019, <NAME> <http://dx.doi.org/10.1111/j.1755-0998.2009.02732.x>
 * Translated to HaXe by <NAME>
 *
 * Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is
 * hereby granted, provided that the above copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE
 * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
 * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

import haxe.ds.StringMap;
import haxe.ds.ArraySort;

/**
 * This class principally contains the content of SeqPhase1.pl. Since Haxe expects you to have classes
 * but the original python script doesn't contain any, there are some changes in structure and code
 * ordering. These changes are also needed because the code needs to get converted into JavaScript code
 * later on and reading / storing files works asynchronly in JavaScript (which is quite different to the
 * reading mechanisms of Perl).
 */
class FastaAlignmentParser {

    private static var authorizedCharacters:StringMap<Bool> = [
        'A' => true, 'T' => true, 'G' => true, 'C' => true, 'N' => true, '-' => true, '?' => true,
        'R' => false, 'Y' => false, 'M' => false, 'K' => false, 'W' => false, 'S' => false
    ];

    private var fastaContent:Array<Entry>;
    private var seqLength:Int;

    public function new(fileContent:Null<String>, allChecks:Bool, allSort:Bool, fileNr:Int) {
        fastaContent = new Array<Entry>();
        seqLength = -1;
        // checks
        if (fileContent == null) return;
        if (stripString(fileContent) == "") {
            SeqPhase1Result.instance().addErr("Empty file!", fileNr);
            return;
        }
        if (!startsWith(fileContent, ">") && !startsWith(fileContent, ";")) {
            SeqPhase1Result.instance().addErr("File does not seem to be a fasta file!", fileNr);
            return;
        }
        // parse data
        var lines:Array<String> = fileContent.split("\n");
        if (lines.length == 0) {
            SeqPhase1Result.instance().addErr("Not a fasta but an empty file!", fileNr);
            return;
        } else if(lines.length == 1) {
            SeqPhase1Result.instance().addErr("Only 1 line detected! Please check data format (opening the alignment in MEGA (http://www.megasoftware.net/) and exporting it as FASTA again may solve the problem; alternatively, there may be a problem with end-of-line characters - see http://en.wikipedia.org/wiki/Newline for details!", fileNr);
            return;
        }
        var entryMap:StringMap<Entry> = new StringMap<Entry>();
        var current:Null<Entry> = null;
        var lineNo:Int = 0;
        for (line in lines) {
            lineNo++;
            line = stripString(line);
            if (startsWith(line, ";")) { // comment
                continue;
            } else if (startsWith(line, ">")) { // header
                var indName:String = stripString(line.substr(1));
                if(entryMap.exists(indName)) {
                    SeqPhase1Result.instance().addErr("Repeat of name " + indName
                        + " encountered in alignment (line " +  lineNo + ", line "
                        + entryMap.get(indName).getLineNo() + ")", fileNr);
                }
                current = new Entry(lineNo, indName);
                entryMap.set(indName, current);
                if(indName.length == 0) {
                    SeqPhase1Result.instance().addErr("Missing sequence name, line " + lineNo, fileNr);
                }
            } else {
                for(i in 0...line.length) {
                    var char:String = line.charAt(i).toUpperCase();
                    if(!authorizedCharacters.exists(char)) {
                        SeqPhase1Result.instance().addErr("Unknown character state " + char + " in "
                        + current.getName() + ", line " + lineNo, fileNr);
                    } else if(allChecks && authorizedCharacters.get(char) == false) {
                        SeqPhase1Result.instance().addErr("Unallowed state " + char + " in "
                        + current.getName() + ", line " + lineNo, fileNr);
                    }
                }<commit_msg>Replace spaces by underscore<commit_after>/**
 * Copyright (c) 2009-2019, <NAME> <http://dx.doi.org/10.1111/j.1755-0998.2009.02732.x>
 * Translated to HaXe by <NAME>
 *
 * Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is
 * hereby granted, provided that the above copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE
 * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
 * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

import haxe.ds.StringMap;
import haxe.ds.ArraySort;

/**
 * This class principally contains the content of SeqPhase1.pl. Since Haxe expects you to have classes
 * but the original python script doesn't contain any, there are some changes in structure and code
 * ordering. These changes are also needed because the code needs to get converted into JavaScript code
 * later on and reading / storing files works asynchronly in JavaScript (which is quite different to the
 * reading mechanisms of Perl).
 */
class FastaAlignmentParser {

    private static var authorizedCharacters:StringMap<Bool> = [
        'A' => true, 'T' => true, 'G' => true, 'C' => true, 'N' => true, '-' => true, '?' => true,
        'R' => false, 'Y' => false, 'M' => false, 'K' => false, 'W' => false, 'S' => false
    ];

    private var fastaContent:Array<Entry>;
    private var seqLength:Int;

    public function new(fileContent:Null<String>, allChecks:Bool, allSort:Bool, fileNr:Int) {
        fastaContent = new Array<Entry>();
        seqLength = -1;
        // checks
        if (fileContent == null) return;
        if (stripString(fileContent) == "") {
            SeqPhase1Result.instance().addErr("Empty file!", fileNr);
            return;
        }
        if (!startsWith(fileContent, ">") && !startsWith(fileContent, ";")) {
            SeqPhase1Result.instance().addErr("File does not seem to be a fasta file!", fileNr);
            return;
        }
        // parse data
        var lines:Array<String> = fileContent.split("\n");
        if (lines.length == 0) {
            SeqPhase1Result.instance().addErr("Not a fasta but an empty file!", fileNr);
            return;
        } else if(lines.length == 1) {
            SeqPhase1Result.instance().addErr("Only 1 line detected! Please check data format (opening the alignment in MEGA (http://www.megasoftware.net/) and exporting it as FASTA again may solve the problem; alternatively, there may be a problem with end-of-line characters - see http://en.wikipedia.org/wiki/Newline for details!", fileNr);
            return;
        }
        var entryMap:StringMap<Entry> = new StringMap<Entry>();
        var current:Null<Entry> = null;
        var lineNo:Int = 0;
        var underscoreWarningOutputted:Bool = false;
        for (line in lines) {
            lineNo++;
            line = stripString(line);
            if (startsWith(line, ";")) { // comment
                continue;
            } else if (startsWith(line, ">")) { // header
                var indName:String = stripString(line.substr(1));
                var indNameCor:String = StringTools.replace(indName, " ", "_");
                if(indName != indNameCor) { // PHASE does not accept spaces in individual names. So replace by underscore
                    if(!underscoreWarningOutputted) {
                        SeqPhase1Result.instance().addWrn("Warning: PHASE does not accept spaces in individual names. These spaces got replaced by underscore characters.", fileNr);
                        underscoreWarningOutputted = true;
                    }
                    indName = indNameCor;
                }
                if(entryMap.exists(indName)) {
                    SeqPhase1Result.instance().addErr("Repeat of name " + indName
                        + " encountered in alignment (line " +  lineNo + ", line "
                        + entryMap.get(indName).getLineNo() + ")", fileNr);
                }
                current = new Entry(lineNo, indName);
                entryMap.set(indName, current);
                if(indName.length == 0) {
                    SeqPhase1Result.instance().addErr("Missing sequence name, line " + lineNo, fileNr);
                }
            } else {
                for(i in 0...line.length) {
                    var char:String = line.charAt(i).toUpperCase();
                    if(!authorizedCharacters.exists(char)) {
                        SeqPhase1Result.instance().addErr("Unknown character state " + char + " in "
                        + current.getName() + ", line " + lineNo, fileNr);
                    } else if(allChecks && authorizedCharacters.get(char) == false) {
                        SeqPhase1Result.instance().addErr("Unallowed state " + char + " in "
                        + current.getName() + ", line " + lineNo, fileNr);
                    }
                }
{'fraction_non_alphanumeric': 0.11174089068825911, 'fraction_numerical': 0.009817813765182187, 'mean_word_length': 2.262132717068339, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '393821', 'n_tokens_mistral': 2775, 'n_tokens_neox': 2581, 'n_words': 861}