<commit_before>// Remove label before colon.
function stripColon(text) {
    var i = text.indexOf(":");
    if (i >= 0) {
        text = text.substr(i).trim();
        if (text[1] == " ") {
            text = text.substr(1).trim();
        }
    }
    return text;
}

// Split right label
function splitLabel(text) {
    var label = "", right = "";
    var i = text.lastIndexOf(":");
    if (i >= 0) {
        right = text.substr(i + 1).trim();
        if (right) {
            text = text.substr(0, i);
            
            i = text.lastIndexOf(" ");
            if (i >= 0) {
                label = text.substr(i + 1);
                if (label) {
                    text = text.substr(0, i);
                }
            }
        }
    }
    return [text, label, right];
}

var escapeTable = {
    "<": "&lt",
    "&": "&amp;"
};

// escape character to html entity
function escape(s) {
    var s2 = "", i;
    
    for (i = 0; i < s.length; i++) {
        if (s.charCodeAt(i) & 0x80) {
            // ignore cjk
            s2 += s.substr(i, 2);
            i++;
            continue;
        } else if (escapeTable[s[i]]) {
            s2 += escapeTable[s[i]];
        } else {
            s2 += s[i];
        }
    }
    
    return s2;
}

function Span(f, b, l) {
    this.f = f;
    this.b = b;
    this.l = l;
    this.flash = false;
    this.halfStart = false;
    this.halfEnd = false;
    this.text = "";
    this.i = 0;
}

Span.prototype.toString = function() {
    if (!this.text) {
        return "";
    }
    
    var cls = "f" + this.f + " b" + this.b,
        w = this.text.length;
        
    if (this.l) {
        cls += " l";
    }
    if (this.flash) {
        cls += " flash";
    }
    if (this.halfStart) {
        cls += " half-start";
        w--;
    }
    if (this.halfEnd) {
        cls += " half-end";
        w--;
    }
    cls += " w" + w;
    return "<span class='" + cls + "'>" + escape(this.text) + "</span>";
};

Span.prototype.copy = function() {
    var span = new Span(this.f, this.b, this.l);
    span.flash = this.flash;
    return span;
};

Span.prototype.reset = function() {
    this.f = 7;
    this.b = 0;
    this.l = false;
    this.flash = false;
}

// create head line
function makeHead(lLabel, lText, rLabel, rText) {
    if (!lText) {
        return "";
    }
    
    var fillSpace = 78, result = "<div class='line'>";
    
    fillSpace -= 2 + lLabel.length + 1 + lText.length;
    
    if (rText) {
        fillSpace -= 2 + rLabel.length + 2 + rText.length;
    }
    
    if (fillSpace < 0) {
        fillSpace = 0;
    }
    
    var span = new Span(4, 7, false);
    
    span.text = " " + lLabel + " ";
    
    result += span.toString();
    
    span.f = 7;
    span.b = 4;
    span.text = " " + lText + " ".repeat(fillSpace);
    
    result += span.toString();
    
    // result += makeSpan({f:4,b:7,text:" "+lLabel+" "}) + makeSpan({f:7,b:4,text:" "+lText+" ".repeat(fillSpace)});
    
    if (rText) {
        span.f = 4;
        span.b = 7;
        span.text = " " + rLabel + " ";
        
        result += span.toString();
        
        span.f = 7;
        span.b = 4;
        span.text = " " + rText + " ";
        
        result += span.toString();
        // result += makeSpan({f:4,b:7,text:" "+rLabel+" "}) + makeSpan({f:7,b:4,text:" "+rText+" "});
    }
    
    result += "</div>";
    
    return result;
}

// extract text to color
function extractColor(text, i, color) {
    var re = /\033\[([\d;]*)m/g;
    re.lastIndex = i;
    var matches = [],
        match;
    do {
        match = re.exec(text);
        if (!match || match.index != i) {
            break;
        }
        matches.push(match);
        i = re.lastIndex;
    } while (text[re.lastIndex] == "\x1b");
    
    if (!matches.length) {
        return null;
    }
    
    var tokens = matches.map(function(match){
        return match[1].split(";");
    });
    tokens = Array.prototype.concat.call([], tokens);
    
    var span = color.copy();
    span.i = re.lastIndex;
    
    var tokens = match[1].split(";");
    var code;
    
    for (i = 0; i < tokens.length; i++) {
        code = +tokens[i];
        if (code == 0) {
            span.reset();
        } else if (code == 1) {
            span.l = true;
        } else if (code == 5) {
            span.flash = true;
        } else if (code == 7) {
            var t = span.f;
            span.f = span.b;
            span.b = t;
        } else if (code < 40) {
            span.f = code - 30;
        } else if (code < 50) {
            span.b = code - 40;
        }
    }
    return span;
}<commit_msg>Fix escape table and ansi sequence bug<commit_after>// Remove label before colon.
function stripColon(text) {
    var i = text.indexOf(":");
    if (i >= 0) {
        text = text.substr(i).trim();
        if (text[1] == " ") {
            text = text.substr(1).trim();
        }
    }
    return text;
}

// Split right label
function splitLabel(text) {
    var label = "", right = "";
    var i = text.lastIndexOf(":");
    if (i >= 0) {
        right = text.substr(i + 1).trim();
        if (right) {
            text = text.substr(0, i);
            
            i = text.lastIndexOf(" ");
            if (i >= 0) {
                label = text.substr(i + 1);
                if (label) {
                    text = text.substr(0, i);
                }
            }
        }
    }
    return [text, label, right];
}

var escapeTable = {
    "<": "&lt;",
    "&": "&amp;"
};

function escapeRepl(match) {
    return escapeTable[match];
}

// escape character to html entity
function escape(s) {
    // this should be safe since there is no CJK contains "&", "<"
    return s.replace(/<|&/g, escapeRepl);
}

function Span(f, b, l) {
    this.f = f;
    this.b = b;
    this.l = l;
    this.flash = false;
    this.halfStart = false;
    this.halfEnd = false;
    this.text = "";
    this.i = 0;
}

Span.prototype.toString = function() {
    if (!this.text) {
        return "";
    }
    
    var cls = "f" + this.f + " b" + this.b,
        w = this.text.length;
        
    if (this.l) {
        cls += " l";
    }
    if (this.flash) {
        cls += " flash";
    }
    if (this.halfStart) {
        cls += " half-start";
        w--;
    }
    if (this.halfEnd) {
        cls += " half-end";
        w--;
    }
    cls += " w" + w;
    return "<span class='" + cls + "'>" + escape(this.text) + "</span>";
};

Span.prototype.copy = function() {
    var span = new Span(this.f, this.b, this.l);
    span.flash = this.flash;
    return span;
};

Span.prototype.reset = function() {
    this.f = 7;
    this.b = 0;
    this.l = false;
    this.flash = false;
}

// create head line
function makeHead(lLabel, lText, rLabel, rText) {
    if (!lText) {
        return "";
    }
    
    var fillSpace = 78, result = "<div class='line'>";
    
    fillSpace -= 2 + lLabel.length + 1 + lText.length;
    
    if (rText) {
        fillSpace -= 2 + rLabel.length + 2 + rText.length;
    }
    
    if (fillSpace < 0) {
        fillSpace = 0;
    }
    
    var span = new Span(4, 7, false);
    
    span.text = " " + lLabel + " ";
    
    result += span.toString();
    
    span.f = 7;
    span.b = 4;
    span.text = " " + lText + " ".repeat(fillSpace);
    
    result += span.toString();
    
    // result += makeSpan({f:4,b:7,text:" "+lLabel+" "}) + makeSpan({f:7,b:4,text:" "+lText+" ".repeat(fillSpace)});
    
    if (rText) {
        span.f = 4;
        span.b = 7;
        span.text = " " + rLabel + " ";
        
        result += span.toString();
        
        span.f = 7;
        span.b = 4;
        span.text = " " + rText + " ";
        
        result += span.toString();
        // result += makeSpan({f:4,b:7,text:" "+rLabel+" "}) + makeSpan({f:7,b:4,text:" "+rText+" "});
    }
    
    result += "</div>";
    
    return result;
}

// extract text to color
function extractColor(text, i, color) {
    var re = /\033\[([\d;]*)m/g;
    re.lastIndex = i;
    var matches = [],
        match;
    do {
        match = re.exec(text);
        if (!match || match.index != i) {
            break;
        }
        matches.push(match);
        i = re.lastIndex;
    } while (text[re.lastIndex] == "\x1b");
    
    if (!matches.length) {
        return null;
    }
    
    var tokens = matches.map(function(match){
        return match[1].split(";");
    });
    tokens = Array.prototype.concat.call([], tokens);
    
    var span = color.copy();
    span.i = re.lastIndex;
    
    var code;
    
    for (i = 0; i < tokens.length; i++) {
        code = +tokens[i];
        if (code == 0) {
            span.reset();
        } else if (code == 1) {
            span.l = true;
        } else if (code == 5) {
            span.flash = true;
        } else if (code == 7) {
            var t = span.f;
            span.f = span.b;
            span.b = t;
        } else if (code < 40) {
            span.f = code - 30;
        } else if (code < 50) {
            span.b = code - 40;
        }
    }
    return span;
}
{'fraction_non_alphanumeric': 0.17827574787504138, 'fraction_numerical': 0.013136107738160945, 'mean_word_length': 1.4666485161992922, 'pattern_counts': {'":': 8, '<': 24, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '564040', 'n_tokens_mistral': 3241, 'n_tokens_neox': 3160, 'n_words': 731}