<commit_before>// trello.js
// https://developers.trello.com/apis/webhooks
// ========
const BaseProvider = require('../util/BaseProvider');
const rpn = require('request-promise-native');
const urlMod = require('url');

class Trello extends BaseProvider {

    constructor(){
        super();
        this.baseLink = 'https://trello.com/';
        this.avatarurl = 'https://trello-avatars.s3.amazonaws.com/';
        this.defTrelloColors = {
            blue: 0x0079bf,
            yellow: 0xd9b51c,
            orange: 0xd29034,
            green: 0x519839,
            red: 0xb04632,
            purple: 0x89609e,
            pink: 0xcd5a91,
            lime: 0x4bbf6b,
            sky: 0x00aecc,
            grey: 0x838c91,
            trello: 0x026aa7,
            nocolor: 0xb6bbbf
        };
    }

    static getName() {
        return 'Trello';
    }

    async getType(){
        return this.req.body.action.type;
    }

    // Utility Functions

    static _addMemberThumbnail(avatarHash, embed){
        if(avatarHash != null && avatarHash !== 'null'){
            embed.thumbnail = {
                url: this.avatarurl + avatarHash + '/170.png'
            };
        }
    }

    static _formatLargeString(str, limit = 256){
        return (str.length > limit ? str.substring(0, limit-1) + '\u2026' : str);
    }

    static _formatMarkdownHeader(str){
        const headerRegex = new RegExp('#{1,3} ?(.*)');
        let match;
        while(headerRegex.test(str)){
            match = headerRegex.exec(str);
            str = str.replace(headerRegex, '**' + match[1] + '**');
        }
        return str;
    }

    static _formatMarkdownImage(str, embed){
        const imageRegex = new RegExp('!\\[.*\]\\((.*)\\)');
        if(imageRegex.test(str)){
            let match = imageRegex.exec(str);
            embed.image = {url: match[1]};
            str = str.replace(imageRegex, '');
        }
        return str;
    }

    static _formatMarkdownBullets(str){
        const bulletRegex = new RegExp('\\*', 'g');
        if(bulletRegex.test(str)){
            str = str.replace(bulletRegex, '\u2022');
        }
        return str;
    }

    static _formatMarkdown(str, embed){
        str = Trello._formatMarkdownBullets(str);
        str = Trello._formatMarkdownHeader(str);
        str = Trello._formatMarkdownImage(str, embed);
<commit_msg>Fixes and additions for markdown conversion.<commit_after>// trello.js
// https://developers.trello.com/apis/webhooks
// ========
const BaseProvider = require('../util/BaseProvider');
const rpn = require('request-promise-native');
const urlMod = require('url');

// Regular Expressions
const mdUL1 = new RegExp('^={3,}$');
const mdUL2 = new RegExp('^-{3,}$');
const boldRegex = new RegExp('\\*\\*([^\\\\]*)\\*\\*');
const cleanupRegex = new RegExp('__([^\\\\]*)__');
const italicRegex = new RegExp('\\*([^\\\\]*)\\*');
const imageRegex = new RegExp('!\\[.*\]\\((.*)\\)');

class Trello extends BaseProvider {

    constructor(){
        super();
        this.baseLink = 'https://trello.com/';
        this.avatarurl = 'https://trello-avatars.s3.amazonaws.com/';
        this.defTrelloColors = {
            blue: 0x0079bf,
            yellow: 0xd9b51c,
            orange: 0xd29034,
            green: 0x519839,
            red: 0xb04632,
            purple: 0x89609e,
            pink: 0xcd5a91,
            lime: 0x4bbf6b,
            sky: 0x00aecc,
            grey: 0x838c91,
            trello: 0x026aa7,
            nocolor: 0xb6bbbf
        };
    }

    static getName() {
        return 'Trello';
    }

    async getType(){
        return this.req.body.action.type;
    }

    // Utility Functions

    static _addMemberThumbnail(avatarHash, embed){
        if(avatarHash != null && avatarHash !== 'null'){
            embed.thumbnail = {
                url: this.avatarurl + avatarHash + '/170.png'
            };
        }
    }

    static _formatLargeString(str, limit = 256){
        return (str.length > limit ? str.substring(0, limit-1) + '\u2026' : str);
    }

    static _formatMarkdownHeader(str){
        let lines = str.split('\n');
        for(let i=0; i<lines.length; i++){
            let line = lines[i].trim();
            if(line.startsWith('#')){
                let start = 0;
                let end = line.length;
                let goFront = true;
                let goBack = true;
                let finished = false;
                for(let j=0; j<line.length && !finished; j++){
                    if(goFront && line.substring(j, j+1) != '#'){
                        start = j;
                        goFront = false;
                    }
                    if(goBack && line.substring(line.length-1-j, line.length-j) != '#'){
                        end = line.length-j;
                        goBack = false;
                    }
                    if(!goBack && !goFront) {
                        finished = true;
                    }
                }
                if(end-start === line.length){
                    line = '**#**';
                } else {
                    line = '**' + line.substring(start, end).trim() + '**';
                }
                lines[i] = line;
            }
            if(mdUL1.test(line) || mdUL2.test(line)){
                if(i > 0){
                    lines[i-1] = '**' + lines[i-1] + '**';
                }
                lines.splice(i, 1);
            }
        }
        return lines.join('\n');
    }

    static _formatMarkdownBold(str){
        let match;
        while(boldRegex.test(str)){
            match = boldRegex.exec(str);
            str = str.replace(boldRegex, '__' + match[1] + '__');
        }
        return str;
    }

    static _cleanupMarkdownBold(str){
        let match;
        while(cleanupRegex.test(str)){
            match = cleanupRegex.exec(str);
            str = str.replace(cleanupRegex, '**' + match[1] + '**');
        }
        return str;
    }

    static _formatMarkdownItalic(str){
        let match;
        while(italicRegex.test(str)){
            match = italicRegex.exec(str);
            str = str.replace(italicRegex, '_' + match[1] + '_');
        }
        return str;
    }

    static _formatMarkdownImage(str, embed){
        if(imageRegex.test(str)){
            let match = imageRegex.exec(str);
            embed.image = {url: match[1]};
            str = str.replace(imageRegex, '');
        }
        return str;
    }

    static _formatMarkdownBullets(str){
        const lines = str.split('\n');
        for(let i=0; i<lines.length; i++){
            let line = lines[i].trim();
            if(line.startsWith('*')){
                lines[i] = lines[i].replace('*', '\u2022');
            } else if(line.startsWith('+')){
                lines[i] = lines[i].replace('+', '\u2022');
            } else if(line.startsWith('-')){
                lines[i] = lines[i].replace('-', '\u2022');
            }
        }
        return lines.join('\n');
    }

    static _formatMarkdown(str, embed){
        str = Trello._formatMarkdownBold(str);
        str = Trello._formatMarkdownItalic(str);
        str = Trello._formatMarkdownBullets(str);
        str = Trello._cleanupMarkdownBold(str);
        str = Trello._formatMarkdownHeader(str);
        str = Trello._formatMarkdownImage(str, embed);

{'fraction_non_alphanumeric': 0.15449706564760474, 'fraction_numerical': 0.024566671216050225, 'mean_word_length': 1.6378689704823615, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '309311', 'n_tokens_mistral': 2426, 'n_tokens_neox': 2250, 'n_words': 412}