<commit_before>import fs from 'fs'
export function parseBoardList(boardList) {
    // console.log(JSON.stringify(boardList))
    try {
        boardList.data.children
    } catch (e) {
        log.error(`Bad reddit data returned => ${e}`)
    }

    const boards = boardList.data.children.map(board => board.data) // thanks, reddit!

    return boards.map(({url, title}) => {
        let boardID = url.split('/').slice(-2)[0]
        return {
            boardID,
            description: `/${boardID}/ - ${title}`
        }
    });
}<commit_msg>Update reddit boardlist parser<commit_after>import fs from 'fs'

export function parseBoardList(boardList) {
    // console.log(JSON.stringify(boardList))
    try {
        boardList.data.children
    } catch (e) {
        return log.error(`Bad reddit data returned: ${e}`)
    }

    const boards = boardList.data.children.map(board => board.data) // thanks, reddit!

    return boards.map( board => {
        // Essentials
        const { url, title, public_description } = board
        let boardID = url.split('/').slice(-2)[0]
        
        // Board info
        const { subscribers, over18, icon_img } = board

        return {
            url, 
            boardID,
            title,
            short_desc: `/${boardID}/ - ${title}`,
            description: public_description,
            info: {
                subscribers, 
                NSFW: over18, 
                icon_img                
            }

        }
    });
}

{'fraction_non_alphanumeric': 0.1318607764390897, 'fraction_numerical': 0.00535475234270415, 'mean_word_length': 1.7481617647058822, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6522284', 'n_tokens_mistral': 434, 'n_tokens_neox': 424, 'n_words': 94}