<commit_before>import * as $ from 'jquery';
import * as moment from 'moment';

import {
    Article,
} from '..';
import {
    clearStyles,
} from '../util';


// 유투브 동영상(iframe)이 포함된 기사: http://news.kmib.co.kr/article/view.asp?arcid=0012123329&code=61161111&sid1=spo
// 따라서 단순하게 iframe을 날려버리면 기사 내용이 날라가는 문제가 있을 수 있습니다.
export const cleanup = () => $('#scrollDiv').remove();

export function parse(): Article {
    const t = $('.nwsti_btm .date .t11');
    const format = 'YYYY-MM-DD HH:mm';
    return {
        title: $('.nwsti h3').text(),
        content: (() => {
            const article = $('#article')[0].cloneNode(true) as HTMLElement;
            $('.best_nw', article).remove();
            return clearStyles(article).innerHTML;<commit_msg>Clean up for kmib.co.kr (Fix #309)<commit_after>import * as $ from 'jquery';
import * as moment from 'moment';

import {
    Article,
} from '..';
import {
    clearStyles,
    waitElement,
} from '../util';

export const readyToParse = () => waitElement('.best_nw');

// 유투브 동영상(iframe)이 포함된 기사: http://news.kmib.co.kr/article/view.asp?arcid=0012123329&code=61161111&sid1=spo
// 따라서 단순하게 iframe을 날려버리면 기사 내용이 날라가는 문제가 있을 수 있습니다.
export const cleanup = () => $('#scrollDiv, body>script, body>iframe, #NeoInteractiveScreenViewLayer, body>div:not([id])').remove();

export function parse(): Article {
    const t = $('.nwsti_btm .date .t11');
    const format = 'YYYY-MM-DD HH:mm';
    return {
        title: $('.nwsti h3').text(),
        content: (() => {
            const article = $('#articleBody')[0].cloneNode(true) as HTMLElement;
            $('a[href^="http://news.kmib.co.kr/article/fbchoice/index.asp"]', article).remove();
            $('a[href^="http://www.kmib.co.kr/photo/index.asp"]', article).remove();
            $('a[href^="http://www.kmib.co.kr/vod/index.asp"]', article).remove();
            const nodes = article.childNodes;
            const blankRegexp = /^\s*$/;
            for (var i = nodes.length - 1; i >= 0; i--) {
                if (nodes[i].nodeType === 3) {
                    const text = nodes[i].nodeValue as string;
                    if (blankRegexp.test(text)) {
                        article.removeChild(nodes[i]);
                    } else {
                        break;
                    }
                } else if (nodes[i].nodeType === 1 && (nodes[i].nodeName === 'BR' || nodes[i].nodeName === 'FONT')) {
                    article.removeChild(nodes[i]);
                } else {
                    break;
                }
            }
            return clearStyles(article).innerHTML;
{'fraction_non_alphanumeric': 0.17717140661029976, 'fraction_numerical': 0.020368946963873945, 'mean_word_length': 2.193865030674847, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4520045', 'n_tokens_mistral': 948, 'n_tokens_neox': 951, 'n_words': 181}