<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
import re
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities


def medium2markdown(url):
    # driver = webdriver.Chrome(driver_path)
    driver = webdriver.Remote(command_executor='http://127.0.0.1:4444/wd/hub',
                              desired_capabilities=DesiredCapabilities.HTMLUNITWITHJS)
    # desired_capabilities=DesiredCapabilities.CHROME)
    driver.get(url)
    # assert 'android' in driver.title
    content_tag = driver.find_element_by_class_name('postArticle-content')
    content_html = BeautifulSoup(content_tag.get_attribute('innerHTML'), "html.parser") \
        .find_all()
    # print(content_html)
    # output_file(content_html)

    for i in range(0, len(content_html)):
        tag = content_html[i]
        # text = strip_space(tag.text)
        # print(i, tag.name, '[NONE]' if len(text) == 0 else text)
        md = to_markdown(tag)
        # if md is not None:
        #     print(i, md)
    driver.quit()


def output_file(content_html):
    with open('content.html', 'w') as f:
        f.writelines(content_html.prettify())


def strip_space(text, trim_space=True):
    text = re.sub(r'\s+', ' ', text)
    if trim_space:
        return text.strip()
    else:
        return text


def to_markdown(medium_tag):
    text = strip_space(medium_tag.text)
    if medium_tag.name == 'h3':
        return '## {}'.format(text)
    elif medium_tag.name == 'h4':
        return '### {}'.format(text)
    elif medium_tag.name == 'p': # text paragraph
        ## find style, link inside a paragraph
        plain_text = ''
        for child in medium_tag.children:
            if child.name is None:
                if len(strip_space(child.string)) > 0:
                    plain_text += strip_space(child.string)
            else:
                content = strip_space(child.text)
                if child.name == 'strong':
                    plain_text += " **{0}** ".format(content)
                elif child.name == 'em':
                    plain_text += " _{0}_ ".format(content)
                elif child.name == 'a':
                    plain_text += " [{0}]({1}) ".format(content, child['href'])
                elif child.name == 'code' or child.name == '':
                    plain_text += " `{0}` ".format(content)
                    # print(child.name, child)
        return plain_text
    elif medium_tag.name == 'figure': # image and comment
        img_tag = medium_tag.find('img', class_='progressiveMedia-image')
        if img_tag is not None and img_tag.has_attr('data-src'):
            figcaption_tag = medium_tag.find('figcaption')
            if figcaption_tag is not None:
                return '![{0}]({1})'.format(strip_space(figcaption_tag.text),
                                            img_tag['data-src'])
            else:
                return '![]({})'.format(img_tag['data-src'])
    elif medium_tag.name == 'blockquote': # quote
        return '> {}'.format(strip_space(medium_tag.text))
    elif medium_tag.name == 'ul':
        li_tags = medium_tag.find_all('li')
        # use newline to join several item lines
        list_text = '\n'.join(['* {}'.format(strip_space(li.text)) for li in li_tags])
        return list_text
    elif medium_tag.name == 'ol':
        li_tags = medium_tag.find_all('li')
        # use newline to join several item lines
        list_text = '\n'.join(['{0}. {1}'.format(i + 1, strip_space(li_tags[i].text))
                               for i in range(len(li_tags))])
        return list_text
    elif medium_tag.name == 'pre': # code snippet (not inline code or embed code)
        # TODO: 1. trim all html tags from original root tag
        # FIXME: this current one can't solve the code tab or original intent format
        xs = medium_tag.prettify().split('<br/>')

        for i in range(len(xs)):
            t = BeautifulSoup(xs[i], "html.parser")
            code = re.sub(r'\r\n(\s{10})', '', t.text).replace('\n', '')
            print(i, code)
        # print(repr(medium_tag.prettify()))
        # print(medium_tag.text)
        code_snippet = ''
        for child in medium_tag.children:
            code = strip_space(str(child.string))
            if child.name is None: # plain text
                code_snippet += code
                pass
            elif child.name == 'br': # new line
                code_snippet += '\n'
                pass
            elif child.name == 'strong': # bold
                code_snippet += '{} '.format(code)
                pass
            elif child.name == 'em': # comment
                pass

            # print(child.name, strip_space(str(child.string)))
        # return code_snippet
        # print(code_snippet)
    elif medium_tag.name == 'hr':
        return '\n----\n'
    elif medium_tag.name == 'iframe':
        # gist, video, github, link...etc.
        # print(medium_tag.prettify())
        pass
    elif medium_tag.name == 'a':
        # print(medium_tag.prettify())
        pass
    else:
        return None

<commit_msg>Finish code block parsing from me2md<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
import re
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities


def medium2markdown(url):
    # driver = webdriver.Chrome(driver_path)
    driver = webdriver.Remote(command_executor='http://127.0.0.1:4444/wd/hub',
                              desired_capabilities=DesiredCapabilities.HTMLUNITWITHJS)
    # desired_capabilities=DesiredCapabilities.CHROME)
    driver.get(url)
    # assert 'android' in driver.title
    content_tag = driver.find_element_by_class_name('postArticle-content')
    content_html = BeautifulSoup(content_tag.get_attribute('innerHTML'), "html.parser") \
        .find_all()
    # print(content_html)
    # output_file(content_html)

    for i in range(0, len(content_html)):
        tag = content_html[i]
        # text = strip_space(tag.text)
        # print(i, tag.name, '[NONE]' if len(text) == 0 else text)
        md = to_markdown(tag)
        # if md is not None:
        #     print(i, md)
    driver.quit()


def output_file(content_html):
    with open('content.html', 'w') as f:
        f.writelines(content_html.prettify())


def strip_space(text, trim_space=True):
    text = re.sub(r'\s+', ' ', text)
    if trim_space:
        return text.strip()
    else:
        return text


def to_markdown(medium_tag):
    text = strip_space(medium_tag.text)
    if medium_tag.name == 'h3':
        return '## {}'.format(text)
    elif medium_tag.name == 'h4':
        return '### {}'.format(text)
    elif medium_tag.name == 'p': # text paragraph
        ## find style, link inside a paragraph
        plain_text = ''
        for child in medium_tag.children:
            if child.name is None:
                if len(strip_space(child.string)) > 0:
                    plain_text += strip_space(child.string)
            else:
                content = strip_space(child.text)
                if child.name == 'strong':
                    plain_text += " **{0}** ".format(content)
                elif child.name == 'em':
                    plain_text += " _{0}_ ".format(content)
                elif child.name == 'a':
                    plain_text += " [{0}]({1}) ".format(content, child['href'])
                elif child.name == 'code' or child.name == '':
                    plain_text += " `{0}` ".format(content)
                    # print(child.name, child)
        return plain_text
    elif medium_tag.name == 'figure': # image and comment
        img_tag = medium_tag.find('img', class_='progressiveMedia-image')
        if img_tag is not None and img_tag.has_attr('data-src'):
            figcaption_tag = medium_tag.find('figcaption')
            if figcaption_tag is not None:
                return '![{0}]({1})'.format(strip_space(figcaption_tag.text),
                                            img_tag['data-src'])
            else:
                return '![]({})'.format(img_tag['data-src'])
    elif medium_tag.name == 'blockquote': # quote
        return '> {}'.format(strip_space(medium_tag.text))
    elif medium_tag.name == 'ul':
        li_tags = medium_tag.find_all('li')
        # use newline to join several item lines
        list_text = '\n'.join(['* {}'.format(strip_space(li.text)) for li in li_tags])
        return list_text
    elif medium_tag.name == 'ol':
        li_tags = medium_tag.find_all('li')
        # use newline to join several item lines
        list_text = '\n'.join(['{0}. {1}'.format(i + 1, strip_space(li_tags[i].text))
                               for i in range(len(li_tags))])
        return list_text
    elif medium_tag.name == 'pre': # code block (not inline code or embed code)
        code_block = ''
        code_tags = medium_tag.prettify().split('<br/>')
        for i in range(len(code_tags)):
            t = BeautifulSoup(code_tags[i], "html.parser")
            code = re.sub(r'\r\n(\s{10})', '', t.text).replace('\n', '')
            code_block += '{}\n'.format(code)
            # print(i, code)
        # print(code_block)
        return code_block
    elif medium_tag.name == 'hr':
        return '\n----\n'
    elif medium_tag.name == 'iframe':
        # gist, video, github, link...etc.
        # print(medium_tag.prettify())
        pass
    elif medium_tag.name == 'a':
        # print(medium_tag.prettify())
        pass
    else:
        return None


{'fraction_non_alphanumeric': 0.13645833333333332, 'fraction_numerical': 0.006458333333333333, 'mean_word_length': 2.106114526043352, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2022958', 'n_tokens_mistral': 2969, 'n_tokens_neox': 2877, 'n_words': 647}