<commit_before>COURSE_URL = "/course/977288/korean-grammar-in-use-11/"
CARD_COLUMNS = ("col_a", "col_b")
    
import codecs, sys
import requests
from bs4 import BeautifulSoup

sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())

def get_soup(url):
    # TODO: it works actually w/o cookies:
    res = requests.get(
        url if url.strip().startswith("http") else "http://www.memrise.com" + url)
    soup = BeautifulSoup(res.text, "html.parser", from_encoding="utf-8")
    return soup

    
def levels(*, course_url : str):
    """
    :course_url:   course URL
    """
    soup = get_soup(course_url)
    #levels = soup.find(lambda tag: tag.name == "div" and "levels" in tag.attrs.get("class"))
    levels = soup.find_all("a", class_="level")
    
    for l in levels:
        yield l.attrs.get("href")


def cards(*, level_url : str):
    """
    :level_url:   level URL
    """
    soup = get_soup(level_url)
    
    for thing in soup.find_all(lambda tag: tag.has_attr("data-thing-id")):
        
        try:
            cols = (thing.find("div", class_=col_name).find("div", class_="text").text
                for col_name in CARD_COLUMNS)
        except:
            continue
            
        yield cols

            
def dump_course(*, course_url : str):
    """
    :course_url:   course URL
    """
    for level_url in levels(course_url=course_url):
        print("***", level_url)
        for card in cards(level_url=level_url):
            print('\t'.join(card))
        
if __name__ == "__main__":
    course_url = COURSE_URL if len(sys.argv) < 2 else sys.argv[1]
    dump_course(course_url=course_url)
    
    <commit_msg>quickfix: None for found nodes<commit_after>COURSE_URL = "/course/977288/korean-grammar-in-use-11/"
CARD_COLUMNS = ("col_a", "col_b")
    
import codecs, sys
import requests
from bs4 import BeautifulSoup

sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())

def get_soup(url):
    # TODO: it works actually w/o cookies:
    res = requests.get(
        url if url.strip().startswith("http") else "http://www.memrise.com" + url)
    soup = BeautifulSoup(res.text, "html.parser", from_encoding="utf-8")
    return soup

    
def levels(*, course_url : str):
    """
    :course_url:   course URL
    """
    soup = get_soup(course_url)
    #levels = soup.find(lambda tag: tag.name == "div" and "levels" in tag.attrs.get("class"))
    levels = soup.find_all("a", class_="level")
    
    for l in levels:
        yield l.attrs.get("href")


def cards(*, level_url : str):
    """
    :level_url:   level URL
    """
    def get_text(value):
        return '' if value is None else value.text
        
    soup = get_soup(level_url)
    
    for thing in soup.find_all(lambda tag: tag.has_attr("data-thing-id")):
        
        try:
            cols = (get_text(thing.find("div", class_=col_name).find("div", class_="text"))
                for col_name in CARD_COLUMNS)
        except:
            continue
            
        yield cols

            
def dump_course(*, course_url : str):
    """
    :course_url:   course URL
    """
    for level_url in levels(course_url=course_url):
        print("***", level_url)
        for card in cards(level_url=level_url):
            print('\t'.join(card))
        
if __name__ == "__main__":
    course_url = COURSE_URL if len(sys.argv) < 2 else sys.argv[1]
    dump_course(course_url=course_url)
    
    
{'fraction_non_alphanumeric': 0.14005273952534428, 'fraction_numerical': 0.007617931438617052, 'mean_word_length': 2.497950819672131, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '555589', 'n_tokens_mistral': 1191, 'n_tokens_neox': 1172, 'n_words': 257}