<commit_before>
from bs4 import BeautifulSoup
import bs4
import requests
import sys
import os
import json
import stat
import inspect
from random import randint
import glob

def dbg_printf(format, *args):
    """
    C-Style function that writes debugging output to the terminal. If debug flag is off
    this function does not print anything

    :param format: The format string
    :param args: Arguments
    :return: None
    """
    global debug_flag

    # Do not print anything annoying
    if debug_flag is False:
        return

    frame = inspect.currentframe()
    prev_frame = frame.f_back
    code = prev_frame.f_code
    prev_name = code.co_name

    # Make it more human readable by replacing the name with
    # an easy to understand one
    if prev_name == "<module>":
        prev_name = "[Top Level Module]"
    else:
        prev_name += "()"

    # Write the prologue of debugging information
    sys.stderr.write("%-28s: " % (prev_name,))

    format = format % tuple(args)
    sys.stderr.write(format)

    # So we do not need to worry about new lines
    sys.stderr.write('\n')

    return

def get_webpage(word):
    """
    This function returns a string which is the webpage of a given word
    Returns None if the return code is not HTTP status 200 which means an error
    happened
    
    :return: str/None 
    """
    url = "http://dict.youdao.com/search?le=eng&q=%s&keyfrom=dict2.index" % (word, )
    r = requests.get(url)
    if r.status_code != 200:
        dbg_printf("Error executing HTTP request to %s; return code %d",
                   url,
                   r.status_code)

        return None

    return r.text

def parse_webpage(s):
    """
    Given the text of the webpage, parse it as an instance of a beautiful soup,
    using the default parser and encoding
    
    :param s: The text of the webpage 
    :return: beautiful soup object
    """
    return BeautifulSoup(s, 'html.parser')

def get_collins_dict(tree):
    """
    This function returns results by collins dictionary, given the beautiful soup
    tree object
    
    The return value is defined as follows: 
      
    {
      "word": "The word"
      "phonetic": "The pronunciation"
      "frequency": 4,    // This is always a number between 1 and 5, or -1 to indicate unknown
      "meanings": [
        {
          "category": "n. v. adj. adv. , etc.",
          "text": "Meaning of the word",
          "examples": [
            {
              "text": "Text of the example, with <b></b> being the keyword"
              "translation": "Translation of the text"
            },
          ]
        }, ...
      ]
    }
    
    :param tree: The beautiful soup tree
    :return: A list of the dict as specified as above, or None if fails
    """
    collins_result = tree.find(id="collinsResult")
    if isinstance(collins_result, bs4.element.Tag) is False:
        dbg_printf("Did not find id='collinsResult'")
        # it is not a valid div object (usually None if the tag does
        # not exist)
        return None
    elif collins_result.name != "div":
        dbg_printf("id='collinsResult' is not a div tag")
        # It is a valid tag, but the name is not div
        # This should be strange, though
        return None

    # This list contains all meanings of the word, each with a pronunciation
    top_level_list = collins_result.select("div.wt-container")
    ret_list = []
    # We set this to be the first word
    actual_key = None
    for tree in top_level_list:
        # We append this into a list
        ret = {}

        # This <h4> contains the main word, the pronunciation
        h4 = tree.find("h4")
        if h4 is None:
            dbg_printf("Did not find <h4>")
            return None

        span_list = h4.find_all("span")
        if len(span_list) < 1:
            dbg_printf("Did not find <span> under <h4>")
            return None

        # This is the word we are looking for
        ret["word"] = span_list[0].text

        # This contains the phonetic
        em = h4.find("em")
        if em is None:
            dbg_printf("Did not find <em> under <h4>")
            return None

        # Save the phonetic (note: this is not ASCII)
        ret["phonetic"] = em.text
        # Initialize the meanings list
        ret["meanings"] = []

        # Get the frequency span; If no such element just set it to -1
        # which means the freq is invalid
        freq_span = h4.select("span.star")
        if len(freq_span) == 0:
            ret["frequency"] = -1
        else:
            freq = -1
            star_attr = freq_span[0].attrs["class"]
            if "star1" in star_attr:
                freq = 1
            elif "star2" in star_attr:
                freq = 2
            elif "star3" in star_attr:
                freq = 3
            elif "star4" in star_attr:
                freq = 4
            elif "star5" in star_attr:
                freq = 5

            ret["frequency"] = freq

        # This is all meanings
        li_list = tree.find_all("li")
        li_count = -1
        for li in li_list:
            li_count += 1

            # Just add stuff into this dict object
            d = {}

            # find main div and example div list
            main_div_list = li.select("div.collinsMajorTrans")
            if len(main_div_list) == 0:
                continue
            else:
                main_div = main_div_list[0]

            example_div_list = li.select("div.exampleLists")

            # Then find the <p> in the first div, which contains word category and
            # the meaning of the word
            p = main_div.find("p")
            if p is None:
                dbg_printf("Did not find the <p> under main <div> (index = %d)",
                           li_count)
                return None

            span = p.find("span")
            # This is possible if this entry is simply a redirection
            if span is None:
                # Make it invisible
                d["category"] = "REDIRECTION"

                # Quick path: No examples, and just concatenates everything inside the
                # <p> tag
                meaning = ""
                for content in p.contents:
                    if isinstance(content, bs4.element.Tag) is True and \
                       content.name == "a" :
                        meaning = ("See <green>" + content.text.strip() + "</green> ")

                d["text"] = meaning
                d["examples"] = []

                ret["meanings"].append(d)

                continue

            # Save the category as category attribute
            d["category"] = span.text
            # Then for all text and child nodes in p, find the span
            # and then add all strings together after it
            start_concat = False
            meaning = ""
            for content in p.contents:
                if isinstance(content, bs4.element.Tag) is True and \
                   content.name == "span":
                    start_concat = True
                    continue

                if start_concat is True:
                    # for keywords in the article we manually surround them with
                    # <b></b> tags
                    if isinstance(content, bs4.element.Tag) is True and \
                       content.name == "b":
                        content = "<red>" + content.text + "</red>"

                    content = content.strip()
                    if len(content) == 0:
                        continue

                    # Then use space to separate the contents
                    meaning += (content + " ")

            # If we did not find anything then return
            if start_concat is False or \
               len(meaning) == 0:
                dbg_printf("Did not find the <span> in the meaning of the word")
                return None

            # Save the meaning of the word as the text
            d["text"] = meaning

            # Push examples into this list
            l = []
            d["examples"] = l
            # These are all examples
            for div in example_div_list:
                # Text is the first p and translation is the second p
                # We do not care the remaining <p>, but if there are less than
                # two then we simply return
                p_list = div.find_all("p")
                if len(p_list) < 2:
                    return None

                l.append({
                    "text": p_list[0].text.strip(),
                    "translation": p_list[1].text.strip(),
                })

            # Append the dict here such that if we continue before here
            # the changes will not be committed
            ret["meanings"].append(d)

        # Set the actual key on the webpage
        if actual_key is None:
            actual_key = ret["word"]

        ret_list.append(ret)

    # Last check to avoid adding a None key into the cache
    if actual_key is None:
        dbg_printf("Did not find actual key")
        return None

    # Add the word to the cache
    add_to_cache(actual_key, ret_list)

    return ret_list

def get_cache_file_list(path):
    """
    This function counts the number of json files under a given directory.
    To save an system call the path is given as the argument
    
    If the passed path is not a valid directory then the result is undefined
    
    :return: int, as the number of json files 
    """
    return glob.glob(os.join(path, "*.json"))

# The name of the directory under the file directory as the word cache
CACHE_DIRECTORY = "cache"

# The max number of entries we allow for the cache
# When we add to the cache we check this first, and if the actual number of
# json files is greater than this we randomly delete files from the cache
# If this is set to -1 then there is no limit
# If this is set to 0 then cache is disabled
CACHE_MAX_ENTRY = 10

def trim_cache(cache_dir, limit):
    """
    Randomly remove cache content under given path until the number of file equals
    or is less than the given limit
    
    :param cache_dir: Under which we store cached file
    :param limit: The maximum number of entries allowed for the cache
    :return: int, the number of files we actually deleted
    """
    cache_file_list = get_cache_file_list(cache_dir)
    current_cache_size = len(cache_file_list)

    deleted_count = 0
    if current_cache_size > limit:
        # This is the number of files we need to delete
        delta = current_cache_size - limit
        # Then do a permutation of the list and pick the first
        # "deleted_count" elements to delete
        for i in range(0, delta):
            exchange_index = randint(0, current_cache_size)
            # Then exchange the elements
            t = cache_file_list[exchange_index]
            cache_file_list[exchange_index] = cache_file_list[i]
            cache_file_list[i] = t

        for i in range(0, delta):
            try:
                os.unlink(cache_file_list[i])
            except OSError:
                # Offset the += 1 later
                deleted_count -= 1
            deleted_count += 1

    return deleted_count

def add_to_cache(word, d):
    """
    This function adds a word and its associated dictionary object into the local cache
    If this word is queried in the future, it will be served from the cache
    
    If the word is already in the cache we just ignore it
    
    :param word: The word queried 
    :param d: The dictionary object returned by the parser
    :return: None
    """
    # If cache is disabled then return directly
    if CACHE_MAX_ENTRY == 0:
        return

    # This is the directory of the current file
    file_dir = get_file_dir()
    cache_dir = os.path.join(file_dir, CACHE_DIRECTORY)

    # If the cache directory has not yet been created then just create it
    if os.path.isdir(cache_dir) is False:
        os.mkdir(cache_dir)

    # Then before we check for the existence of the file, we
    # check whether the cache directory is full, and if it is
    # randomly choose one and then remove it
    # -1 means there is no limit
    if CACHE_MAX_ENTRY != -1:
        ret = trim_cache(cache_dir, CACHE_MAX_ENTRY)
        dbg_printf("Deleted %d files from the cache", ret)

    # This is the word file
    word_file = os.path.join(cache_dir, "%s.json" % (word, ))
    # If the file exists then warning
    if os.path.isfile(word_file) is True:<commit_msg>Small change; Fix minor problems<commit_after>
from bs4 import BeautifulSoup
import bs4
import requests
import sys
import os
import json
import stat
import inspect
from random import randint
import glob

def dbg_printf(format, *args):
    """
    C-Style function that writes debugging output to the terminal. If debug flag is off
    this function does not print anything

    :param format: The format string
    :param args: Arguments
    :return: None
    """
    global debug_flag

    # Do not print anything annoying
    if debug_flag is False:
        return

    frame = inspect.currentframe()
    prev_frame = frame.f_back
    code = prev_frame.f_code
    prev_name = code.co_name

    # Make it more human readable by replacing the name with
    # an easy to understand one
    if prev_name == "<module>":
        prev_name = "[Top Level Module]"
    else:
        prev_name += "()"

    # Write the prologue of debugging information
    sys.stderr.write("%-28s: " % (prev_name,))

    format = format % tuple(args)
    sys.stderr.write(format)

    # So we do not need to worry about new lines
    sys.stderr.write('\n')

    return

def get_webpage(word):
    """
    This function returns a string which is the webpage of a given word
    Returns None if the return code is not HTTP status 200 which means an error
    happened
    
    :return: str/None 
    """
    url = "http://dict.youdao.com/search?le=eng&q=%s&keyfrom=dict2.index" % (word, )
    r = requests.get(url)
    if r.status_code != 200:
        dbg_printf("Error executing HTTP request to %s; return code %d",
                   url,
                   r.status_code)

        return None

    return r.text

def parse_webpage(s):
    """
    Given the text of the webpage, parse it as an instance of a beautiful soup,
    using the default parser and encoding
    
    :param s: The text of the webpage 
    :return: beautiful soup object
    """
    return BeautifulSoup(s, 'html.parser')

def get_collins_dict(tree):
    """
    This function returns results by collins dictionary, given the beautiful soup
    tree object
    
    The return value is defined as follows: 
      
    {
      "word": "The word"
      "phonetic": "The pronunciation"
      "frequency": 4,    // This is always a number between 1 and 5, or -1 to indicate unknown
      "meanings": [
        {
          "category": "n. v. adj. adv. , etc.",
          "text": "Meaning of the word",
          "examples": [
            {
              "text": "Text of the example, with <b></b> being the keyword"
              "translation": "Translation of the text"
            },
          ]
        }, ...
      ]
    }
    
    :param tree: The beautiful soup tree
    :return: A list of the dict as specified as above, or None if fails
    """
    collins_result = tree.find(id="collinsResult")
    if isinstance(collins_result, bs4.element.Tag) is False:
        dbg_printf("Did not find id='collinsResult'")
        # it is not a valid div object (usually None if the tag does
        # not exist)
        return None
    elif collins_result.name != "div":
        dbg_printf("id='collinsResult' is not a div tag")
        # It is a valid tag, but the name is not div
        # This should be strange, though
        return None

    # This list contains all meanings of the word, each with a pronunciation
    top_level_list = collins_result.select("div.wt-container")
    ret_list = []
    # We set this to be the first word
    actual_key = None
    for tree in top_level_list:
        # We append this into a list
        ret = {}

        # This <h4> contains the main word, the pronunciation
        h4 = tree.find("h4")
        if h4 is None:
            dbg_printf("Did not find <h4>")
            return None

        span_list = h4.find_all("span")
        if len(span_list) < 1:
            dbg_printf("Did not find <span> under <h4>")
            return None

        # This is the word we are looking for
        ret["word"] = span_list[0].text

        # This contains the phonetic
        em = h4.find("em")
        if em is None:
            dbg_printf("Did not find <em> under <h4>")
            return None

        # Save the phonetic (note: this is not ASCII)
        ret["phonetic"] = em.text
        # Initialize the meanings list
        ret["meanings"] = []

        # Get the frequency span; If no such element just set it to -1
        # which means the freq is invalid
        freq_span = h4.select("span.star")
        if len(freq_span) == 0:
            ret["frequency"] = -1
        else:
            freq = -1
            star_attr = freq_span[0].attrs["class"]
            if "star1" in star_attr:
                freq = 1
            elif "star2" in star_attr:
                freq = 2
            elif "star3" in star_attr:
                freq = 3
            elif "star4" in star_attr:
                freq = 4
            elif "star5" in star_attr:
                freq = 5

            ret["frequency"] = freq

        # This is all meanings
        li_list = tree.find_all("li")
        li_count = -1
        for li in li_list:
            li_count += 1

            # Just add stuff into this dict object
            d = {}

            # find main div and example div list
            main_div_list = li.select("div.collinsMajorTrans")
            if len(main_div_list) == 0:
                continue
            else:
                main_div = main_div_list[0]

            example_div_list = li.select("div.exampleLists")

            # Then find the <p> in the first div, which contains word category and
            # the meaning of the word
            p = main_div.find("p")
            if p is None:
                dbg_printf("Did not find the <p> under main <div> (index = %d)",
                           li_count)
                return None

            span = p.find("span")
            # This is possible if this entry is simply a redirection
            if span is None:
                # Make it invisible
                d["category"] = "REDIRECTION"

                # Quick path: No examples, and just concatenates everything inside the
                # <p> tag
                meaning = ""
                for content in p.contents:
                    if isinstance(content, bs4.element.Tag) is True and \
                       content.name == "a" :
                        meaning = ("See <green>" + content.text.strip() + "</green> ")

                d["text"] = meaning
                d["examples"] = []

                ret["meanings"].append(d)

                continue

            # Save the category as category attribute
            d["category"] = span.text
            # Then for all text and child nodes in p, find the span
            # and then add all strings together after it
            start_concat = False
            meaning = ""
            for content in p.contents:
                if isinstance(content, bs4.element.Tag) is True and \
                   content.name == "span":
                    start_concat = True
                    continue

                if start_concat is True:
                    # for keywords in the article we manually surround them with
                    # <b></b> tags
                    if isinstance(content, bs4.element.Tag) is True and \
                       content.name == "b":
                        content = "<red>" + content.text + "</red>"
                    elif isinstance(content, bs4.element.Tag):
                        content = content.text.strip()
                    else:
                        content = content.strip()

                    if len(content) == 0:
                        continue

                    # Then use space to separate the contents
                    meaning += (content + " ")

            # If we did not find anything then return
            if start_concat is False or \
               len(meaning) == 0:
                dbg_printf("Did not find the <span> in the meaning of the word")
                return None

            # Save the meaning of the word as the text
            d["text"] = meaning

            # Push examples into this list
            l = []
            d["examples"] = l
            # These are all examples
            for div in example_div_list:
                # Text is the first p and translation is the second p
                # We do not care the remaining <p>, but if there are less than
                # two then we simply return
                p_list = div.find_all("p")
                if len(p_list) < 2:
                    return None

                l.append({
                    "text": p_list[0].text.strip(),
                    "translation": p_list[1].text.strip(),
                })

            # Append the dict here such that if we continue before here
            # the changes will not be committed
            ret["meanings"].append(d)

        # Set the actual key on the webpage
        if actual_key is None:
            actual_key = ret["word"]

        ret_list.append(ret)

    # Last check to avoid adding a None key into the cache
    if actual_key is None:
        dbg_printf("Did not find actual key")
        return None

    # Add the word to the cache
    add_to_cache(actual_key, ret_list)

    return ret_list

def get_cache_file_list(path):
    """
    This function counts the number of json files under a given directory.
    To save an system call the path is given as the argument
    
    If the passed path is not a valid directory then the result is undefined
    
    :return: int, as the number of json files 
    """
    return glob.glob(os.path.join(path, "*.json"))

# The name of the directory under the file directory as the word cache
CACHE_DIRECTORY = "cache"

# The max number of entries we allow for the cache
# When we add to the cache we check this first, and if the actual number of
# json files is greater than this we randomly delete files from the cache
# If this is set to -1 then there is no limit
# If this is set to 0 then cache is disabled
CACHE_MAX_ENTRY = 10

def trim_cache(cache_dir, limit):
    """
    Randomly remove cache content under given path until the number of file equals
    or is less than the given limit
    
    :param cache_dir: Under which we store cached file
    :param limit: The maximum number of entries allowed for the cache
    :return: int, the number of files we actually deleted
    """
    cache_file_list = get_cache_file_list(cache_dir)
    current_cache_size = len(cache_file_list)

    deleted_count = 0
    if current_cache_size >= limit:
        # This is the number of files we need to delete
        delta = current_cache_size - limit + 1
        # Then do a permutation of the list and pick the first
        # "deleted_count" elements to delete
        for i in range(0, delta):
            exchange_index = randint(0, current_cache_size - 1)
            # Then exchange the elements
            t = cache_file_list[exchange_index]
            cache_file_list[exchange_index] = cache_file_list[i]
            cache_file_list[i] = t

        for i in range(0, delta):
            try:
                os.unlink(cache_file_list[i])
            except OSError:
                # Offset the += 1 later
                deleted_count -= 1
            deleted_count += 1

    return deleted_count

def add_to_cache(word, d):
    """
    This function adds a word and its associated dictionary object into the local cache
    If this word is queried in the future, it will be served from the cache
    
    If the word is already in the cache we just ignore it
    
    :param word: The word queried 
    :param d: The dictionary object returned by the parser
    :return: None
    """
    # If cache is disabled then return directly
    if CACHE_MAX_ENTRY == 0:
        return

    # This is the directory of the current file
    file_dir = get_file_dir()
    cache_dir = os.path.join(file_dir, CACHE_DIRECTORY)

    # If the cache directory has not yet been created then just create it
    if os.path.isdir(cache_dir) is False:
        os.mkdir(cache_dir)

    # Then before we check for the existence of the file, we
    # check whether the cache directory is full, and if it is
    # randomly choose one and then remove it
    # -1 means there is no limit
    if CACHE_MAX_ENTRY != -1:
        ret = trim_cache(cache_dir, CACHE_MAX_ENTRY)
        dbg_printf("Deleted %d file(s) from the cache", ret)

    # This is the word file
    word_file = os.path.join(cache_dir, "%s.json" % (word, ))
    # If the file exists then warning
    if os.path.isfile(word_file) is True:
{'fraction_non_alphanumeric': 0.07659252775235746, 'fraction_numerical': 0.0056101539808220266, 'mean_word_length': 1.7735599205473405, 'pattern_counts': {'":': 30, '<': 49, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '754117', 'n_tokens_mistral': 7031, 'n_tokens_neox': 6648, 'n_words': 2701}