<commit_before>import requests, re
from bs4 import BeautifulSoup

"""
Google Translate loads a JavaScript Array of 'languages
codes' that can be read. We intersect with all the
languages Google Translate provides.
"""

class LanguagesFetchError(Exception):
    pass

class Languages:
    """ Supported languages by Google's Text to Speech API """
    
    URL_BASE = 'http://translate.google.com'
    JS_FILE = 'desktop_module_main.js'
   
    """
    Special undocumented language codes observed
    to provide different dialects or accents
    """
    SPECIAL_LANGS = {
        # Chinese
        'zh-cn': 'Chinese (Mandarin/China)',
        'zh-tw': 'Chinese (Mandarin/Taiwan)',
        # 'zh-yue': 'Chinese (Cantonese)',
        # English
        'en-us': 'English (US)',
        'en-ca': 'English (Canada)',
        'en-uk': 'English (UK)',
        'en-gb': 'English (UK)', 
        'en-au': 'English (Australia)',
        # French
        'fr-ca': 'French (Canada)',
        # Portuguese
        'pt-br': 'Portuguese (Brazil)',
        'pt-pt': 'Portuguese (Portugal)',
        # Spanish
        'es-es' : 'Spanish (Spain)',
        'es-us' : 'Spanish (United States)'
    } 

    def __init__(self):
        self.langs = dict()

    def get(self):
        self.langs = self._fetch_langs()
        self.langs.update(self.SPECIAL_LANGS)
        return self.langs

    def _fetch_langs(self):
        try:
            """ Load HTML """
            page = requests.get(self.URL_BASE)
            soup = BeautifulSoup(page.content, 'html.parser')

            """ JavaScript URL
            The <script src=''> path can change, but not the file.
            Ex: /zyx/abc/20180211/desktop_module_main.js
            """
            js_path = soup.find(src=re.compile(self.JS_FILE))['src']
            js_url = "{}/{}".format(self.URL_BASE, js_path)

            """ Load JavaScript """
            js_contents = str(requests.get(js_url).content)

            """ Approximately extract TTS-enabled language codes
            RegEx pattern search because minified variables can change.
            Extra garbage will be dealt with later as we keep languages only.
            In: "[...]Fv={af:1,ar:1,[...],zh:1,"zh-cn":1,"zh-tw":1}[...]"
            Out: ['is', '12', [...], 'af', 'ar', [...], 'zh', 'zh-cn', 'zh-tw']
            """
            pattern = '[{,\"](\w{2}|\w{2}-\w{2,3})(?=:1|\":1)'
            tts_langs = re.findall(pattern, js_contents)

            """ Build lang. dict. from HTML lang. <select>
            Filtering with the TTS-enabled languages
            In: [<option value='af'>Afrikaans</option>, [...]]
            Out: {'af': 'Afrikaans', [...]}
            """
            langs_html = soup.find('select', {'id': 'gt-sl'}).findAll('option')
            langs = {l['value']:l.text for l in langs_html if l['value'] in tts_langs}
            return langs
        except Exception as e:
            raise LanguagesFetchError("Unable to get language list: {}".format(str(e)))

if __name__ == "__main__":
    pass<commit_msg>Added method to get lang. codes list. Style<commit_after>import requests, re
from bs4 import BeautifulSoup

"""Google Translate loads a JavaScript Array of 'languages
codes' that can be read. We intersect with all the
languages Google Translate provides.
"""

class LanguagesFetchError(Exception):
    pass

class Languages:
    """Supported languages by Google's Text to Speech API"""
    
    URL_BASE = 'http://translate.google.com'
    JS_FILE = 'desktop_module_main.js'
   
    """Special undocumented language codes observed
    to provide different dialects or accents
    """
    SPECIAL_LANGS = {
        # Chinese
        'zh-cn': 'Chinese (Mandarin/China)',
        'zh-tw': 'Chinese (Mandarin/Taiwan)',
        # 'zh-yue': 'Chinese (Cantonese)',
        # English
        'en-us': 'English (US)',
        'en-ca': 'English (Canada)',
        'en-uk': 'English (UK)',
        'en-gb': 'English (UK)', 
        'en-au': 'English (Australia)',
        # French
        'fr-ca': 'French (Canada)',
        # Portuguese
        'pt-br': 'Portuguese (Brazil)',
        'pt-pt': 'Portuguese (Portugal)',
        # Spanish
        'es-es' : 'Spanish (Spain)',
        'es-us' : 'Spanish (United States)'
    } 

    def __init__(self):
        self.langs = dict()

    def get(self):
        self.langs = self._fetch_langs()
        self.langs.update(self.SPECIAL_LANGS)
        return self.langs

    def get_list(self):
        langs_dict = self.get()
        langs_list = list(langs_dict.keys()) 
        return langs_list

    def _fetch_langs(self):
        try:
            """Load HTML"""
            page = requests.get(self.URL_BASE)
            soup = BeautifulSoup(page.content, 'html.parser')

            """JavaScript URL
            The <script src=''> path can change, but not the file.
            Ex: /zyx/abc/20180211/desktop_module_main.js
            """
            js_path = soup.find(src=re.compile(self.JS_FILE))['src']
            js_url = "{}/{}".format(self.URL_BASE, js_path)

            """Load JavaScript"""
            js_contents = str(requests.get(js_url).content)

            """Approximately extract TTS-enabled language codes
            RegEx pattern search because minified variables can change.
            Extra garbage will be dealt with later as we keep languages only.
            In: "[...]Fv={af:1,ar:1,[...],zh:1,"zh-cn":1,"zh-tw":1}[...]"
            Out: ['is', '12', [...], 'af', 'ar', [...], 'zh', 'zh-cn', 'zh-tw']
            """
            pattern = '[{,\"](\w{2}|\w{2}-\w{2,3})(?=:1|\":1)'
            tts_langs = re.findall(pattern, js_contents)

            """Build lang. dict. from HTML lang. <select>
            Filtering with the TTS-enabled languages
            In: [<option value='af'>Afrikaans</option>, [...]]
            Out: {'af': 'Afrikaans', [...]}
            """
            langs_html = soup.find('select', {'id': 'gt-sl'}).findAll('option')
            langs = {l['value']:l.text for l in langs_html if l['value'] in tts_langs}
            return langs
        except Exception as e:
            raise LanguagesFetchError("Unable to get language list: {}".format(str(e)))

if __name__ == "__main__":
    pass
{'fraction_non_alphanumeric': 0.15822378967617826, 'fraction_numerical': 0.007053542802180186, 'mean_word_length': 2.1304565980933265, 'pattern_counts': {'":': 8, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1114629', 'n_tokens_mistral': 1986, 'n_tokens_neox': 1897, 'n_words': 486}