<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys
import re
import urlparse
import urllib
import json

helptext = """
Take language statistics from stdin and write candidates,
i.e. url with some language identifier removed to stdout.

If a list of pre-generated candidates is given, only those
stripped urls that also occur in the list of candidates will
be written.

Input format:
tld url crawl<TAB>json_data

Example:
ex http://www.ex.com/id.html 2014_42  {"languages": [["en", 1000]]}

"""


def stoi(s):
    """ works like int(s) but also accepts floats and scientific notation """
    try:
        return int(s)
    except ValueError:
        return int(float(s))


class LanguageStripper(object):

    def __init__(self, languages=None):
        self.code_to_language = {}
        for code in ["arabic", "ara", "ar"]:
            self.code_to_language[code] = "ar"
        for code in ["bulgarian", "bul", "bg"]:
            self.code_to_language[code] = "bg"
        for code in ["czech", "cze", "cz", "cs"]:
            self.code_to_language[code] = "cs"
        for code in ["deutsch", "german", "ger", "deu", "de"]:
            self.code_to_language[code] = "de"
        for code in ["english", "eng", "en"]:
            self.code_to_language[code] = "en"
        for code in ["espanol", "spanish", "spa", "esp", "es"]:
            self.code_to_language[code] = "es"
        for code in ["french", "francais", "fran", "fra", "fre", "fr"]:
            self.code_to_language[code] = "fr"
        for code in ["chinese", "chi", "zh"]:
            self.code_to_language[code] = "zh"
        # new, not in "Dirt-Cheap"-paper
        for code in ["tedesco", "de-de", "de-ch", "de-at", "de-li", 'de-lu',
                     'allemand']:
            self.code_to_language[code] = "de"
        for code in ["fr-be", "fr-ca", "fr-fr", "fr-lu", "fr-ch"]:
            self.code_to_language[code] = "fr"
        for code in ["italian", "italiano", "ital", 'ita', 'it-it', 'it-ch',
                     'it']:
            self.code_to_language[code] = "it"
        for code in ["en-en", "en-us", "en-uk", 'en-ca', 'en-bz', 'en-ab',
                     'en-in', 'en-ie', 'en-jm', 'en-nz', 'en-ph', 'en-za',
                     'en-tt', 'inglese']:
            self.code_to_language[code] = "en"
        for code in ["romanian", "romana", "romlang", 'rom', 'ro-ro', 'ro']:
            self.code_to_language[code] = "ro"


        if languages is not None:
            kv_pairs = [(k, v) for k, v in self.code_to_language.items()
                        if v in languages]
            self.code_to_language = dict(kv_pairs)

        for code, lang in self.code_to_language.items():
            # add de_de from de-de
            self.code_to_language[code.replace('-', '_')] = lang

        keys = self.code_to_language.keys()
        keys.sort(key=len, reverse=True)
        regexp_string = "(?<![a-zA-Z0-9])(?:%s)(?![a-zA-Z0-9])" % (
            "|".join(keys))
        self.re_code = re.compile(regexp_string, re.IGNORECASE)

        # remove "-eng" including the hyphen but not -fr from fr-fr
        keys = [key for key in keys if '-' not in key and '_' not in key]<commit_msg>added somali identifiers<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys
import re
import urlparse
import urllib
import json

helptext = """
Take language statistics from stdin and write candidates,
i.e. url with some language identifier removed to stdout.

If a list of pre-generated candidates is given, only those
stripped urls that also occur in the list of candidates will
be written.

Input format:
tld url crawl<TAB>json_data

Example:
ex http://www.ex.com/id.html 2014_42  {"languages": [["en", 1000]]}

"""


def stoi(s):
    """ works like int(s) but also accepts floats and scientific notation """
    try:
        return int(s)
    except ValueError:
        return int(float(s))


class LanguageStripper(object):

    def __init__(self, languages=None):
        self.code_to_language = {}
        # These should all be lower-case, matching is case-insensitive
        for code in ["arabic", "ara", "ar"]:
            self.code_to_language[code] = "ar"
        for code in ["bulgarian", "bul", "bg"]:
            self.code_to_language[code] = "bg"
        for code in ["czech", "cze", "cz", "cs"]:
            self.code_to_language[code] = "cs"
        for code in ["deutsch", "german", "ger", "deu", "de"]:
            self.code_to_language[code] = "de"
        for code in ["english", "eng", "en"]:
            self.code_to_language[code] = "en"
        for code in ["espanol", "spanish", "spa", "esp", "es"]:
            self.code_to_language[code] = "es"
        for code in ["french", "francais", "fran", "fra", "fre", "fr"]:
            self.code_to_language[code] = "fr"
        for code in ["chinese", "chi", "zh"]:
            self.code_to_language[code] = "zh"
        # new, not in "Dirt-Cheap"-paper
        for code in ["tedesco", "de-de", "de-ch", "de-at", "de-li", 'de-lu',
                     'allemand']:
            self.code_to_language[code] = "de"
        for code in ["fr-be", "fr-ca", "fr-fr", "fr-lu", "fr-ch"]:
            self.code_to_language[code] = "fr"
        for code in ["italian", "italiano", "ital", 'ita', 'it-it', 'it-ch',
                     'it']:
            self.code_to_language[code] = "it"
        for code in ["en-en", "en-us", "en-uk", 'en-ca', 'en-bz', 'en-ab',
                     'en-in', 'en-ie', 'en-jm', 'en-nz', 'en-ph', 'en-za',
                     'en-tt', 'inglese']:
            self.code_to_language[code] = "en"
        for code in ["romanian", "romana", "romlang", 'rom', 'ro-ro', 'ro']:
            self.code_to_language[code] = "ro"
        for code in ["soma", "som", "so", 'somal', 'somali', 'so-so',
                     'af-soomaali', 'soomaali']:
            self.code_to_language[code] = "so"

        if languages is not None:
            kv_pairs = [(k, v) for k, v in self.code_to_language.items()
                        if v in languages]
            self.code_to_language = dict(kv_pairs)

        for code, lang in self.code_to_language.items():
            # add de_de from de-de
            self.code_to_language[code.replace('-', '_')] = lang

        keys = self.code_to_language.keys()
        keys.sort(key=len, reverse=True)
        regexp_string = "(?<![a-zA-Z0-9])(?:%s)(?![a-zA-Z0-9])" % (
            "|".join(keys))
        self.re_code = re.compile(regexp_string, re.IGNORECASE)

        # remove "-eng" including the hyphen but not -fr from fr-fr
        keys = [key for key in keys if '-' not in key and '_' not in key]
{'fraction_non_alphanumeric': 0.16789529110877088, 'fraction_numerical': 0.0045133142771174965, 'mean_word_length': 2.2732644017725256, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5144218', 'n_tokens_mistral': 2338, 'n_tokens_neox': 2273, 'n_words': 614}