<commit_before>import copy
import logging
import os
import pkgutil
import re
import requests
import shutil
import urllib.parse
from functools import partial
from urllib.parse import urlparse
from contentpacks.models import Item, AssessmentItem
from peewee import Using, SqliteDatabase, fn
import polib
import ujson
import zipfile
import tempfile
import pathlib


class UnexpectedKindError(Exception):
    pass


class NotTranslatable(Exception):
    pass


class NodeType:
    exercise = "Exercise"
    video = "Video"
    topic = "Topic"


NODE_FIELDS_TO_TRANSLATE = [
    "title",
    "description",
    "display_name",
]


LANGUAGELOOKUP_DATA = pkgutil.get_data('contentpacks', "resources/languagelookup.json")


class Catalog(dict):
    """
    Just like a dict, but computes some additional metadata specific to i18n catalog files.
    """

    def __init__(self, pofile=None):
        """
        Extract the strings from the given pofile, and computes the metadata.
        """
        # Add an entry for the empty message
        self[""] = ""
        if not pofile:
            pofile = []
            self.percent_translated = 0
        else:

            self.update({m.msgid: m.msgstr for m in pofile if m.translated()})

            # compute metadata -- needs to be after we add the translated strings
            self.percent_translated = self.compute_translated(pofile)

        super().__init__()

    def compute_translated(self, pofile: polib._BaseFile) -> int:
        """
        Returns the percentage of strings translated. Returned number is between 0
        to 100.

        """
        trans_count = len(self)
        all_strings_count = len(pofile)

        return (trans_count / all_strings_count) * 100


def cache_file(func):
    """
    Execute the decorated function only if the file in question is not already cached.
    Returns the path to the file. Always download the file if ignorecache is True.
    All decorated functions must only accept 2 args, 'url' and 'path'.
    """
    def func_wrapper(url, cachedir=None, ignorecache=False, filename=None, **kwargs):
            if not cachedir:
                cachedir = os.path.join(os.getcwd(), "build")

            if not filename:
                filename = os.path.basename(urlparse(url).path)

            path = os.path.join(cachedir, filename)

            os.makedirs(os.path.dirname(path), exist_ok=True)

            if ignorecache or not os.path.exists(path):
                func(url, path, **kwargs)

            return path

    return func_wrapper


@cache_file
def download_and_cache_file(url: str, path: str, headers: dict={}) -> str:
    """
    Download the given url if it's not saved in cachedir. Returns the
    path to the file. Always download the file if ignorecache is True.
    """

    logging.info("Downloading file from {url}".format(url=url))

    r = requests.get(url, stream=True, headers=headers)
    r.raise_for_status()

    with open(path, "wb") as f:
        shutil.copyfileobj(r.raw, f)

    return path


def translate_nodes(nodes: list, catalog: Catalog) -> list:<commit_msg>Use python.requests's recommended way of writing response chunks.<commit_after>import copy
import logging
import os
import pkgutil
import re
import requests
import shutil
import urllib.parse
from functools import partial
from urllib.parse import urlparse
from contentpacks.models import Item, AssessmentItem
from peewee import Using, SqliteDatabase, fn
import polib
import ujson
import zipfile
import tempfile
import pathlib


class UnexpectedKindError(Exception):
    pass


class NotTranslatable(Exception):
    pass


class NodeType:
    exercise = "Exercise"
    video = "Video"
    topic = "Topic"


NODE_FIELDS_TO_TRANSLATE = [
    "title",
    "description",
    "display_name",
]


LANGUAGELOOKUP_DATA = pkgutil.get_data('contentpacks', "resources/languagelookup.json")


class Catalog(dict):
    """
    Just like a dict, but computes some additional metadata specific to i18n catalog files.
    """

    def __init__(self, pofile=None):
        """
        Extract the strings from the given pofile, and computes the metadata.
        """
        # Add an entry for the empty message
        self[""] = ""
        if not pofile:
            pofile = []
            self.percent_translated = 0
        else:

            self.update({m.msgid: m.msgstr for m in pofile if m.translated()})

            # compute metadata -- needs to be after we add the translated strings
            self.percent_translated = self.compute_translated(pofile)

        super().__init__()

    def compute_translated(self, pofile: polib._BaseFile) -> int:
        """
        Returns the percentage of strings translated. Returned number is between 0
        to 100.

        """
        trans_count = len(self)
        all_strings_count = len(pofile)

        return (trans_count / all_strings_count) * 100


def cache_file(func):
    """
    Execute the decorated function only if the file in question is not already cached.
    Returns the path to the file. Always download the file if ignorecache is True.
    All decorated functions must only accept 2 args, 'url' and 'path'.
    """
    def func_wrapper(url, cachedir=None, ignorecache=False, filename=None, **kwargs):
            if not cachedir:
                cachedir = os.path.join(os.getcwd(), "build")

            if not filename:
                filename = os.path.basename(urlparse(url).path)

            path = os.path.join(cachedir, filename)

            os.makedirs(os.path.dirname(path), exist_ok=True)

            if ignorecache or not os.path.exists(path):
                func(url, path, **kwargs)

            return path

    return func_wrapper


@cache_file
def download_and_cache_file(url: str, path: str, headers: dict={}) -> str:
    """
    Download the given url if it's not saved in cachedir. Returns the
    path to the file. Always download the file if ignorecache is True.
    """

    logging.info("Downloading file from {url}".format(url=url))

    r = requests.get(url, stream=True, headers=headers)
    r.raise_for_status()

    with open(path, "wb") as f:
        for chunk in r.iter_content(1024):
            f.write(chunk)

    return path


def translate_nodes(nodes: list, catalog: Catalog) -> list:
{'fraction_non_alphanumeric': 0.08949292640279764, 'fraction_numerical': 0.004132888253059927, 'mean_word_length': 2.7033549146556797, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1962331', 'n_tokens_mistral': 1893, 'n_tokens_neox': 1778, 'n_words': 591}