<commit_before># Copyright 2012 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Common utility functions."""

__author__ = '<EMAIL> (<NAME>)'

import base64
import copy
import hashlib
import json
import logging
import os
import unicodedata
import yaml

from jinja2 import Environment
from jinja2 import meta

import feconf


class InvalidInputException(Exception):
    """Error class for invalid input."""
    pass


class EntityIdNotFoundError(Exception):
    """Error class for when an entity ID is not in the datastore."""
    pass


def log(message):
    """Logs info messages in development/debug mode.

    Args:
        message: the message to be logged.
    """
    if feconf.DEV or feconf.DEBUG:
        if isinstance(message, dict):
            logging.info(json.dumps(message, sort_keys=True, indent=4))
        else:
            logging.info(str(message))


def create_enum(*sequential, **names):
    enums = dict(zip(sequential, sequential), **names)
    return type('Enum', (), enums)


def get_new_id(entity, entity_name):
    """Gets a new id for an entity, based on its name.

    Args:
        entity: the entity's class.
        entity_name: string representing the name of the entity

    Returns:
        string - the 12-character id representing the entity
    """
    seed = 0
    new_id = base64.urlsafe_b64encode(
        hashlib.sha1(entity_name.encode('utf-8')).digest())[:12]

    while entity.get_by_id(new_id):
        seed += 1
        new_id = base64.urlsafe_b64encode(
            hashlib.sha1('%s%s' % (
                entity_name.encode('utf-8'), seed)).digest())[:12]

    return new_id


def get_file_contents(root, filepath):
    """Gets the contents of a file.

    Args:
        root: the path to prepend to the filepath.
        filepath: a path to a HTML, JS or CSS file. It should not include the
            template/dev/head or template/prod/head prefix.

    Returns:
        the file contents.
    """
    with open(os.path.join(root, filepath)) as f:
        return f.read().decode('utf-8')


def get_js_controllers(filenames):
    """Gets the concatenated contents of some JS controllers.

    Args:
        filenames: an array with names of JS files (without the '.js' suffix).

    Returns:
        the concatenated contents of these JS files.
    """
    return '\n'.join([
        get_file_contents(
            feconf.TEMPLATE_DIR, 'js/controllers/%s.js' % filename
        ) for filename in filenames
    ])


def convert_to_js_string(value):
    """Converts a value to a unicode string without extra 'u' characters."""

    def recursively_convert_to_unicode(value, built_string):
        if value is None:
            return '%s%s' % (built_string, 'null')
        if (isinstance(value, int) or isinstance(value, float)):
            return '%s%s' % (built_string, value)
        elif isinstance(value, unicode):
            return "%s'%s'" % (built_string, value.replace("'", "\\'"))
        elif isinstance(value, str):
            return u"%s'%s'" % (
                built_string, value.decode('utf-8').replace("'", "\\'"))
        elif isinstance(value, list) or isinstance(value, set):
            string = u'['
            for index, item in enumerate(value):
                string = recursively_convert_to_unicode(item, string)
                if index != len(value) - 1:
                    string += ', '
            string += ']'
            return '%s%s' % (built_string, string)
        elif isinstance(value, dict):
            string = u'{'
            index = 0
            for key, val in value.iteritems():
                string = recursively_convert_to_unicode(key, string)
                string += ': '
                string = recursively_convert_to_unicode(val, string)
                if index != len(value) - 1:<commit_msg>Add better JS escaping.<commit_after># Copyright 2012 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Common utility functions."""

__author__ = '<EMAIL> (<NAME>)'

import base64
import copy
import hashlib
import json
import logging
import os
import unicodedata
import yaml

from jinja2 import Environment
from jinja2 import meta

import feconf


class InvalidInputException(Exception):
    """Error class for invalid input."""
    pass


class EntityIdNotFoundError(Exception):
    """Error class for when an entity ID is not in the datastore."""
    pass


def log(message):
    """Logs info messages in development/debug mode.

    Args:
        message: the message to be logged.
    """
    if feconf.DEV or feconf.DEBUG:
        if isinstance(message, dict):
            logging.info(json.dumps(message, sort_keys=True, indent=4))
        else:
            logging.info(str(message))


def create_enum(*sequential, **names):
    enums = dict(zip(sequential, sequential), **names)
    return type('Enum', (), enums)


def get_new_id(entity, entity_name):
    """Gets a new id for an entity, based on its name.

    Args:
        entity: the entity's class.
        entity_name: string representing the name of the entity

    Returns:
        string - the 12-character id representing the entity
    """
    seed = 0
    new_id = base64.urlsafe_b64encode(
        hashlib.sha1(entity_name.encode('utf-8')).digest())[:12]

    while entity.get_by_id(new_id):
        seed += 1
        new_id = base64.urlsafe_b64encode(
            hashlib.sha1('%s%s' % (
                entity_name.encode('utf-8'), seed)).digest())[:12]

    return new_id


def get_file_contents(root, filepath):
    """Gets the contents of a file.

    Args:
        root: the path to prepend to the filepath.
        filepath: a path to a HTML, JS or CSS file. It should not include the
            template/dev/head or template/prod/head prefix.

    Returns:
        the file contents.
    """
    with open(os.path.join(root, filepath)) as f:
        return f.read().decode('utf-8')


def get_js_controllers(filenames):
    """Gets the concatenated contents of some JS controllers.

    Args:
        filenames: an array with names of JS files (without the '.js' suffix).

    Returns:
        the concatenated contents of these JS files.
    """
    return '\n'.join([
        get_file_contents(
            feconf.TEMPLATE_DIR, 'js/controllers/%s.js' % filename
        ) for filename in filenames
    ])


def js_escape(string):
    """Escapes a string so it can be safely inserted into JavaScript code."""
    if not string:
        return string
    return string.replace('\\', '\\\\').replace('"', '\\"').replace(
        "'", "\\'").replace('\n', '\\n').replace('\r', '\\r').replace(
            '\b', '\\b')


def convert_to_js_string(value):
    """Converts a value to a unicode string without extra 'u' characters."""

    def recursively_convert_to_unicode(value, built_string):
        if value is None:
            return '%s%s' % (built_string, 'null')
        if (isinstance(value, int) or isinstance(value, float)):
            return '%s%s' % (built_string, value)
        elif isinstance(value, unicode):
            return "%s'%s'" % (built_string, js_escape(value))
        elif isinstance(value, str):
            return u"%s'%s'" % (
                built_string, js_escape(value.decode('utf-8')))
        elif isinstance(value, list) or isinstance(value, set):
            string = u'['
            for index, item in enumerate(value):
                string = recursively_convert_to_unicode(item, string)
                if index != len(value) - 1:
                    string += ', '
            string += ']'
            return '%s%s' % (built_string, string)
        elif isinstance(value, dict):
            string = u'{'
            index = 0
            for key, val in value.iteritems():
                string = recursively_convert_to_unicode(key, string)
                string += ': '
                string = recursively_convert_to_unicode(val, string)
                if index != len(value) - 1:
{'fraction_non_alphanumeric': 0.11180331516297697, 'fraction_numerical': 0.00845477806207587, 'mean_word_length': 2.522727272727273, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6995924', 'n_tokens_mistral': 2652, 'n_tokens_neox': 2522, 'n_words': 858}