<commit_before># Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE
"""
	frappe.translate
	~~~~~~~~~~~~~~~~

	Translation tools for frappe
"""

import io
import itertools
import json
import operator
import functools
import os
import re
from csv import reader
from typing import List, Union, Tuple

import frappe
from frappe.model.utils import InvalidIncludePath, render_include
from frappe.utils import get_bench_path, is_html, strip, strip_html_tags
from frappe.query_builder import Field, DocType
from pypika.terms import PseudoColumn

TRANSLATE_PATTERN = re.compile(
	r"_\("  # starts with literal `_(` work with both python / JS

	# BEGIN: message search
	r"([\"']{,3})"  # start of message string identifier - allows: ', ", """, '''; 1st capture group
		r"(?P<message>((?!\1).)*)" # Keep matching until string closing identifier is met which is same as 1st capture group
	r"\1" # match exact string closing identifier
	# END: message search

	# BEGIN: python context search
	r"(\s*,\s*context\s*=\s*"  # capture `context=` with ignoring whitespace
		r"([\"'])"  # start of context string identifier; 5th capture group
		r"(?P<py_context>((?!\5).)*)" # capture context string till closing id is found
		r"\5"  # match context string closure
	r")*"  # match one or more context string (?wat this should be 0 or 1)
	# END: python context search

	# BEGIN: JS context search
	r"(\s*,\s*(.)*?\s*(,\s*" # skip message format replacements: ["format", ...] | null | []
		r"([\"'])"  # start of context string; 11th capture group
		r"(?P<js_context>((?!\11).)*)" # capture context string till closing id is found
		r"\11"  # match context string closure
		r")*"
	r")*"  # match one or more context string
	# END: JS context search

	r"\)"  # Closing function call
)
<commit_msg>fix: only match 1 python context string at most<commit_after># Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE
"""
	frappe.translate
	~~~~~~~~~~~~~~~~

	Translation tools for frappe
"""

import io
import itertools
import json
import operator
import functools
import os
import re
from csv import reader
from typing import List, Union, Tuple

import frappe
from frappe.model.utils import InvalidIncludePath, render_include
from frappe.utils import get_bench_path, is_html, strip, strip_html_tags
from frappe.query_builder import Field, DocType
from pypika.terms import PseudoColumn

TRANSLATE_PATTERN = re.compile(
	r"_\("  # starts with literal `_(` work with both python / JS

	# BEGIN: message search
	r"([\"']{,3})"  # start of message string identifier - allows: ', ", """, '''; 1st capture group
		r"(?P<message>((?!\1).)*)" # Keep matching until string closing identifier is met which is same as 1st capture group
	r"\1" # match exact string closing identifier
	# END: message search

	# BEGIN: python context search
	r"(\s*,\s*context\s*=\s*"  # capture `context=` with ignoring whitespace
		r"([\"'])"  # start of context string identifier; 5th capture group
		r"(?P<py_context>((?!\5).)*)" # capture context string till closing id is found
		r"\5"  # match context string closure
	r")?"  # match 0 or 1 context strings
	# END: python context search

	# BEGIN: JS context search
	r"(\s*,\s*(.)*?\s*(,\s*" # skip message format replacements: ["format", ...] | null | []
		r"([\"'])"  # start of context string; 11th capture group
		r"(?P<js_context>((?!\11).)*)" # capture context string till closing id is found
		r"\11"  # match context string closure
		r")*"
	r")*"  # match one or more context string
	# END: JS context search

	r"\)"  # Closing function call
)

{'fraction_non_alphanumeric': 0.14022546054440474, 'fraction_numerical': 0.011273027220236459, 'mean_word_length': 4.303206997084548, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5352830', 'n_tokens_mistral': 1196, 'n_tokens_neox': 1123, 'n_words': 431}