<commit_before>#regex_replacer.py
# -*- coding: utf-8 -*-

#Script for replacing strings using regular expressions specified in a file

# See python_mysql_connector for the keywords that specify your MySQL credentials (user, password, db name, host).
# To run on the server, you must specify --user, --pwd, --db, --host.

#Note that you must download mysql.connector, since it doesn't come with the default distribution.
#Use: sudo apt-get update && sudo apt-get install python-mysql.connector
from __future__ import print_function
import codecs
import os
import re
import sys
from python_mysql_connector import PythonMySQLConnector

class RegexReplacer(PythonMySQLConnector):
    """
    Class for replacing text using a regex specified in a file.
    
    Only tested with Python 2.7.
    
    Reads a JSON file that contains a restrictor for the SELECT 
    statement (possibly empty), a MySQL regex for matching an existing pattern,
    a Python regex for matching the same pattern, and a substitution string to
    replace that pattern (to be fed to the re.sub() command). 
    
    Contents of file for replacing "is" or its case variants by "**" in Afrikaans sentences:
    [{"ascii": 1, "python_regex": "[iI][sS]", "mysql_regex": "[iI][sS]", "substitution_string": "**", "select_restrictor": " AND lang='afr'"}]

    Contents of file for replacing runs of whitespace with a single space character
    in all sentences:
    [{"ascii": 1, "python_regex": "\\s{2,}", "mysql_regex": "[[:space:]]{2,}", "substitution_string": " ", "select_restrictor": ""}]

    Contents of file for replacing "à" (UTF-8 representation = 0xC3A0) by "À" in Italian sentences: 
    [{"ascii": 0, "python_regex": "à", "mysql_regex": "C3A0", "substitution_string": "À", "select_restrictor": " AND lang='ita'"}]    
    """
    ASCII = "ascii"
    MYSQL_REGEX = "mysql_regex"
    PYTHON_REGEX = "python_regex"
    SELECT_RESTRICTOR = "select_restrictor"
    SUBSTITUTION_STRING = "substitution_string"
    def __init__(self):
        PythonMySQLConnector.__init__(self)
        self.json_fields = None

    def get_ascii(self):
        return self.json_fields[0][self.ASCII]
        
    def get_mysql_regex(self):
        return self.json_fields[0][self.MYSQL_REGEX]
        
    def get_python_regex(self):
        return self.json_fields[0][self.PYTHON_REGEX]
        
    def get_select_restrictor(self):
        return self.json_fields[0][self.SELECT_RESTRICTOR]
        
    def get_substitution_string(self):
        return self.json_fields[0][self.SUBSTITUTION_STRING]
        
    def init_parser(self):
        PythonMySQLConnector.init_parser(self, description='Uses a regex to replace instances of a matching pattern',   
            epilog='Sample invocation: python regex_replacer.py --json_file config.json --user root --db tatoeba --pwd "" --csv_dir c:\\temp')
        self.parser.add_argument('--csv_basename', default='regex_replacer.csv', help='basename of intermediate csv file') 

    def process(self):
        filename = os.path.join(self.parsed.csv_dir, self.parsed.csv_basename)
        self.write_csv(filename)
        self.read_csv(filename)

    def read_csv(self, filename):
        """Read a CSV file (id<TAB>text) produced by an earlier step and execute an SQL query to update text."""
        self.print_output("\n\nfilename: {0}".format(filename))
        if self.parsed.dry_run:
            self.print_output("---NOT executing these lines---")<commit_msg>updated comments in script<commit_after>#regex_replacer.py
# -*- coding: utf-8 -*-

#Script for replacing strings using regular expressions specified in a file

# See python_mysql_connector for the keywords that specify your MySQL credentials (user, password, db name, host).
# To run on the server, you must specify --user, --pwd, --db, --host.

#Note that you must download mysql.connector, since it doesn't come with the default distribution.
#Use: sudo apt-get update && sudo apt-get install python-mysql.connector
from __future__ import print_function
import codecs
import os
import re
import sys
from python_mysql_connector import PythonMySQLConnector

class RegexReplacer(PythonMySQLConnector):
    """
    Class for replacing text using a regex specified in a file.
    
    Only tested with Python 2.7.
    
    Reads a JSON file that contains the following elements:
    - a restrictor (possibly empty) for the SELECT statement
        - example: " AND lang='afr'"
    - a MySQL regex for matching an existing pattern
        - example: "[[:space:]]{2,}"
    - a Python regex for matching the same pattern
        - example: "\\s{2,}"
    - a substitution string to be fed to the re.sub() command
    to replace that pattern
        - example: " "
    - a flag that indicates whether the input string is ASCII (1=yes, 0=no)
        
    The fields may occur in arbitrary error, as they do in the 
    following examples.
    
    Contents of file for replacing "is" or its case variants by "**" 
    in Afrikaans sentences:
    [{"ascii": 1, "python_regex": "[iI][sS]", "mysql_regex": "[iI][sS]", "substitution_string": "**", "select_restrictor": " AND lang='afr'"}]

    Contents of file for replacing runs of whitespace with a single 
    space character in all sentences:
    [{"ascii": 1, "python_regex": "\\s{2,}", "mysql_regex": "[[:space:]]{2,}", "substitution_string": " ", "select_restrictor": ""}]

    Contents of file for replacing "à" (UTF-8 representation = 0xC3A0) 
    by "À" in Italian sentences: 
    [{"ascii": 0, "python_regex": "à", "mysql_regex": "C3A0", "substitution_string": "À", "select_restrictor": " AND lang='ita'"}]    
    """
    ASCII = "ascii"
    MYSQL_REGEX = "mysql_regex"
    PYTHON_REGEX = "python_regex"
    SELECT_RESTRICTOR = "select_restrictor"
    SUBSTITUTION_STRING = "substitution_string"
    def __init__(self):
        PythonMySQLConnector.__init__(self)
        self.json_fields = None

    def get_ascii(self):
        return self.json_fields[0][self.ASCII]
        
    def get_mysql_regex(self):
        return self.json_fields[0][self.MYSQL_REGEX]
        
    def get_python_regex(self):
        return self.json_fields[0][self.PYTHON_REGEX]
        
    def get_select_restrictor(self):
        return self.json_fields[0][self.SELECT_RESTRICTOR]
        
    def get_substitution_string(self):
        return self.json_fields[0][self.SUBSTITUTION_STRING]
        
    def init_parser(self):
        PythonMySQLConnector.init_parser(self, description='Uses a regex to replace instances of a matching pattern',   
            epilog='Sample invocation: python regex_replacer.py --json_file config.json --user root --db tatoeba --pwd "" --csv_dir c:\\temp')
        self.parser.add_argument('--csv_basename', default='regex_replacer.csv', help='basename of intermediate csv file') 

    def process(self):
        filename = os.path.join(self.parsed.csv_dir, self.parsed.csv_basename)
        self.write_csv(filename)
        self.read_csv(filename)

    def read_csv(self, filename):
        """
        Read a CSV file (id<TAB>text) produced earlier and execute an SQL query to update text.
        """
        self.print_output("\n\nfilename: {0}".format(filename))
        if self.parsed.dry_run:
            self.print_output("---NOT executing these lines---")
{'fraction_non_alphanumeric': 0.11235955056179775, 'fraction_numerical': 0.006029049054535489, 'mean_word_length': 3.3420582986317666, 'pattern_counts': {'":': 30, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1545511', 'n_tokens_mistral': 2261, 'n_tokens_neox': 2171, 'n_words': 673}