<commit_before>#!/usr/bin/env python3
# -*-coding:UTF-8 -*

"""
The Regex Module
================

Search for regular expressions stored in a flat file + tag.
The flat file is automatically reloaded when the MTIME changed.

It uses the file 'packagess/regex.cfg'. Format:
Tag||Regex

<NAME> <<EMAIL>>

"""

import time
import os
import re
import signal
from pubsublogger import publisher

#from bin.packages import Paste
#from bin.Helper import Process

from packages import Paste
from Helper import Process

class TimeoutException(Exception):
    pass

def timeout_handler(signum, frame):
    raise TimeoutException

signal.signal(signal.SIGALRM, timeout_handler)

# Change the path to your preferred one
regexConfig = 'packages/regex.cfg'

regexes     = []

def load_regex(force = False):
    '''
    Load regexes from the config file and validate them
    If 'True' passed as argument, force to reload
    '''

    lregexes = regexes
    validate_regex = False

    try:
        stats = os.stat(regexConfig)
        mtime = int(stats.st_mtime)
        if mtime > time.time()-60 or force == True:
            # Regex config changed, reload the file
            print('Loading regular expressions')
            with open(regexConfig) as f:
                lines = f.readlines()
            lines = [x.strip() for x in lines] 
            validate_regex = True
    except:
        print('Cannot read {}'.format(regexConfig))
        return []

    if validate_regex:
        # Validate regexes read from the file
        line=1
        lregexes = []
        for l in lines:
            # Skip comments and empty lines
            if len(l) > 0:
                if l[0] == '#':
                    continue
                try:
                    re.compile(l.split('||')[1])
                except:  
                    print('Ignored line {}: Syntax error in "{}"'.format(line, regexConfig))
                    continue
                line += 1
                lregexes.append(l)
        print('DEBUG: regexes:')
        print(lregexes)
    return lregexes

def search_regex(paste):
    content = paste.get_p_content()
    find = False
    global regexes

    regexes = load_regex(False)<commit_msg>Fixed line counter (count all lines)<commit_after>#!/usr/bin/env python3
# -*-coding:UTF-8 -*

"""
The Regex Module
================

Search for regular expressions stored in a flat file + tag.
The flat file is automatically reloaded when the MTIME changed.

It uses the file 'packagess/regex.cfg'. Format:
Tag||Regex

<NAME> <<EMAIL>>

"""

import time
import os
import re
import signal
from pubsublogger import publisher

#from bin.packages import Paste
#from bin.Helper import Process

from packages import Paste
from Helper import Process

class TimeoutException(Exception):
    pass

def timeout_handler(signum, frame):
    raise TimeoutException

signal.signal(signal.SIGALRM, timeout_handler)

# Change the path to your preferred one
regexConfig = 'packages/regex.cfg'

regexes     = []

def load_regex(force = False):
    '''
    Load regexes from the config file and validate them
    If 'True' passed as argument, force to reload
    '''

    lregexes = regexes
    validate_regex = False

    try:
        stats = os.stat(regexConfig)
        mtime = int(stats.st_mtime)
        if mtime > time.time()-60 or force == True:
            # Regex config changed, reload the file
            print('Loading regular expressions')
            with open(regexConfig) as f:
                lines = f.readlines()
            lines = [x.strip() for x in lines] 
            validate_regex = True
    except:
        print('Cannot read {}'.format(regexConfig))
        return []

    if validate_regex:
        # Validate regexes read from the file
        line=1
        lregexes = []
        for l in lines:
            # Skip comments and empty lines
            if len(l) > 0:
                if l[0] == '#':
                    continue
                try:
                    re.compile(l.split('||')[1])
                except:  
                    print('Ignored line {}: Syntax error in "{}"'.format(line, regexConfig))
                    continue
                lregexes.append(l)
            line += 1
        print('DEBUG: regexes:')
        print(lregexes)
    return lregexes

def search_regex(paste):
    content = paste.get_p_content()
    find = False
    global regexes

    regexes = load_regex(False)
{'fraction_non_alphanumeric': 0.098486559747007, 'fraction_numerical': 0.0040659588886379034, 'mean_word_length': 2.1448863636363638, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '287564', 'n_tokens_mistral': 1265, 'n_tokens_neox': 1221, 'n_words': 404}