<commit_before># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import division, absolute_import, with_statement, print_function, unicode_literals
import os
import sys
import time
from common import parseDir

__authors__ = "<NAME>, <NAME>"
__license__ = "MPL 2.0"

TYPE = "Pattern Scan"
NAME = "Yara"
DEFAULTCONF = {"ruledir":os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'etc', 'yarasigs'),
    "fileextensions":[".yar", ".yara", ".sig"],
    "ignore-tags":["TLPRED"],
    'ENABLED': True
    }
    
try:
    import yara
except:
    print("yara-python module not installed...")
    yara = False
    
def check(conf=DEFAULTCONF):
    if not conf['ENABLED']:
        return False
    if not yara:
        return False
    return True

def scan(filelist, conf=DEFAULTCONF):
    ruleDir = conf["ruledir"]
    extlist = conf["fileextensions"]
    ruleset = {}
    rules = parseDir(ruleDir, recursive=True)
    for r in rules:
        for ext in extlist:
            if r.endswith(ext):
                ruleset[r] = os.path.join(ruleDir, r)
                break
    
    #Ran into a weird issue with file locking, this fixes it
    goodtogo = False
    i = 0
    yararules = None
    while not goodtogo and i < 5:
        try:
            yararules = yara.compile(filepaths=ruleset)
            goodtogo = True
        except Exception as e:
            print('yara:', e)
            time.sleep(3)
            i += 1

    matches = []
    for m in filelist:
        #Ran into a weird issue with file locking, this fixes it
        goodtogo = False
        i = 0
        while not goodtogo and i < 5:
            try:<commit_msg>Yara module now removes signature files that fail to compile<commit_after># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import division, absolute_import, with_statement, print_function, unicode_literals
import os
import sys
import time
from common import parseDir

__authors__ = "<NAME>, <NAME>"
__license__ = "MPL 2.0"

TYPE = "Pattern Scan"
NAME = "Yara"
DEFAULTCONF = {"ruledir":os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'etc', 'yarasigs'),
    "fileextensions":[".yar", ".yara", ".sig"],
    "ignore-tags":["TLPRED"],
    'ENABLED': True
    }
    
try:
    import yara
except:
    print("yara-python module not installed...")
    yara = False
    
def check(conf=DEFAULTCONF):
    if not conf['ENABLED']:
        return False
    if not yara:
        return False
    return True

def scan(filelist, conf=DEFAULTCONF):
    ruleDir = conf["ruledir"]
    extlist = conf["fileextensions"]
    ruleset = {}
    rules = parseDir(ruleDir, recursive=True)
    for r in rules:
        for ext in extlist:
            if r.endswith(ext):
                ruleset[r] = os.path.join(ruleDir, r)
                break
    
    #Ran into a weird issue with file locking, this fixes it
    goodtogo = False
    i = 0
    yararules = None
    while not goodtogo:
        try:
            yararules = yara.compile(filepaths=ruleset)
            goodtogo = True
        except yara.SyntaxError as e:
            bad_file = e.message.split('(')[0]
            del ruleset[bad_file]
            print(e)

    matches = []
    for m in filelist:
        #Ran into a weird issue with file locking, this fixes it
        goodtogo = False
        i = 0
        while not goodtogo and i < 5:
            try:
{'fraction_non_alphanumeric': 0.10622113556098886, 'fraction_numerical': 0.006519967400162999, 'mean_word_length': 2.3841911764705883, 'pattern_counts': {'":': 6, '<': 10, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '766154', 'n_tokens_mistral': 1191, 'n_tokens_neox': 1147, 'n_words': 379}