<commit_before>import os
import re
import sys
from time import sleep
import datetime
import calendar
import collections
import json
from simple_salesforce import Salesforce

def log_time_delta(start, end):
    """ Returns microsecond difference between two debug log timestamps
        in the format HH:MM:SS.micro
    """
    dummy_date = datetime.date(2001,1,1)
    dummy_date_next = datetime.date(2001,1,2)
    
    # Split out the parts of the start and end string
    start_parts = re.split(':|\.', start)
    start_parts = [int(part) for part in start_parts]
    start_parts[3] = start_parts[3] * 1000
    t_start = datetime.time(*start_parts)

    end_parts = re.split(':|\.', end)
    end_parts = [int(part) for part in end_parts]
    end_parts[3] = end_parts[3] * 1000
    t_end = datetime.time(*end_parts)

    # Combine with dummy date to do date math
    d_start = datetime.datetime.combine(dummy_date, t_start)
    
    # If end was on the next day, attach to next dummy day
    if start_parts[0] > end_parts[0]:
        d_end = datetime.datetime.combine(dummy_date_next, t_end)
    else:
        d_end = datetime.datetime.combine(dummy_date, t_end)

    delta = d_end - d_start

    return delta.total_seconds()

def parse_log(class_name, log):
    methods = {}
    for method, stats in parse_log_by_method(class_name, log):
        methods[method] = stats
    return methods
    
def parse_log_by_method(class_name, log):
    stats = {}
    in_limits = False
    unit = None
    method = None
    for line in log:
        # Strip newline character
        line = line.strip()

        if line.find('|CODE_UNIT_STARTED|[EXTERNAL]|01p') != -1:
            # Parse the method name
            unit = line.split('|')[-1]
            if not unit.startswith(class_name + '.'):
                # Skip child code units (i.e. triggers, Global Scheduler, etc)
                continue
            method = unit.split('.')[-1]
            start_timestamp = line.split(' ')[0]
            continue

        # NOTE: If we are in a child code unit of the test method, this section
        # will currently set the limits for the child.  However, they will later
        # be overwritten by the test method's limits when they are parsed.  Since
        # nothing is yielded until the end of the test method's code limit, this
        # should be safe for now but should be refactored soon.
        if line.find('|LIMIT_USAGE_FOR_NS|(default)|') != -1:
            # Parse the start of the limits section
            in_limits = True
            continue

        if in_limits and line.find(':') == -1:
            # Parse the end of the limits section
            in_limits = False
            continue

        if in_limits:
            # Parse the limit name, used, and allowed values
            limit, value = line.split(': ')
            used, allowed = value.split(' out of ')
            stats[limit] = {
                'used': used,
                'allowed': allowed,
            }
            continue

        if line.find('|CODE_UNIT_FINISHED|%s.%s' % (class_name, method)) != -1:
            end_timestamp = line.split(' ')[0]
            stats['duration'] = log_time_delta(start_timestamp, end_timestamp)

            # Yield the stats for the method
            yield method, stats
            stats = {}
            in_limits = False
        
        # If debug log size limit was reached, fail gracefully
        if line.find('* MAXIMUM DEBUG LOG SIZE REACHED *') != -1:
            break
        

def run_tests():
    username = os.environ.get('SF_USERNAME')
    password = os.environ.get('SF_PASSWORD')
    serverurl = os.environ.get('SF_SERVERURL')
    test_name_match = os.environ.get('APEX_TEST_NAME_MATCH', '%_TEST')
    test_name_exclude = os.environ.get('APEX_TEST_NAME_EXCLUDE', '')
    namespace = os.environ.get('NAMESPACE', None)
    poll_interval = int(os.environ.get('POLL_INTERVAL', 10))
    debug = os.environ.get('DEBUG_TESTS',False) == 'true'
    debug_logdir = os.environ.get('DEBUG_LOGDIR')
    json_output = os.environ.get('TEST_JSON_OUTPUT', None)
    
    if namespace:
        namespace = "'%s'" % namespace
    else:
        namespace = 'null'
    
    sandbox = False
    if serverurl.find('test.salesforce.com') != -1:
        sandbox = True
    
    sf = Salesforce(username=username, password=password, security_token='', sandbox=sandbox, version='32.0')
    
    # Change base_url to use the tooling api
    sf.base_url = sf.base_url + 'tooling/'
    
    # Split test_name_match by commas to allow multiple class name matching options
    where_name = []
    for pattern in test_name_match.split(','):
        where_name.append("Name LIKE '%s'" % pattern)

    # Add any excludes to the where clause
    for pattern in test_name_exclude.split(','):
        where_name.append("NOT Name LIKE '%s'" % pattern)
   
    # Get all test classes for namespace
    query = "SELECT Id, Name FROM ApexClass WHERE NamespacePrefix = %s and (%s)" % (namespace, ' OR '.join(where_name))

    print "Running Query: %s" % query
    sys.stdout.flush()
<commit_msg>Fix to test name exclude logic in soql query<commit_after>import os
import re
import sys
from time import sleep
import datetime
import calendar
import collections
import json
from simple_salesforce import Salesforce

def log_time_delta(start, end):
    """ Returns microsecond difference between two debug log timestamps
        in the format HH:MM:SS.micro
    """
    dummy_date = datetime.date(2001,1,1)
    dummy_date_next = datetime.date(2001,1,2)
    
    # Split out the parts of the start and end string
    start_parts = re.split(':|\.', start)
    start_parts = [int(part) for part in start_parts]
    start_parts[3] = start_parts[3] * 1000
    t_start = datetime.time(*start_parts)

    end_parts = re.split(':|\.', end)
    end_parts = [int(part) for part in end_parts]
    end_parts[3] = end_parts[3] * 1000
    t_end = datetime.time(*end_parts)

    # Combine with dummy date to do date math
    d_start = datetime.datetime.combine(dummy_date, t_start)
    
    # If end was on the next day, attach to next dummy day
    if start_parts[0] > end_parts[0]:
        d_end = datetime.datetime.combine(dummy_date_next, t_end)
    else:
        d_end = datetime.datetime.combine(dummy_date, t_end)

    delta = d_end - d_start

    return delta.total_seconds()

def parse_log(class_name, log):
    methods = {}
    for method, stats in parse_log_by_method(class_name, log):
        methods[method] = stats
    return methods
    
def parse_log_by_method(class_name, log):
    stats = {}
    in_limits = False
    unit = None
    method = None
    for line in log:
        # Strip newline character
        line = line.strip()

        if line.find('|CODE_UNIT_STARTED|[EXTERNAL]|01p') != -1:
            # Parse the method name
            unit = line.split('|')[-1]
            if not unit.startswith(class_name + '.'):
                # Skip child code units (i.e. triggers, Global Scheduler, etc)
                continue
            method = unit.split('.')[-1]
            start_timestamp = line.split(' ')[0]
            continue

        # NOTE: If we are in a child code unit of the test method, this section
        # will currently set the limits for the child.  However, they will later
        # be overwritten by the test method's limits when they are parsed.  Since
        # nothing is yielded until the end of the test method's code limit, this
        # should be safe for now but should be refactored soon.
        if line.find('|LIMIT_USAGE_FOR_NS|(default)|') != -1:
            # Parse the start of the limits section
            in_limits = True
            continue

        if in_limits and line.find(':') == -1:
            # Parse the end of the limits section
            in_limits = False
            continue

        if in_limits:
            # Parse the limit name, used, and allowed values
            limit, value = line.split(': ')
            used, allowed = value.split(' out of ')
            stats[limit] = {
                'used': used,
                'allowed': allowed,
            }
            continue

        if line.find('|CODE_UNIT_FINISHED|%s.%s' % (class_name, method)) != -1:
            end_timestamp = line.split(' ')[0]
            stats['duration'] = log_time_delta(start_timestamp, end_timestamp)

            # Yield the stats for the method
            yield method, stats
            stats = {}
            in_limits = False
        
        # If debug log size limit was reached, fail gracefully
        if line.find('* MAXIMUM DEBUG LOG SIZE REACHED *') != -1:
            break
        

def run_tests():
    username = os.environ.get('SF_USERNAME')
    password = <PASSWORD>('SF_PASSWORD')
    serverurl = os.environ.get('SF_SERVERURL')
    test_name_match = os.environ.get('APEX_TEST_NAME_MATCH', '%_TEST')
    test_name_exclude = os.environ.get('APEX_TEST_NAME_EXCLUDE', '')
    namespace = os.environ.get('NAMESPACE', None)
    poll_interval = int(os.environ.get('POLL_INTERVAL', 10))
    debug = os.environ.get('DEBUG_TESTS',False) == 'true'
    debug_logdir = os.environ.get('DEBUG_LOGDIR')
    json_output = os.environ.get('TEST_JSON_OUTPUT', None)
    
    if namespace:
        namespace = "'%s'" % namespace
    else:
        namespace = 'null'
    
    sandbox = False
    if serverurl.find('test.salesforce.com') != -1:
        sandbox = True
    
    sf = Salesforce(username=username, password=password, security_token='', sandbox=sandbox, version='32.0')
    
    # Change base_url to use the tooling api
    sf.base_url = sf.base_url + 'tooling/'
    
    # Split test_name_match by commas to allow multiple class name matching options
    where_name = []
    for pattern in test_name_match.split(','):
        if pattern:
            where_name.append("Name LIKE '%s'" % pattern)

    # Add any excludes to the where clause
    where_exclude = []
    for pattern in test_name_exclude.split(','):
        if pattern:
            where_exclude.append("(NOT Name LIKE '%s')" % pattern)
   
    # Get all test classes for namespace
    query = "SELECT Id, Name FROM ApexClass WHERE NamespacePrefix = %s" % namespace
    if where_name:
        query += " AND (%s)" % ' OR '.join(where_name)
    if where_exclude:
        query += " AND %s" % ' AND '.join(where_exclude)

    print "Running Query: %s" % query
    sys.stdout.flush()

{'fraction_non_alphanumeric': 0.10526315789473684, 'fraction_numerical': 0.008259700345754899, 'mean_word_length': 2.414098360655738, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3583234', 'n_tokens_mistral': 3206, 'n_tokens_neox': 3108, 'n_words': 929}