<commit_before>#!/usr/bin/env python

# Copyright (c) 2006-2012 sqlmap developers (http://sqlmap.org/)
# See the file 'doc/COPYING' for copying permission

# Removes duplicate entries in wordlist like files
import codecs
import os
import re
import smtplib
import subprocess
import time

from email.mime.text import MIMEText

TIME = time.strftime("%H:%M:%S", time.gmtime())
DATE = time.strftime("%d-%m-%Y", time.gmtime())

SMTP_SERVER = "10.129.121.194"
SMTP_PORT = 25
SMTP_TIMEOUT = 30
FROM = "<EMAIL>"
TO = "<EMAIL>"
SUBJECT = "Regression test results - started at %s on %s" % (TIME, DATE)
CONTENT = ""

command_line = "cd ../../ ; rm -f $REGRESSION_FILE ; python sqlmap.py --live-test --run-case 'Invalid logic'"
proc = subprocess.Popen(command_line, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
proc.wait()
stdout, stderr = proc.communicate()
failed_tests = re.findall("running live test case: (.+?) \(\d+\/\d+\)[\r]*\n.+test failed (.*?)at parsing item: (.+) \- scan folder is (\/.+)[\r]*\n", stdout, re.I | re.M)

for failed_test in failed_tests:
    title = failed_test[0]
    traceback = failed_test[1] or None
    parse = failed_test[2]
    output_folder = failed_test[3]

    console_output_fd = codecs.open(os.path.join(output_folder, "console_output"), "rb", "utf8")
    console_output = console_output_fd.read()
    console_output_fd.close()

    log_fd = codecs.open(os.path.join(output_folder, "debiandev", "log"), "rb", "utf8")
    log = log_fd.read()
    log_fd.close()

    if traceback:
        traceback_fd = codecs.open(os.path.join(output_folder, "traceback"), "rb", "utf8")
        traceback = traceback_fd.read()
        traceback_fd.close()

    CONTENT += "Failed test case '%s' at parsing: %s:\n\n" % (title, parse)
    CONTENT += "### LOG FILE:\n\n"
    CONTENT += "%s\n" % log
    CONTENT += "### CONSOLE OUTPUT\n\n"
    CONTENT += "%s\n" % str(console_output)

    if traceback:
        CONTENT += "### TRACEBACK:\n"
        CONTENT += "%s\n" % str(traceback)

    CONTENT += "\n#######################################\n\n"

if CONTENT:
    msg = MIMEText(CONTENT)
    msg["Subject"] = SUBJECT
    msg["From"] = FROM
    msg["To"] = TO

    s = smtplib.SMTP(host=SMTP_SERVER, port=SMTP_PORT, timeout=SMTP_TIMEOUT)
    #s.set_debuglevel(1)
    s.sendmail(FROM, TO, msg)
    s.quit()<commit_msg>more enhancements - issue #311<commit_after>#!/usr/bin/env python

# Copyright (c) 2006-2012 sqlmap developers (http://sqlmap.org/)
# See the file 'doc/COPYING' for copying permission

# Removes duplicate entries in wordlist like files
import codecs
import os
import re
import smtplib
import subprocess
import time

from email.mime.text import MIMEText

TIME = time.strftime("%H:%M:%S %d-%m-%Y", time.gmtime())

SMTP_SERVER = "127.0.0.1"
SMTP_PORT = 25
SMTP_TIMEOUT = 30
FROM = "<EMAIL>"
TO = "<EMAIL>"
SUBJECT = "Regression test results on on %s" % TIME
CONTENT = ""
TEST_COUNTS = []

command_line = "cd ../../ ; rm -f $REGRESSION_FILE ; python sqlmap.py --live-test --run-case 'Invalid logic'"
proc = subprocess.Popen(command_line, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
proc.wait()
stdout, stderr = proc.communicate()
failed_tests = re.findall("running live test case: (.+?) \((\d+)\/\d+\)[\r]*\n(.+?)test failed (.*?)at parsing item: (.+) \- scan folder is (\/.+)[\r]*\n", stdout, re.I | re.M)

for failed_test in failed_tests:
    title = failed_test[0]
    test_count = failed_test[1]
    error = True if "the test did not identify the SQL injection" in failed_test[2] else False
    traceback = failed_test[2] or None
    parse = failed_test[3]
    output_folder = failed_test[4]

    TEST_COUNTS.append(test_count)

    console_output_fd = codecs.open(os.path.join(output_folder, "console_output"), "rb", "utf8")
    console_output = console_output_fd.read()
    console_output_fd.close()

    if error is False:
        log_fd = codecs.open(os.path.join(output_folder, "debiandev", "log"), "rb", "utf8")
        log = log_fd.read()
        log_fd.close()

    if traceback:
        traceback_fd = codecs.open(os.path.join(output_folder, "traceback"), "rb", "utf8")
        traceback = traceback_fd.read()
        traceback_fd.close()

    CONTENT += "Failed test case '%s' at parsing: %s:\n\n" % (title, parse)

    if error is False:
        CONTENT += "### LOG FILE:\n\n"
        CONTENT += "%s\n" % log
    else:
        CONTENT += "### CONSOLE OUTPUT\n\n"
        CONTENT += "%s\n" % str(console_output)

    if traceback:
        CONTENT += "### TRACEBACK:\n"
        CONTENT += "%s\n" % str(traceback)

    CONTENT += "\n#######################################\n\n"

if CONTENT:
    SUBJECT += " (%s)" % ", ".join("#%d" % count for count in TEST_COUNTS)
    msg = MIMEText(CONTENT)
    msg["Subject"] = SUBJECT
    msg["From"] = FROM
    msg["To"] = TO

    s = smtplib.SMTP(host=SMTP_SERVER, port=SMTP_PORT, timeout=SMTP_TIMEOUT)
    #s.set_debuglevel(1)
    s.sendmail(FROM, TO, msg.as_string())
    s.quit()
{'fraction_non_alphanumeric': 0.1792093116596428, 'fraction_numerical': 0.01244230383303231, 'mean_word_length': 3.5063291139240507, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4868620', 'n_tokens_mistral': 1876, 'n_tokens_neox': 1779, 'n_words': 421}