<commit_before># Python 3 module


from email.mime.text import MIMEText
import configparser
import os
import smtplib
import sys


def SendEmail(subject, msg, sender="<EMAIL>",
              recipient="<EMAIL>"):
    config = LoadConfigFile(sys.argv[0][:-2] + "conf")
    try:
        server_address  = config["SMTPServer"]["server_address"]
        server_user     = config["SMTPServer"]["server_user"]
        server_password = config["SMTPServer"]["server_password"]
    except Exception as e:
        print("failed to read config file! (" + str(e) + ")", file=sys.stderr)
        sys.exit(-1)

    message = MIMEText(msg)
    message["Subject"] = subject
    message["From"] = sender
    message["To"] = recipient
    server = smtplib.SMTP(server_address)
    try:
        server.ehlo()
        server.starttls()
        server.login(server_user, server_password)
        server.sendmail(sender, [recipient], message.as_string())
    except Exception as e:
        print("Failed to send your email: " + str(e), file=sys.stderr)
        sys.exit(-1)
    server.quit()


def Error(msg):
    print(sys.argv[0] + ": " + msg, file=sys.stderr)
    SendEmail("SWB FTP Failed!", msg)
    sys.exit(1)


# Fails if "source" does not exist or if "link_name" exists and is not a symlink.
# Calls Error() upon failure and aborts the program.
def SafeSymlink(source, link_name):
    try:
        os.lstat(source) # Throws an exception if "source" does not exist.
        if os.access(link_name, os.F_OK) != 0:
            if os.path.islink(link_name):
                os.unlink(link_name)
            else:
                Error("in SafeSymlink: trying to create a symlink to \"" + link_name
                      + "\" which is an existing non-symlink file!")
        os.symlink(source, link_name)
    except Exception as e:
        Error("os.symlink() failed: " + str(e))


def LoadConfigFile(path):
    try:
        if not os.access(path, os.R_OK):
            Error("can't open \"" + path + "\" for reading!")
        config = configparser.ConfigParser()
        config.read(path)
        return config
    except Exception as e:
        Error("failed to load the config file from \"" + path + "\"! (" + str(e) + ")")<commit_msg>Added new FoundNewBSZDataFile() function and a way to specify a default email sender for SendEmail().<commit_after># Python 3 module


from email.mime.text import MIMEText
import configparser
import os
import smtplib
import struct
import sys


default_email_sender = "<EMAIL>"


def SendEmail(subject, msg, sender=None, recipient="<EMAIL>"):
    if sender is None:
        sender = default_email_sender
    config = LoadConfigFile(sys.argv[0][:-2] + "conf")
    try:
        server_address  = config["SMTPServer"]["server_address"]
        server_user     = config["SMTPServer"]["server_user"]
        server_password = config["SMTPServer"]["server_password"]
    except Exception as e:
        print("failed to read config file! (" + str(e) + ")", file=sys.stderr)
        sys.exit(-1)

    message = MIMEText(msg)
    message["Subject"] = subject
    message["From"] = sender
    message["To"] = recipient
    server = smtplib.SMTP(server_address)
    try:
        server.ehlo()
        server.starttls()
        server.login(server_user, server_password)
        server.sendmail(sender, [recipient], message.as_string())
    except Exception as e:
        print("Failed to send your email: " + str(e), file=sys.stderr)
        sys.exit(-1)
    server.quit()


def Error(msg):
    print(sys.argv[0] + ": " + msg, file=sys.stderr)
    SendEmail("SWB FTP Failed!", msg)
    sys.exit(1)


# Fails if "source" does not exist or if "link_name" exists and is not a symlink.
# Calls Error() upon failure and aborts the program.
def SafeSymlink(source, link_name):
    try:
        os.lstat(source) # Throws an exception if "source" does not exist.
        if os.access(link_name, os.F_OK) != 0:
            if os.path.islink(link_name):
                os.unlink(link_name)
            else:
                Error("in SafeSymlink: trying to create a symlink to \"" + link_name
                      + "\" which is an existing non-symlink file!")
        os.symlink(source, link_name)
    except Exception as e:
        Error("os.symlink() failed: " + str(e))


def LoadConfigFile(path):
    try:
        if not os.access(path, os.R_OK):
            Error("can't open \"" + path + "\" for reading!")
        config = configparser.ConfigParser()
        config.read(path)
        return config
    except Exception as e:
        Error("failed to load the config file from \"" + path + "\"! (" + str(e) + ")")


# This function looks for symlinks named "XXX-current-YYY" where "YYY" may be the empty string.  If found,
# it reads the creation time of the link target.  Next it looks for a file named "XXX-YYY.timestamp".  If
# the timestamp file does not exist, it assumes it found a new file and creates a new timestamp file with
# the creation time of the original link target and returns True.  If, on the other hand, a timestamp file
# was found, it reads a second timestamp from the timestamp file and compares it against the first
# timestamp.  If the timestamp from the timestamp file is older than the first timestamp, it updates the
# timestamp file with the newer timestamp and returns True, otherwise it returns False.
def FoundNewBSZDataFile(link_filename):
    try:
        statinfo = os.stat(link_filename)
    except FileNotFoundError as e:
        util.Error("Symlink \"" + link_filename + "\" is missing or dangling!")
    new_timestamp = statinfo.st_ctime
    timestamp_filename = sys.argv[0][:-2] + "timestamp"
    if not os.path.exists(timestamp_filename):
        with open(timestamp_filename, "wb") as timestamp_file:
            timestamp_file.write(struct.pack('d', new_timestamp))
        return True
    else:
        with open(timestamp_filename, "rb") as timestamp_file:
            (old_timestamp, ) = struct.unpack('d', timestamp_file.read())
        if (old_timestamp < new_timestamp):
            with open(timestamp_filename, "wb") as timestamp_file:
                timestamp_file.write(struct.pack('f', new_timestamp))
            return True
        else:
            return False
{'fraction_non_alphanumeric': 0.1158908507223114, 'fraction_numerical': 0.0028892455858747996, 'mean_word_length': 2.785540704738761, 'pattern_counts': {'":': 2, '<': 8, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2152109', 'n_tokens_mistral': 1869, 'n_tokens_neox': 1861, 'n_words': 564}