<commit_before>import numpy as np
import urllib2

def len_(L):
    try:
        return np.array(L).shape
    except Exception:
        comb = '' + str(len(L)) + "*"
        if len(L) > 0:
            i = L[0]
            s = str(len_(i))
            if s <> "()":
                comb += s + ' '
        return comb

def send_mail(subject, message, attachment_path = None):
    try:
        import smtplib, os
        from email.mime.multipart import MIMEMultipart
        from email.mime.text import MIMEText
        from email.mime.base import MIMEBase
        from email import encoders

        from mail_secrets import fromaddr, toaddr, passwrd

        msg = MIMEMultipart()

        msg['From'] = fromaddr
        msg['To'] = toaddr
        msg['Subject'] = subject #"SUBJECT OF THE EMAIL"

        body = message #"TEXT YOU WANT TO SEND"

        msg.attach(MIMEText(body, 'plain'))

        if attachment_path is not None:

            attachment_filename = os.path.basename(attachment_path)
            attachment = open(attachment_path, "rb") #"PATH OF THE FILE"

            part = MIMEBase('application', 'octet-stream')
            part.set_payload((attachment).read())
            encoders.encode_base64(part)
            part.add_header('Content-Disposition', "attachment; filename= %s" % attachment_filename)

            msg.attach(part)

        server = smtplib.SMTP('smtp.gmail.com', 587)
        server.starttls()
        server.login(fromaddr, passwrd)
        text = msg.as_string()
        server.sendmail(fromaddr, toaddr, text)
        server.quit()
        return True
    except Exception as inst:
        print "Exception when trying to send mail."
        print type(inst)     # the exception instance
        print inst.args      # arguments stored in .args
        print inst           # __str__ allows args to be printed directly
        return False


#send_mail('automatic mail', 'test', '/home/ekmek/Vitek/Logs/Number_of_FC_blocks_test/graph_together_Number_of_FC_blocks_test.png')

def save_job_report_page(folder_path, job_id, cut = True):
    try:

        url = 'https://metavo.metacentrum.cz/pbsmon2/job/' + job_id

        response = urllib2.urlopen(url)
        webContent = response.read()

        if cut:
            substr = 'Job '+job_id
            index = webContent.replace(substr,'ignore', 1).find(substr)
            offset = 18
            webContent = webContent[index+offset:-1]

        #print(webContent)

        f = open(folder_path + job_id+'.html', 'w')
        f.write(webContent)
        f.close()
        return True
    except Exception as inst:
        print "Exception in the report page downloading from Metacentrum"
        print type(inst)     # the exception instance
        print inst.args      # arguments stored in .args
        print inst           # __str__ allows args to be printed directly
        return False


#save_job_report_page(folder_path='',job_id='1398409.arien-pro.ics.muni.cz')

import os
def file_exists_and_accesible(PATH):
    return os.path.isfile(PATH) and os.access(PATH, os.R_OK)<commit_msg>md5 function for arrays (to check image arrays while debugging)<commit_after>import numpy as np
import urllib2

def len_(L):
    try:
        return np.array(L).shape
    except Exception:
        comb = '' + str(len(L)) + "*"
        if len(L) > 0:
            i = L[0]
            s = str(len_(i))
            if s <> "()":
                comb += s + ' '
        return comb

def send_mail(subject, message, attachment_path = None):
    try:
        import smtplib, os
        from email.mime.multipart import MIMEMultipart
        from email.mime.text import MIMEText
        from email.mime.base import MIMEBase
        from email import encoders

        from mail_secrets import fromaddr, toaddr, passwrd

        msg = MIMEMultipart()

        msg['From'] = fromaddr
        msg['To'] = toaddr
        msg['Subject'] = subject #"SUBJECT OF THE EMAIL"

        body = message #"TEXT YOU WANT TO SEND"

        msg.attach(MIMEText(body, 'plain'))

        if attachment_path is not None:

            attachment_filename = os.path.basename(attachment_path)
            attachment = open(attachment_path, "rb") #"PATH OF THE FILE"

            part = MIMEBase('application', 'octet-stream')
            part.set_payload((attachment).read())
            encoders.encode_base64(part)
            part.add_header('Content-Disposition', "attachment; filename= %s" % attachment_filename)

            msg.attach(part)

        server = smtplib.SMTP('smtp.gmail.com', 587)
        server.starttls()
        server.login(fromaddr, passwrd)
        text = msg.as_string()
        server.sendmail(fromaddr, toaddr, text)
        server.quit()
        return True
    except Exception as inst:
        print "Exception when trying to send mail."
        print type(inst)     # the exception instance
        print inst.args      # arguments stored in .args
        print inst           # __str__ allows args to be printed directly
        return False


#send_mail('automatic mail', 'test', '/home/ekmek/Vitek/Logs/Number_of_FC_blocks_test/graph_together_Number_of_FC_blocks_test.png')

def save_job_report_page(folder_path, job_id, cut = True):
    try:

        url = 'https://metavo.metacentrum.cz/pbsmon2/job/' + job_id

        response = urllib2.urlopen(url)
        webContent = response.read()

        if cut:
            substr = 'Job '+job_id
            index = webContent.replace(substr,'ignore', 1).find(substr)
            offset = 18
            webContent = webContent[index+offset:-1]

        #print(webContent)

        f = open(folder_path + job_id+'.html', 'w')
        f.write(webContent)
        f.close()
        return True
    except Exception as inst:
        print "Exception in the report page downloading from Metacentrum"
        print type(inst)     # the exception instance
        print inst.args      # arguments stored in .args
        print inst           # __str__ allows args to be printed directly
        return False


#save_job_report_page(folder_path='',job_id='1398409.arien-pro.ics.muni.cz')

import os
def file_exists_and_accesible(PATH):
    return os.path.isfile(PATH) and os.access(PATH, os.R_OK)

def array_md5(arr):
    import hashlib
    import cPickle as pickle
    data_pickle = pickle.dumps(arr)
    data_md5 = hashlib.md5(data_pickle).hexdigest()

    return data_md5
{'fraction_non_alphanumeric': 0.09981285090455397, 'fraction_numerical': 0.007330006238303182, 'mean_word_length': 2.2669383596535915, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '710830', 'n_tokens_mistral': 1980, 'n_tokens_neox': 1897, 'n_words': 492}