<commit_before>#!/usr/bin/env python

"""
This script will be executed at the VM startup time.
- It will download the proxy and panda queue from Google instance metadata
- It will download the pilot wrapper from github and execute it
- It will upload the pilot logs to panda cache
"""

import requests
try:
    import subprocess32 as subprocess
except:
    import subprocess
import os
import sys
import logging
import time
import traceback
import zlib
from threading import Thread

logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s',
                    filename='/tmp/vm_script.log', filemode='w')

global loop
loop = True


def upload_logs(url, log_file_name, destination_name, proxy_path):
    try:

        # open and compress the content of the file
        with open(log_file_name, 'rb') as log_file_object:
            files = {'file': (destination_name, log_file_object.read())}

        cert = [proxy_path, proxy_path]
        # verify = '/etc/grid-security/certificates' # not supported in CernVM - requests.exceptions.SSLError: [Errno 21] Is a directory

        logging.debug('[upload_logs] start')
        res = requests.post(url + '/putFile', files=files, timeout=180, verify=False, cert=cert)
        logging.debug('[upload_logs] finished with code={0} msg={1}'.format(res.status_code, res.text))
        if res.status_code == 200:
            return True
    except:
        err_type, err_value = sys.exc_info()[:2]
        err_messsage = "failed to put with {0}:{1} ".format(err_type, err_value)
        err_messsage += traceback.format_exc()
        logging.debug('[upload_logs] excepted with:\n {0}'.format(err_messsage))

    return False


def contact_harvester(harvester_frontend, data, auth_token, proxy_path):
    try:
        headers = {'Content-Type': 'application/json',
                   'Authorization': 'Bearer {0}'.format(auth_token)}
        cert = [proxy_path, proxy_path]
        #verify = '/etc/grid-security/certificates' # not supported in CernVM - requests.exceptions.SSLError: [Errno 21] Is a directory
        verify = False
        resp = requests.post(harvester_frontend, json=data, headers=headers, cert=cert, verify=verify)
        logging.debug('[contact_harvester] harvester returned: {0}'.format(resp.text))
    except Exception as e:
        # message could not be sent
        logging.debug('[contact_harvester] failed to send message to harvester: {0}'.format(e))
        pass


def heartbeat(harvester_frontend, worker_id, auth_token, proxy_path):
    data = {'methodName': 'heartbeat', 'workerID': worker_id, 'data': None}
    logging.debug('[heartbeat] sending heartbeat to harvester: {0}'.format(data))
    return contact_harvester(harvester_frontend, data, auth_token, proxy_path)


def suicide(harvester_frontend, worker_id, auth_token, proxy_path):
    data = {'methodName': 'killWorker', 'workerID': worker_id, 'data': None}
    logging.debug('[suicide] sending suicide message to harvester: {0}'.format(data))
    return contact_harvester(harvester_frontend, data, auth_token, proxy_path)


def heartbeat_loop(harvester_frontend, worker_id, auth_token, proxy_path):
    while loop:
        heartbeat(harvester_frontend, worker_id, auth_token, proxy_path)
        time.sleep(300)


def get_url(url, headers=None):
    """
    get content from specified URL
    """

    reply = requests.get(url, headers=headers)
    if reply.status_code != 200:
        logging.debug('[get_attribute] Failed to open {0}'.format(url))
        return None
    else:
        return reply.content


def get_configuration():

    # get the proxy certificate and save it
    proxy_path = "/tmp/x509up"
    proxy_string = os.environ.get('proxyContent').replace(",", "\n")
    with open(proxy_path, "w") as proxy_file:
        proxy_file.write(proxy_string)
    os.environ['X509_USER_PROXY'] = proxy_path
    logging.debug('[main] initialized proxy')

    # get the panda site name
    panda_site = os.environ.get('computingSite')
    logging.debug('[main] got panda site: {0}'.format(panda_site))

    # get the panda queue name
    panda_queue = os.environ.get('pandaQueueName')
    logging.debug('[main] got panda queue: {0}'.format(panda_queue))

    # get the harvester frontend URL, where we'll send heartbeats
#    harvester_frontend_url = METADATA_URL.format("harvester_frontend")<commit_msg>Fixed production job cannot find proxy file path problem<commit_after>#!/usr/bin/env python

"""
This script will be executed at the VM startup time.
- It will download the proxy and panda queue from Google instance metadata
- It will download the pilot wrapper from github and execute it
- It will upload the pilot logs to panda cache
"""

import requests
try:
    import subprocess32 as subprocess
except:
    import subprocess
import os
import sys
import logging
import time
import traceback
import zlib
from threading import Thread

logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s',
                    filename='/tmp/vm_script.log', filemode='w')

global loop
loop = True


def upload_logs(url, log_file_name, destination_name, proxy_path):
    try:

        # open and compress the content of the file
        with open(log_file_name, 'rb') as log_file_object:
            files = {'file': (destination_name, log_file_object.read())}

        cert = [proxy_path, proxy_path]
        # verify = '/etc/grid-security/certificates' # not supported in CernVM - requests.exceptions.SSLError: [Errno 21] Is a directory

        logging.debug('[upload_logs] start')
        res = requests.post(url + '/putFile', files=files, timeout=180, verify=False, cert=cert)
        logging.debug('[upload_logs] finished with code={0} msg={1}'.format(res.status_code, res.text))
        if res.status_code == 200:
            return True
    except:
        err_type, err_value = sys.exc_info()[:2]
        err_messsage = "failed to put with {0}:{1} ".format(err_type, err_value)
        err_messsage += traceback.format_exc()
        logging.debug('[upload_logs] excepted with:\n {0}'.format(err_messsage))

    return False


def contact_harvester(harvester_frontend, data, auth_token, proxy_path):
    try:
        headers = {'Content-Type': 'application/json',
                   'Authorization': 'Bearer {0}'.format(auth_token)}
        cert = [proxy_path, proxy_path]
        #verify = '/etc/grid-security/certificates' # not supported in CernVM - requests.exceptions.SSLError: [Errno 21] Is a directory
        verify = False
        resp = requests.post(harvester_frontend, json=data, headers=headers, cert=cert, verify=verify)
        logging.debug('[contact_harvester] harvester returned: {0}'.format(resp.text))
    except Exception as e:
        # message could not be sent
        logging.debug('[contact_harvester] failed to send message to harvester: {0}'.format(e))
        pass


def heartbeat(harvester_frontend, worker_id, auth_token, proxy_path):
    data = {'methodName': 'heartbeat', 'workerID': worker_id, 'data': None}
    logging.debug('[heartbeat] sending heartbeat to harvester: {0}'.format(data))
    return contact_harvester(harvester_frontend, data, auth_token, proxy_path)


def suicide(harvester_frontend, worker_id, auth_token, proxy_path):
    data = {'methodName': 'killWorker', 'workerID': worker_id, 'data': None}
    logging.debug('[suicide] sending suicide message to harvester: {0}'.format(data))
    return contact_harvester(harvester_frontend, data, auth_token, proxy_path)


def heartbeat_loop(harvester_frontend, worker_id, auth_token, proxy_path):
    while loop:
        heartbeat(harvester_frontend, worker_id, auth_token, proxy_path)
        time.sleep(300)


def get_url(url, headers=None):
    """
    get content from specified URL
    """

    reply = requests.get(url, headers=headers)
    if reply.status_code != 200:
        logging.debug('[get_attribute] Failed to open {0}'.format(url))
        return None
    else:
        return reply.content


def get_configuration():

    # get the proxy certificate and save it
    proxy_path = "/tmp/x509up"
    proxy_string = os.environ.get('proxyContent').replace(",", "\n")
    with open(proxy_path, "w") as proxy_file:
        proxy_file.write(proxy_string)
    os.chmod(proxy_path, 0600)
    os.environ['X509_USER_PROXY'] = proxy_path
    logging.debug('[main] initialized proxy')

    # get the panda site name
    panda_site = os.environ.get('computingSite')
    logging.debug('[main] got panda site: {0}'.format(panda_site))

    # get the panda queue name
    panda_queue = os.environ.get('pandaQueueName')
    logging.debug('[main] got panda queue: {0}'.format(panda_queue))

    # get the harvester frontend URL, where we'll send heartbeats
#    harvester_frontend_url = METADATA_URL.format("harvester_frontend")
{'fraction_non_alphanumeric': 0.11385801769108642, 'fraction_numerical': 0.009072352007257882, 'mean_word_length': 3.7516163793103448, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4818092', 'n_tokens_mistral': 2818, 'n_tokens_neox': 2680, 'n_words': 726}