<commit_before># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: <NAME> (<EMAIL>)
#
from __future__ import division
from __future__ import unicode_literals

import os
from subprocess import Popen, PIPE
from tempfile import NamedTemporaryFile, mkdtemp
from zipfile import ZipFile, BadZipfile

from activedata_etl import etl2key
from activedata_etl.parse_lcov import parse_lcov_coverage
from activedata_etl.transforms import EtlHeadGenerator
from mo_dots import wrap, set_default
from mo_files import File
from mo_json import json2value
from mo_logs import Log, machine_metadata
from mo_threads import Process
from mo_times import Timer, Date
from pyLibrary.env import http

ACTIVE_DATA_QUERY = "https://activedata.allizom.org/query"
RETRY = {"times": 3, "sleep": 5}
DEBUG = True


def process(source_key, source, destination, resources, please_stop=None):
    """
    This transform will turn a pulse message containing info about a gcov artifact (gcda or gcno file) on taskcluster
    into a list of records of method coverages. Each record represents a method in a source file, given a test.

    :param source_key: The key of the file containing the normalized task cluster messages
    :param source: The file contents, a cr-delimited list of normalized task cluster messages
    :param destination: The destination for the transformed data
    :param resources: not used
    :param please_stop: The stop signal to stop the current thread
    :return: The list of keys of files in the destination bucket
    """
    keys = []
    etl_header_gen = EtlHeadGenerator(source_key)
    for msg_line_index, msg_line in enumerate(list(source.read_lines())): #readline() for local
        # Enter once collected artifacts
        if please_stop:
            Log.error("Shutdown detected. Stopping job ETL.")

        try:
            task_cluster_record = json2value(msg_line)
            # SCRUB PROPERTIES WE DO NOT WANT
            task_cluster_record.action.timings = None
            task_cluster_record.action.etl = None
            task_cluster_record.task.runs = None
            task_cluster_record.task.tags = None
            task_cluster_record.task.env = None
        except Exception, e:
            if "JSON string is only whitespace" in e:
                continue
            else:
                Log.error("unexpected JSON decoding problem", cause=e)
        artifacts, task_cluster_record.task.artifacts = task_cluster_record.task.artifacts, None

        Log.note("{{id}}: {{num}} artifacts", id=task_cluster_record.task.id, num=len(artifacts))
        Log.note("-- Enter Try --")
        try: # TODO rm
            for artifact in artifacts:
                Log.note("{{name}}", name=artifact.name)
                if artifact.name.find("gcda") != -1:
                    keys.extend(process_gcda_artifact(source_key, resources, destination, etl_header_gen, task_cluster_record, artifact))
        except Exception as e:
            import traceback
            Log.note(traceback.format_exc())
    return keys


def process_gcda_artifact(source_key, resources, destination, etl_header_gen, task_cluster_record, gcda_artifact):
    """
    Processes a gcda artifact by downloading any gcno files for it and running lcov on them individually.
    The lcov results are then processed and converted to the standard ccov format.
    TODO this needs to coordinate new ccov json files to add to the s3 bucket. Return?
    """
    Log.note("Processing gcda artifact {{artifact}}", artifact=gcda_artifact.name)

    tmpdir = mkdtemp()
    Log.note('Using temp dir: {{dir}}', dir=tmpdir)

    ccov = File(tmpdir + '/ccov').delete()
    out = File(tmpdir + "/out").delete()

    try:
        Log.note('Fetching gcda artifact: {{url}}', url=gcda_artifact.url)
        gcda_file = download_file(gcda_artifact.url)
        #gcda_file = 'tests/resources/ccov/code-coverage.zip'
        Log.note('Extracting gcda files to {{dir}}/ccov', dir=tmpdir)
        ZipFile(gcda_file).extractall('%s/ccov' % tmpdir)
    except BadZipfile:
        Log.note('Bad zip file for gcda artifact: {{url}}', url=gcda_artifact.url)
        return []


    parent_etl = task_cluster_record.etl
    file_obj = group_to_gcno_artifacts(task_cluster_record.task.group.id)

    # chop some not-needed, and verbose, properties from tc record
    task_cluster_record.etl = None
    task_cluster_record.action.timings = None
    task_cluster_record.action.etl = None
    task_cluster_record.task.artifacts = None
    task_cluster_record.task.runs = None

    remove_files_recursively('%s/ccov' % tmpdir, 'gcno')

    Log.note('Downloading gcno artifact {{file}}', file=file_obj.url)
    _, file_etl = etl_header_gen.next(source_etl=parent_etl, url=gcda_artifact.url)

    etl_key = etl2key(file_etl)
    Log.note('GCNO records will be attached to etl_key: {{etl_key}}', etl_key=etl_key)

    gcno_file = download_file(file_obj.url)

    Log.note('Extracting gcno files to {{dir}}/ccov', dir=tmpdir)
    ZipFile(gcno_file).extractall('%s/ccov' % tmpdir)

    process_directory('%s/ccov' % tmpdir, destination, task_cluster_record, file_etl)
    File(tmpdir).delete()

    keys = [etl_key]
    return keys


def process_directory(source_dir, destination, task_cluster_record, file_etl):
    records = []
    with Timer("Processing LCOV directory {{lcov_directory}}", param={"lcov_directory": source_dir}):
        lcov_coverage = run_lcov_on_directory(source_dir)
<commit_msg>scrub task as late as possible<commit_after># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: <NAME> (<EMAIL>)
#
from __future__ import division
from __future__ import unicode_literals

import os
from subprocess import Popen, PIPE
from tempfile import NamedTemporaryFile, mkdtemp
from zipfile import ZipFile, BadZipfile

from activedata_etl import etl2key
from activedata_etl.imports.task import minimize_task
from activedata_etl.parse_lcov import parse_lcov_coverage
from activedata_etl.transforms import EtlHeadGenerator
from mo_dots import wrap, set_default
from mo_files import File
from mo_json import json2value
from mo_logs import Log, machine_metadata
from mo_threads import Process
from mo_times import Timer, Date
from pyLibrary.env import http

ACTIVE_DATA_QUERY = "https://activedata.allizom.org/query"
RETRY = {"times": 3, "sleep": 5}
DEBUG = True


def process(source_key, source, destination, resources, please_stop=None):
    """
    This transform will turn a pulse message containing info about a gcov artifact (gcda or gcno file) on taskcluster
    into a list of records of method coverages. Each record represents a method in a source file, given a test.

    :param source_key: The key of the file containing the normalized task cluster messages
    :param source: The file contents, a cr-delimited list of normalized task cluster messages
    :param destination: The destination for the transformed data
    :param resources: not used
    :param please_stop: The stop signal to stop the current thread
    :return: The list of keys of files in the destination bucket
    """
    keys = []
    etl_header_gen = EtlHeadGenerator(source_key)
    for msg_line_index, msg_line in enumerate(list(source.read_lines())): #readline() for local
        # Enter once collected artifacts
        if please_stop:
            Log.error("Shutdown detected. Stopping job ETL.")

        try:
            task_cluster_record = json2value(msg_line)
        except Exception, e:
            if "JSON string is only whitespace" in e:
                continue
            else:
                Log.error("unexpected JSON decoding problem", cause=e)
        artifacts, task_cluster_record.task.artifacts = task_cluster_record.task.artifacts, None

        Log.note("{{id}}: {{num}} artifacts", id=task_cluster_record.task.id, num=len(artifacts))
        Log.note("-- Enter Try --")
        try: # TODO rm
            for artifact in artifacts:
                Log.note("{{name}}", name=artifact.name)
                if artifact.name.find("gcda") != -1:
                    keys.extend(process_gcda_artifact(source_key, resources, destination, etl_header_gen, task_cluster_record, artifact))
        except Exception as e:
            import traceback
            Log.note(traceback.format_exc())
    return keys


def process_gcda_artifact(source_key, resources, destination, etl_header_gen, task_cluster_record, gcda_artifact):
    """
    Processes a gcda artifact by downloading any gcno files for it and running lcov on them individually.
    The lcov results are then processed and converted to the standard ccov format.
    TODO this needs to coordinate new ccov json files to add to the s3 bucket. Return?
    """
    Log.note("Processing gcda artifact {{artifact}}", artifact=gcda_artifact.name)

    tmpdir = mkdtemp()
    Log.note('Using temp dir: {{dir}}', dir=tmpdir)

    ccov = File(tmpdir + '/ccov').delete()
    out = File(tmpdir + "/out").delete()

    try:
        Log.note('Fetching gcda artifact: {{url}}', url=gcda_artifact.url)
        gcda_file = download_file(gcda_artifact.url)
        #gcda_file = 'tests/resources/ccov/code-coverage.zip'
        Log.note('Extracting gcda files to {{dir}}/ccov', dir=tmpdir)
        ZipFile(gcda_file).extractall('%s/ccov' % tmpdir)
    except BadZipfile:
        Log.note('Bad zip file for gcda artifact: {{url}}', url=gcda_artifact.url)
        return []

    parent_etl = task_cluster_record.etl
    file_obj = group_to_gcno_artifacts(task_cluster_record.task.group.id)

    remove_files_recursively('%s/ccov' % tmpdir, 'gcno')

    Log.note('Downloading gcno artifact {{file}}', file=file_obj.url)
    _, file_etl = etl_header_gen.next(source_etl=parent_etl, url=gcda_artifact.url)

    etl_key = etl2key(file_etl)
    Log.note('GCNO records will be attached to etl_key: {{etl_key}}', etl_key=etl_key)

    gcno_file = download_file(file_obj.url)

    Log.note('Extracting gcno files to {{dir}}/ccov', dir=tmpdir)
    ZipFile(gcno_file).extractall('%s/ccov' % tmpdir)

    minimize_task(task_cluster_record)
    process_directory('%s/ccov' % tmpdir, destination, task_cluster_record, file_etl)
    File(tmpdir).delete()

    keys = [etl_key]
    return keys


def process_directory(source_dir, destination, task_cluster_record, file_etl):
    records = []
    with Timer("Processing LCOV directory {{lcov_directory}}", param={"lcov_directory": source_dir}):
        lcov_coverage = run_lcov_on_directory(source_dir)

{'fraction_non_alphanumeric': 0.09174311926605505, 'fraction_numerical': 0.0024094152534519507, 'mean_word_length': 3.5574324324324325, 'pattern_counts': {'":': 6, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4611910', 'n_tokens_mistral': 3396, 'n_tokens_neox': 3258, 'n_words': 973}