<commit_before># -*- coding: utf-8 -*-
import json
import os
import shutil
import tarfile
from zipfile import ZipFile
import requests
import hglib
from threading import Condition, Lock
import concurrent.futures
from concurrent.futures import ThreadPoolExecutor
import sqlite3
import time

from cli_common.log import get_logger
from cli_common.command import run_check
from cli_common.taskcluster import get_service

from shipit_code_coverage import taskcluster, uploader
from shipit_code_coverage.utils import mkdir, wait_until, retry, ThreadPoolExecutorResult


logger = get_logger(__name__)


class CodeCov(object):

    def __init__(self, revision, cache_root, coveralls_token, codecov_token,
                 gecko_dev_user, gecko_dev_pwd, client_id, access_token):
        # List of test-suite, sorted alphabetically.
        # This way, the index of a suite in the array should be stable enough.
        self.suites = [
            'cppunit', 'gtest', 'web-platform-tests', 'talos',
        ]

        self.cache_root = cache_root

        assert os.path.isdir(cache_root), 'Cache root {} is not a dir.'.format(cache_root)
        self.repo_dir = os.path.join(cache_root, 'mozilla-central')

        self.gecko_dev_user = gecko_dev_user
        self.gecko_dev_pwd = <PASSWORD>
        self.client_id = client_id
        self.access_token = access_token

        if revision is None:
            self.task_ids = [
                taskcluster.get_last_task('linux'),
                taskcluster.get_last_task('win'),
            ]

            task_data = taskcluster.get_task_details(self.task_ids[0])
            self.revision = task_data['payload']['env']['GECKO_HEAD_REV']
            self.coveralls_token = 'NONE'
            self.codecov_token = 'NONE'
            self.from_pulse = False
        else:
            self.task_ids = [
                taskcluster.get_task('mozilla-central', revision, 'linux'),
                taskcluster.get_task('mozilla-central', revision, 'win'),
            ]
            self.revision = revision
            self.coveralls_token = coveralls_token
            self.codecov_token = codecov_token
            self.from_pulse = True

        self.build_finished = False
        self.build_finished_cv = Condition()

        if self.from_pulse:
            self.suites_to_ignore = ['awsy', 'talos']
        else:
            self.suites_to_ignore = []

        logger.info('Mercurial revision', revision=self.revision)

    def get_chunks(self):
        return list(set([f.split('_')[1] for f in os.listdir('ccov-artifacts')]))

    def get_coverage_artifacts(self, suite=None, chunk=None):
        files = os.listdir('ccov-artifacts')

        if suite is not None and chunk is not None:
            raise Exception('suite and chunk can\'t both have a value')

        filtered_files = []
        for fname in files:
            # grcov artifacts always have 'grcov' in the name and are ZIP files.
            if 'grcov' in fname and not fname.endswith('.zip'):
                continue
            # jsvm artifacts always have 'jsvm' in the name and are not ZIP files.
            if 'jsvm' in fname and fname.endswith('.zip'):
                continue

            # If suite and chunk are None, return all artifacts.
            # Otherwise, only return the ones which have suite or chunk in their name.
            if (
                   (suite is None and chunk is None) or<commit_msg>shipit_code_coverage: Log mercurial revision as soon as possible (#794)<commit_after># -*- coding: utf-8 -*-
import json
import os
import shutil
import tarfile
from zipfile import ZipFile
import requests
import hglib
from threading import Condition, Lock
import concurrent.futures
from concurrent.futures import ThreadPoolExecutor
import sqlite3
import time

from cli_common.log import get_logger
from cli_common.command import run_check
from cli_common.taskcluster import get_service

from shipit_code_coverage import taskcluster, uploader
from shipit_code_coverage.utils import mkdir, wait_until, retry, ThreadPoolExecutorResult


logger = get_logger(__name__)


class CodeCov(object):

    def __init__(self, revision, cache_root, coveralls_token, codecov_token,
                 gecko_dev_user, gecko_dev_pwd, client_id, access_token):
        # List of test-suite, sorted alphabetically.
        # This way, the index of a suite in the array should be stable enough.
        self.suites = [
            'cppunit', 'gtest', 'web-platform-tests', 'talos',
        ]

        self.cache_root = cache_root

        assert os.path.isdir(cache_root), 'Cache root {} is not a dir.'.format(cache_root)
        self.repo_dir = os.path.join(cache_root, 'mozilla-central')

        self.gecko_dev_user = gecko_dev_user
        self.gecko_dev_pwd = <PASSWORD>
        self.client_id = client_id
        self.access_token = access_token

        if revision is None:
            self.task_ids = [
                taskcluster.get_last_task('linux'),
                taskcluster.get_last_task('win'),
            ]

            task_data = taskcluster.get_task_details(self.task_ids[0])
            self.revision = task_data['payload']['env']['GECKO_HEAD_REV']
            self.coveralls_token = 'NONE'
            self.codecov_token = 'NONE'
            self.from_pulse = False
            logger.info('Mercurial revision', revision=self.revision)
        else:
            logger.info('Mercurial revision', revision=revision)
            self.task_ids = [
                taskcluster.get_task('mozilla-central', revision, 'linux'),
                taskcluster.get_task('mozilla-central', revision, 'win'),
            ]
            self.revision = revision
            self.coveralls_token = coveralls_token
            self.codecov_token = codecov_token
            self.from_pulse = True

        self.build_finished = False
        self.build_finished_cv = Condition()

        if self.from_pulse:
            self.suites_to_ignore = ['awsy', 'talos']
        else:
            self.suites_to_ignore = []

    def get_chunks(self):
        return list(set([f.split('_')[1] for f in os.listdir('ccov-artifacts')]))

    def get_coverage_artifacts(self, suite=None, chunk=None):
        files = os.listdir('ccov-artifacts')

        if suite is not None and chunk is not None:
            raise Exception('suite and chunk can\'t both have a value')

        filtered_files = []
        for fname in files:
            # grcov artifacts always have 'grcov' in the name and are ZIP files.
            if 'grcov' in fname and not fname.endswith('.zip'):
                continue
            # jsvm artifacts always have 'jsvm' in the name and are not ZIP files.
            if 'jsvm' in fname and fname.endswith('.zip'):
                continue

            # If suite and chunk are None, return all artifacts.
            # Otherwise, only return the ones which have suite or chunk in their name.
            if (
                   (suite is None and chunk is None) or
{'fraction_non_alphanumeric': 0.08590623213264723, 'fraction_numerical': 0.0015723270440251573, 'mean_word_length': 2.383462282398453, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3746873', 'n_tokens_mistral': 2068, 'n_tokens_neox': 1974, 'n_words': 556}