<commit_before># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: <NAME> (<EMAIL>)

from __future__ import division
from __future__ import unicode_literals

from future.utils import text_type

from activedata_etl.transforms import ACTIVE_DATA_QUERY, download_file
from jx_python.expressions import jx_expression_to_function
from mo_dots import coalesce
from mo_files import TempFile
from mo_json import stream
from mo_logs import Log
from mo_times import Timer
from pyLibrary.env import http
from pyLibrary.env.big_data import scompressed2ibytes

KNOWN_FAILURES = {"or": [
    {"in": {".": [
        "chrome://global/content/bindings/tree.xml",
        "chrome://pageloader/content/Profiler.js",
        "chrome://workerbootstrap/content/worker.js",
        "decorators.py",
        "http://mochi.test:8888/resources/testharnessreport.js",
        "http://mochi.test:8888/tests/SimpleTest/SimpleTest.js",
        "http://mochi.test:8888/tests/SimpleTest/TestRunner.js",
        "http://web-platform.test:8000/dom/common.js",
        "https://example.com/tests/SimpleTest/SimpleTest.js",
        "https://example.com/tests/SimpleTest/TestRunner.js",
        "resource://gre/modules/workers/require.js",
        "resource://services-common/utils.js",
        "resource://services-crypto/utils.js"
    ]}},
    {"suffix": {".": "/build/tests/xpcshell/head.js"}}, {"suffix": {".": "mozilla.org.xpi!/bootstrap.js"}}
]}
KNOWN_MAPPINGS = {
    "http://example.org/tests/SimpleTest/TestRunner.js": "dom/tests/mochitest/ajax/mochikit/tests/SimpleTest/TestRunner.js"
}
EXCLUDE = ('mobile',)  # TUPLE OF SOURCE DIRECTORIES TO EXCLUDE
SUITES = {  # SOME SUITES ARE RELATED TO A NUMBER OF OTHER NAMES, WHICH CAN IMPROVE SCORING
    "web-platform-tests": {"web-platform", "tests", "test", "wpt"}
}


class FileMapper(object):
    """
    MAP FROM COVERAGE FILE RESOURCE NAME TO SOURCE FILENAME
    """

    def __init__(self, task_cluster_record):
        """
        :param task_cluster_record: EXPECTING TC RECORD WITH repo.push.date SO AN APPROXIMATE SOURCE FILE LIST CAN BE FOUND
        """

        # TODO: THERE IS A RISK THE FILE MAPPING MAY CHANGE
        # FIND RECENT FILE LISTING
        timestamp = coalesce(task_cluster_record.repo.push.date, task_cluster_record.repo.changeset.date)
        result = http.post_json(
            ACTIVE_DATA_QUERY,
            json={
                "from": "task.task.artifacts",
                "where": {"and": [
                    {"eq": {"name": "public/components.json.gz"}},
                    {"eq": {"treeherder.symbol": "Bugzilla"}},
                    {"lt": {"repo.push.date": timestamp}}
                ]},
                "sort": {"repo.push.date": "desc"},
                "limit": 1,
                "select": ["url", "repo.push.date"],
                "format": "list"
            }
        )
        files_url = result.data[0].url

        self.predefined_failures = jx_expression_to_function(KNOWN_FAILURES)
        self.known_failures = set()
        self.lookup = {}
        with TempFile() as tempfile:
            Log.note("download {{url}}", url=files_url)
            download_file(files_url, tempfile.abspath)
            with open(tempfile.abspath, b"rb") as fstream:
                with Timer("process {{url}}", param={"url": files_url}):
                    for data in stream.parse(
                        scompressed2ibytes(fstream),
                        {"items": "."},
                        {"name"}
                    ):
                        self._add(data.name)

    def _add(self, filename):
        if filename.startswith(EXCLUDE):
            return

        path = list(reversed(filename.split("/")))
        curr = self.lookup
        for i, p in enumerate(path):
            found = curr.get(p)
            if not found:
                curr[p] = filename
                return
            elif isinstance(found, text_type):
                if i + 1 >= len(path):
                    curr[p] = {".": filename}
                else:
                    curr[p] = {path[i + 1]: filename}
                self._add(found)
                return
            else:
                curr = found

    def find(self, source_key, filename, artifact, task_cluster_record):
        """
        :param source_key: FOR DEBUGGING
        :param filename: THE FILE TO LOOK FOR
        :param artifact: THE ARTIFACT (FOR DEBUGGING)
        :param task_cluster_record: FOR OTHER INFO THAT MAY HELP IDENTIFYING THE RIGHT SOURCE FILE
        :return: {"name":name, "old_name":old_name, "is_firefox":boolean}
        """
        def find_best(path, files, complain):
            path = set(pp for p in path for pp in p.split("-"))
            best = None
            best_score = 0
            peer = None
            for f in files:
                f_path = set(pp for p in f.split("/") for pp in p.split("-"))
                score = len(path & f_path) + (0.5 * len(suite_names & f_path))
                if score > best_score:
                    best = f
                    peer = None
                    best_score = score
                elif score == best_score:
                    peer = f
            if best and not peer:
                if best == filename:
                    return {"name": best, "is_firefox": True}
                else:
                    return {"name": best, "is_firefox": True, "old_name": filename}
            else:
                self.known_failures.add(filename)
                if complain:
                    Log.warning(
                        "Can not resolve {{filename}} in {{url}} for key {{key}}. Too many candidates: {{list|json}}",
                        key=source_key,
                        url=artifact.url,
                        filename=filename,
                        list=files
                    )
                return {"name": filename}

        try:
            found = KNOWN_MAPPINGS.get(filename);
            if found:
                return {"name": found, "is_firefox": True, "old_name": filename}
            if self.predefined_failures(filename):
                return {"name": filename}
            if filename in self.known_failures:
                return {"name": filename}
            suite_names = SUITES.get(task_cluster_record.suite.name, {task_cluster_record.suite.name})

            filename = filename.split(' line ')[0].split(' -> ')[0].split('?')[0].split('#')[0]  # FOR URLS WITH PARAMETERS
            path = list(reversed(filename.split("/")))
            curr = self.lookup
            i = -1
            for i, p in enumerate(path):
                found = curr.get(p)
                if not found:
                    break
                elif isinstance(found, text_type):
                    if found == filename:
                        return {"name": found, "is_firefox": True}
                    else:
                        return {"name": found, "is_firefox": True, "old_name": filename}
                else:
                    curr = found

            if i == 0:  # WE MATCH NOTHING, DO NOT EVEN TRY FOR A BETTER MATCH
                return {"name": filename}
            return find_best(path, list(sorted(_values(curr))), i > 1)
        except Exception as ee:
            Log.warning(
                "Can not resolve {{filename}} in {{url}} for key {{key}}",
                key=source_key,
                url=artifact.url,
                filename=filename,
                cause=ee
            )


def _values(curr):
    for v in curr.values():<commit_msg>better fuzzy match<commit_after># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: <NAME> (<EMAIL>)

from __future__ import division
from __future__ import unicode_literals

import re
from future.utils import text_type

from activedata_etl.transforms import ACTIVE_DATA_QUERY, download_file
from jx_python.expressions import jx_expression_to_function
from mo_dots import coalesce
from mo_files import TempFile
from mo_json import stream
from mo_logs import Log
from mo_times import Timer
from pyLibrary.env import http
from pyLibrary.env.big_data import scompressed2ibytes

KNOWN_FAILURES = {"or": [
    {"in": {".": [
        "chrome://global/content/bindings/tree.xml",
        "chrome://pageloader/content/Profiler.js",
        "chrome://workerbootstrap/content/worker.js",
        "decorators.py",
        "http://mochi.test:8888/resources/testharnessreport.js",
        "http://mochi.test:8888/tests/SimpleTest/SimpleTest.js",
        "http://mochi.test:8888/tests/SimpleTest/TestRunner.js",
        "http://web-platform.test:8000/dom/common.js",
        "http://web-platform.test:8000/testharness_runner.html",
        "https://example.com/tests/SimpleTest/SimpleTest.js",
        "https://example.com/tests/SimpleTest/TestRunner.js",
        "resource://gre/modules/workers/require.js",
        "resource://services-common/utils.js",
        "resource://services-crypto/utils.js"
    ]}},
    {"suffix": {".": "/build/tests/xpcshell/head.js"}},
    {"suffix": {".": "mozilla.org.xpi!/bootstrap.js"}},
    {"prefix": {".": "data:"}}
]}
KNOWN_MAPPINGS = {
    "http://example.org/tests/SimpleTest/TestRunner.js": "dom/tests/mochitest/ajax/mochikit/tests/SimpleTest/TestRunner.js"
}
EXCLUDE = ('mobile',)  # TUPLE OF SOURCE DIRECTORIES TO EXCLUDE
SUITES = {  # SOME SUITES ARE RELATED TO A NUMBER OF OTHER NAMES, WHICH CAN IMPROVE SCORING
    "web-platform-tests": {"web-platform", "tests", "test", "wpt"}
}


class FileMapper(object):
    """
    MAP FROM COVERAGE FILE RESOURCE NAME TO SOURCE FILENAME
    """

    def __init__(self, task_cluster_record):
        """
        :param task_cluster_record: EXPECTING TC RECORD WITH repo.push.date SO AN APPROXIMATE SOURCE FILE LIST CAN BE FOUND
        """

        # TODO: THERE IS A RISK THE FILE MAPPING MAY CHANGE
        # FIND RECENT FILE LISTING
        timestamp = coalesce(task_cluster_record.repo.push.date, task_cluster_record.repo.changeset.date)
        result = http.post_json(
            ACTIVE_DATA_QUERY,
            json={
                "from": "task.task.artifacts",
                "where": {"and": [
                    {"eq": {"name": "public/components.json.gz"}},
                    {"eq": {"treeherder.symbol": "Bugzilla"}},
                    {"lt": {"repo.push.date": timestamp}}
                ]},
                "sort": {"repo.push.date": "desc"},
                "limit": 1,
                "select": ["url", "repo.push.date"],
                "format": "list"
            }
        )
        files_url = result.data[0].url

        self.predefined_failures = jx_expression_to_function(KNOWN_FAILURES)
        self.known_failures = set()
        self.lookup = {}
        with TempFile() as tempfile:
            Log.note("download {{url}}", url=files_url)
            download_file(files_url, tempfile.abspath)
            with open(tempfile.abspath, b"rb") as fstream:
                with Timer("process {{url}}", param={"url": files_url}):
                    for data in stream.parse(
                        scompressed2ibytes(fstream),
                        {"items": "."},
                        {"name"}
                    ):
                        self._add(data.name)

    def _add(self, filename):
        if filename.startswith(EXCLUDE):
            return

        path = list(reversed(filename.split("/")))
        curr = self.lookup
        for i, p in enumerate(path):
            found = curr.get(p)
            if not found:
                curr[p] = filename
                return
            elif isinstance(found, text_type):
                if i + 1 >= len(path):
                    curr[p] = {".": filename}
                else:
                    curr[p] = {path[i + 1]: filename}
                self._add(found)
                return
            else:
                curr = found

    def find(self, source_key, filename, artifact, task_cluster_record):
        """
        :param source_key: FOR DEBUGGING
        :param filename: THE FILE TO LOOK FOR
        :param artifact: THE ARTIFACT (FOR DEBUGGING)
        :param task_cluster_record: FOR OTHER INFO THAT MAY HELP IDENTIFYING THE RIGHT SOURCE FILE
        :return: {"name":name, "old_name":old_name, "is_firefox":boolean}
        """
        def find_best(files, complain):
            path = set(re.split(r"\W", filename))
            best = None
            best_score = 0
            peer = None
            for f in files:
                f_path = set(re.split("\W", f))
                score = len(path & f_path) + (0.5 * len(suite_names & f_path))
                if score > best_score:
                    best = f
                    peer = None
                    best_score = score
                elif score == best_score:
                    peer = f
            if best and not peer:
                if best == filename:
                    return {"name": best, "is_firefox": True}
                else:
                    return {"name": best, "is_firefox": True, "old_name": filename}
            else:
                self.known_failures.add(filename)
                if complain:
                    Log.warning(
                        "Can not resolve {{filename}} in {{url}} for key {{key}}. Too many candidates: {{list|json}}",
                        key=source_key,
                        url=artifact.url,
                        filename=filename,
                        list=files
                    )
                return {"name": filename}

        try:
            found = KNOWN_MAPPINGS.get(filename);
            if found:
                return {"name": found, "is_firefox": True, "old_name": filename}
            if self.predefined_failures(filename):
                return {"name": filename}
            if filename in self.known_failures:
                return {"name": filename}
            suite_names = SUITES.get(task_cluster_record.suite.name, {task_cluster_record.suite.name})

            filename = filename.split(' line ')[0].split(' -> ')[0].split('?')[0].split('#')[0]  # FOR URLS WITH PARAMETERS
            path = list(reversed(filename.split("/")))
            curr = self.lookup
            i = -1
            for i, p in enumerate(path):
                found = curr.get(p)
                if not found:
                    break
                elif isinstance(found, text_type):
                    if found == filename:
                        return {"name": found, "is_firefox": True}
                    else:
                        return {"name": found, "is_firefox": True, "old_name": filename}
                else:
                    curr = found

            if i == 0:  # WE MATCH NOTHING, DO NOT EVEN TRY FOR A BETTER MATCH
                return {"name": filename}
            return find_best(list(sorted(_values(curr))), i > 1)
        except Exception as ee:
            Log.warning(
                "Can not resolve {{filename}} in {{url}} for key {{key}}",
                key=source_key,
                url=artifact.url,
                filename=filename,
                cause=ee
            )


def _values(curr):
    for v in curr.values():
{'fraction_non_alphanumeric': 0.1205564142194745, 'fraction_numerical': 0.005023183925811438, 'mean_word_length': 1.7253422253422253, 'pattern_counts': {'":': 94, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3295452', 'n_tokens_mistral': 4611, 'n_tokens_neox': 4320, 'n_words': 1092}