<commit_before># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: <NAME> (<EMAIL>)
#
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

from mo_dots import coalesce, wrap, unwrap
from mo_logs import Log
from mo_logs import constants
from mo_logs import startup
from mo_math import MIN
from mo_math import UNION
from mo_threads import Thread, Signal, Queue, THREAD_STOP

from mo_dots.datas import leaves
from mo_times.dates import Date, unicode2Date
from mo_times.timer import Timer
from pyLibrary.env import http, elasticsearch
from jx_python import jx

DEBUG = True
NUM_THREAD = 4


def process_batch(todo, coverage_index, coverage_summary_index, settings, please_stop):
    for not_summarized in todo:
        if please_stop:
            return True

        # IS THERE MORE THAN ONE COVERAGE FILE FOR THIS REVISION?
        Log.note("Find dups for file {{file}}", file=not_summarized.source.file.name)
        dups = http.post_json(settings.url, json={
            "from": "coverage",
            "select": [
                # THIS IS FAR FROM GOOD, WE WANT THE LATEST ETL ID, BUT etl.source.id IS ONLY A SUB-ID
                # WE SHOULD BE ABLE TO GET THE max OF A TUPLE
                {"name": "max_id", "value": "etl.source.id", "aggregate": "max"},
                {"name": "min_id", "value": "etl.source.id", "aggregate": "min"}
            ],
            "where": {"and": [
                {"missing": "source.method.name"},
                {"neq": {"source.file.total_covered": 0}},
                {"eq": {
                    "source.file.name": not_summarized.source.file.name,
                    "build.revision12": not_summarized.build.revision12
                }},
            ]},
            "groupby": [
                "test.suite",
                "test.chunk",
                "test.url",
                "test.name"
            ],
            "limit": 100000,
            "format": "list"
        })

        dups_found = False
        for d in dups.data:
            if d.max_id != d.min_id:
                # FIND ALL INDEXES
                dups_found = True
                Log.note(
                    "removing dups {{details|json}}",
                    details={"and": [
                        {"not": {"term": {"etl.source.id": int(d.max_id)}}},
                        {"and":[
                            {"term": {k: v}} for k, v in leaves({"test": d.test})
                        ]},
                        {"term": {"source.file.name": not_summarized.source.file.name}},
                        {"term": {"build.revision12": not_summarized.build.revision12}}
                    ]}
                )

                coverage_index.delete_record({"and": [
                    {"not": {"term": {"etl.source.id": int(d.max_id)}}},
                    {"and": [
                        {"term": {k: v}} for k, v in leaves({"test": d.test})
                    ]},
                    {"term": {"test.url": d.test.url}},
                    {"term": {"source.file.name": not_summarized.source.file.name}},
                    {"term": {"build.revision12": not_summarized.build.revision12}}
                ]})
        if dups_found:
            continue

        # LIST ALL TESTS THAT COVER THIS FILE, AND THE LINES COVERED
        test_count = http.post_json(settings.url, json={
            "from": "coverage.source.file.covered",
            "where": {"and": [
                {"missing": "source.method.name"},
                {"neq":{"source.file.total_covered":0}},
                {"eq": {
                    "source.file.name": not_summarized.source.file.name,
                    "build.revision12": not_summarized.build.revision12
                }},<commit_msg>do not perform dup checks, they are valid<commit_after># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: <NAME> (<EMAIL>)
#
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

from mo_dots import coalesce, wrap, unwrap
from mo_logs import Log
from mo_logs import constants
from mo_logs import startup
from mo_math import MIN
from mo_math import UNION
from mo_threads import Thread, Signal, Queue, THREAD_STOP

from mo_dots.datas import leaves
from mo_times.dates import Date, unicode2Date
from mo_times.timer import Timer
from pyLibrary.env import http, elasticsearch
from jx_python import jx

DEBUG = True
NUM_THREAD = 4


def process_batch(todo, coverage_index, coverage_summary_index, settings, please_stop):
    for not_summarized in todo:
        if please_stop:
            return True

        # IS THERE MORE THAN ONE COVERAGE FILE FOR THIS REVISION?
        # Log.note("Find dups for file {{file}}", file=not_summarized.source.file.name)
        # dups = http.post_json(settings.url, json={
        #     "from": "coverage",
        #     "select": [
        #         # THIS IS FAR FROM GOOD, WE WANT THE LATEST ETL ID, BUT etl.source.id IS ONLY A SUB-ID
        #         # WE SHOULD BE ABLE TO GET THE max OF A TUPLE
        #         {"name": "max_id", "value": "etl.source.id", "aggregate": "max"},
        #         {"name": "min_id", "value": "etl.source.id", "aggregate": "min"}
        #     ],
        #     "where": {"and": [
        #         {"missing": "source.method.name"},
        #         {"neq": {"source.file.total_covered": 0}},
        #         {"eq": {
        #             "source.file.name": not_summarized.source.file.name,
        #             "build.revision12": not_summarized.build.revision12
        #         }},
        #     ]},
        #     "groupby": [
        #         "test.suite",
        #         "test.chunk",
        #         "test.url",
        #         "test.name"
        #     ],
        #     "limit": 100000,
        #     "format": "list"
        # })

        # dups_found = False
        # for d in dups.data:
        #     if d.max_id != d.min_id:
        #         # FIND ALL INDEXES
        #         dups_found = True
        #         Log.note(
        #             "removing dups {{details|json}}",
        #             details={"and": [
        #                 {"not": {"term": {"etl.source.id": int(d.max_id)}}},
        #                 {"and":[
        #                     {"term": {k: v}} for k, v in leaves({"test": d.test})
        #                 ]},
        #                 {"term": {"source.file.name": not_summarized.source.file.name}},
        #                 {"term": {"build.revision12": not_summarized.build.revision12}}
        #             ]}
        #         )
        #
        #         coverage_index.delete_record({"and": [
        #             {"not": {"term": {"etl.source.id": int(d.max_id)}}},
        #             {"and": [
        #                 {"term": {k: v}} for k, v in leaves({"test": d.test})
        #             ]},
        #             {"term": {"test.url": d.test.url}},
        #             {"term": {"source.file.name": not_summarized.source.file.name}},
        #             {"term": {"build.revision12": not_summarized.build.revision12}}
        #         ]})
        # if dups_found:
        #     continue

        # LIST ALL TESTS THAT COVER THIS FILE, AND THE LINES COVERED
        test_count = http.post_json(settings.url, json={
            "from": "coverage.source.file.covered",
            "where": {"and": [
                {"missing": "source.method.name"},
                {"neq": {"source.file.total_covered": 0}},
                {"eq": {
                    "source.file.name": not_summarized.source.file.name,
                    "build.revision12": not_summarized.build.revision12
                }},
{'fraction_non_alphanumeric': 0.1324643078833023, 'fraction_numerical': 0.007697082557417753, 'mean_word_length': 1.6631404958677687, 'pattern_counts': {'":': 104, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6063750', 'n_tokens_mistral': 2477, 'n_tokens_neox': 2337, 'n_words': 609}