<commit_before>#!/usr/bin/python

from __future__ import print_function

from argparse import ArgumentParser
import json
import re
import sys
import traceback


RELEASE = 'release'
PROPOSED = 'proposed'
DEVEL = 'devel'
TESTING = 'testing'
PURPOSES = (RELEASE, PROPOSED, DEVEL, TESTING)


def find_tools(file_path):
    with open(file_path) as f:
        stream = json.load(f)
    tools = {}
    for name, product in stream['products'].items():
        versions = product['versions']
        for version in versions.values():
            if isinstance(version, dict):
                items = version['items']
                tools.update(items)
    return tools


def check_devel_not_stable(old_tools, new_tools, purpose):
    """Return an error message if the version can be included in the stream.

    Devel versions cannot be proposed or release because the letters in
    the version break older jujus.
    """
    if purpose in (TESTING, DEVEL):
        return None
    stable_pattern = re.compile(r'\d+\.\d+\.\d+-*')
    devel_versions = [
        v for v in new_tools.keys() if not stable_pattern.match(v)]
    if devel_versions:
        return 'Devel versions in {} stream: {}'.format(
            purpose, devel_versions)


def check_expected_tools(old_tools, new_tools, added=None, removed=None):
    """Return a 4-tuple of new_expected, new_errors, old_expected, old_errors

    The new and old expected dicts are the tools common to old and new streams.
    The new and old errors are strings of missing or extra versions.

    :param old_tools: the dict of all the products/versions/*/items
                      in the old json.
    :param new_tools: the dict of all the products/versions/*/items
                      in the new json.
    :param added: the version added to the new json, eg '1.20.9'
    :param retracted: the version removed to the new json, eg '1.20.8'
    """
    # Remove the expected difference between the two collections of tools.
    old_expected = dict(old_tools)
    new_expected = dict(new_tools)
    missing_errors = None
    extra_errors = None
    expected_differences = {}
    missing = []
    if removed:
        for n, t in old_expected.items():
            if t['version'] == removed:
                expected_differences.update([(n, t)])
                del old_expected[n]
    if added:
        for n, t in new_expected.items():
            if t['version'] == added:
                expected_differences.update([(n, t)])
                del new_expected[n]
        if not expected_differences:
            missing.append(added)
    # The old and new should be identical. but if there is a problem,
    # we want to explain what problems are in each set of versions.
    old_versions = set(old_expected.keys())
    new_versions = set(new_expected.keys())
    missing.extend(old_versions - new_versions)
    if missing:
        missing_errors = 'Missing versions: {}'.format(missing)
    extras = list(new_versions - old_versions)
    if extras:
        extra_errors = 'Extra versions: {}'.format(extras)
    return new_expected, extra_errors, old_expected, missing_errors


def check_tools_content(old_tools, new_tools):<commit_msg>expected_differences is not needed.<commit_after>#!/usr/bin/python

from __future__ import print_function

from argparse import ArgumentParser
import json
import re
import sys
import traceback


RELEASE = 'release'
PROPOSED = 'proposed'
DEVEL = 'devel'
TESTING = 'testing'
PURPOSES = (RELEASE, PROPOSED, DEVEL, TESTING)


def find_tools(file_path):
    with open(file_path) as f:
        stream = json.load(f)
    tools = {}
    for name, product in stream['products'].items():
        versions = product['versions']
        for version in versions.values():
            if isinstance(version, dict):
                items = version['items']
                tools.update(items)
    return tools


def check_devel_not_stable(old_tools, new_tools, purpose):
    """Return an error message if the version can be included in the stream.

    Devel versions cannot be proposed or release because the letters in
    the version break older jujus.
    """
    if purpose in (TESTING, DEVEL):
        return None
    stable_pattern = re.compile(r'\d+\.\d+\.\d+-*')
    devel_versions = [
        v for v in new_tools.keys() if not stable_pattern.match(v)]
    if devel_versions:
        return 'Devel versions in {} stream: {}'.format(
            purpose, devel_versions)


def check_expected_tools(old_tools, new_tools, added=None, removed=None):
    """Return a 4-tuple of new_expected, new_errors, old_expected, old_errors

    The new and old expected dicts are the tools common to old and new streams.
    The new and old errors are strings of missing or extra versions.

    :param old_tools: the dict of all the products/versions/*/items
                      in the old json.
    :param new_tools: the dict of all the products/versions/*/items
                      in the new json.
    :param added: the version added to the new json, eg '1.20.9'
    :param retracted: the version removed to the new json, eg '1.20.8'
    """
    # Remove the expected difference between the two collections of tools.
    old_expected = dict(old_tools)
    new_expected = dict(new_tools)
    missing_errors = None
    extra_errors = None
    missing = []
    extras = []
    found = []
    if removed:
        for n, t in old_expected.items():
            if t['version'] == removed:
                del old_expected[n]
    if added:
        for n, t in new_expected.items():
            if t['version'] == added:
                del new_expected[n]
                found.append(n)
        if not found:
            missing.append(added)
    # The old and new should be identical. but if there is a problem,
    # we want to explain what problems are in each set of versions.
    old_versions = set(old_expected.keys())
    new_versions = set(new_expected.keys())
    missing.extend(old_versions - new_versions)
    if missing:
        missing_errors = 'Missing versions: {}'.format(missing)
    extras.extend(new_versions - old_versions)
    if extras:
        extra_errors = 'Extra versions: {}'.format(extras)
    return new_expected, extra_errors, old_expected, missing_errors


def check_tools_content(old_tools, new_tools):
{'fraction_non_alphanumeric': 0.09101016518424397, 'fraction_numerical': 0.0028589580686149934, 'mean_word_length': 2.7150442477876107, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3056991', 'n_tokens_mistral': 1829, 'n_tokens_neox': 1772, 'n_words': 623}