<commit_before># coding: utf8
from __future__ import unicode_literals, print_function

import requests
import pkg_resources
from pathlib import Path

from ..compat import path2str, locale_escape
from ..util import prints, get_data_path, read_json
from .. import about


def validate(cmd):
    """Validate that the currently installed version of spaCy is compatible
    with the installed models. Should be run after `pip install -U spacy`.
    """
    r = requests.get(about.__compatibility__)
    if r.status_code != 200:
        prints("Couldn't fetch compatibility table.",
               title="Server error (%d)" % r.status_code, exits=1)
    compat = r.json()['spacy']
    all_models = set()
    for spacy_v, models in dict(compat).items():
        all_models.update(models.keys())
        for model, model_vs in models.items():
            compat[spacy_v][model] = [reformat_version(v) for v in model_vs]

    current_compat = compat[about.__version__]
    model_links = get_model_links(current_compat)
    model_pkgs = get_model_pkgs(current_compat, all_models)
    incompat_links = {l for l, d in model_links.items() if not d['compat']}
    incompat_models = {d['name'] for _, d in model_pkgs.items()
                       if not d['compat']}
    incompat_models.update([d['name'] for _, d in model_links.items()
                            if not d['compat']])
    na_models = [m for m in incompat_models if m not in current_compat]
    update_models = [m for m in incompat_models if m in current_compat]

    prints(path2str(Path(__file__).parent.parent),
           title="Installed models (spaCy v{})".format(about.__version__))
    if model_links or model_pkgs:
        print(get_row('TYPE', 'NAME', 'MODEL', 'VERSION', ''))
        for name, data in model_pkgs.items():
            print(get_model_row(current_compat, name, data, 'package'))
        for name, data in model_links.items():
            print(get_model_row(current_compat, name, data, 'link'))
    else:
        prints("No models found in your current environment.", exits=0)

    if update_models:
        cmd = '    python -m spacy download {}'
        print("\n    Use the following commands to update the model packages:")
        print('\n'.join([cmd.format(pkg) for pkg in update_models]))

    if na_models:
        prints("The following models are not available for spaCy v{}: {}"
               .format(about.__version__, ', '.join(na_models)))

    if incompat_links:
        prints("You may also want to overwrite the incompatible links using "
               "the `python -m spacy link` command with `--force`, or remove "
               "them from the data directory. Data path: {}"
               .format(path2str(get_data_path())))


def get_model_links(compat):
    links = {}
    data_path = get_data_path()
    if data_path:
        models = [p for p in data_path.iterdir() if is_model_path(p)]
        for model in models:
            meta_path = Path(model) / 'meta.json'
            if not meta_path.exists():
                continue
            meta = read_json(meta_path)
            link = model.parts[-1]
            name = meta['lang'] + '_' + meta['name']
            links[link] = {'name': name, 'version': meta['version'],
                           'compat': is_compat(compat, name, meta['version'])}
    return links


def get_model_pkgs(compat, all_models):<commit_msg>Exit with 1 if incompatible models found (see #1714)<commit_after># coding: utf8
from __future__ import unicode_literals, print_function

import requests
import pkg_resources
from pathlib import Path
import sys

from ..compat import path2str, locale_escape
from ..util import prints, get_data_path, read_json
from .. import about


def validate(cmd):
    """Validate that the currently installed version of spaCy is compatible
    with the installed models. Should be run after `pip install -U spacy`.
    """
    r = requests.get(about.__compatibility__)
    if r.status_code != 200:
        prints("Couldn't fetch compatibility table.",
               title="Server error (%d)" % r.status_code, exits=1)
    compat = r.json()['spacy']
    all_models = set()
    for spacy_v, models in dict(compat).items():
        all_models.update(models.keys())
        for model, model_vs in models.items():
            compat[spacy_v][model] = [reformat_version(v) for v in model_vs]

    current_compat = compat[about.__version__]
    model_links = get_model_links(current_compat)
    model_pkgs = get_model_pkgs(current_compat, all_models)
    incompat_links = {l for l, d in model_links.items() if not d['compat']}
    incompat_models = {d['name'] for _, d in model_pkgs.items()
                       if not d['compat']}
    incompat_models.update([d['name'] for _, d in model_links.items()
                            if not d['compat']])
    na_models = [m for m in incompat_models if m not in current_compat]
    update_models = [m for m in incompat_models if m in current_compat]

    prints(path2str(Path(__file__).parent.parent),
           title="Installed models (spaCy v{})".format(about.__version__))
    if model_links or model_pkgs:
        print(get_row('TYPE', 'NAME', 'MODEL', 'VERSION', ''))
        for name, data in model_pkgs.items():
            print(get_model_row(current_compat, name, data, 'package'))
        for name, data in model_links.items():
            print(get_model_row(current_compat, name, data, 'link'))
    else:
        prints("No models found in your current environment.", exits=0)

    if update_models:
        cmd = '    python -m spacy download {}'
        print("\n    Use the following commands to update the model packages:")
        print('\n'.join([cmd.format(pkg) for pkg in update_models]))

    if na_models:
        prints("The following models are not available for spaCy v{}: {}"
               .format(about.__version__, ', '.join(na_models)))

    if incompat_links:
        prints("You may also want to overwrite the incompatible links using "
               "the `python -m spacy link` command with `--force`, or remove "
               "them from the data directory. Data path: {}"
               .format(path2str(get_data_path())))

    if incompat_models or incompat_links:
        sys.exit(1)


def get_model_links(compat):
    links = {}
    data_path = get_data_path()
    if data_path:
        models = [p for p in data_path.iterdir() if is_model_path(p)]
        for model in models:
            meta_path = Path(model) / 'meta.json'
            if not meta_path.exists():
                continue
            meta = read_json(meta_path)
            link = model.parts[-1]
            name = meta['lang'] + '_' + meta['name']
            links[link] = {'name': name, 'version': meta['version'],
                           'compat': is_compat(compat, name, meta['version'])}
    return links


def get_model_pkgs(compat, all_models):
{'fraction_non_alphanumeric': 0.11327613785080703, 'fraction_numerical': 0.003780718336483932, 'mean_word_length': 2.729934924078091, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7352732', 'n_tokens_mistral': 2054, 'n_tokens_neox': 1993, 'n_words': 606}