<commit_before>#!/bin/bash

# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.

set -ex
set -o pipefail

if [[ $GROUP != nonode ]]; then
    python -c "from jupyterlab.commands import build_check; build_check()"
fi


if [[ $GROUP == python ]]; then
    # Run the python tests
    py.test -v --junitxml=junit.xml
fi


if [[ $GROUP == js* ]]; then

    if [[ $GROUP == js-* ]]; then
        # extract the group name
        export PKG="${GROUP#*-}"
        jlpm run build:packages:scope --scope "@jupyterlab/$PKG"
        here=$(pwd)
        if [[ -d "${here}/tests/test-${GROUP}" ]];then
            scope="@jupyterlab/test-${PKG}"
        else
            scope="@jupyterlab/${PKG}"
            jlpm run build:testutils
        fi
        jlpm run build:test:scope --scope ${scope}
        FORCE_COLOR=1 jlpm run test:scope --loglevel success --scope ${scope}
    else
        jlpm build:packages
        jlpm build:test
        FORCE_COLOR=1 jlpm test --loglevel success
    fi

    jlpm run clean
fi

if [[ $GROUP == jsscope ]]; then

    jlpm build:packages
    jlpm build:test
    FORCE_COLOR=1 jlpm test:scope --loglevel success --scope $JSTESTGROUP

    jlpm run clean
fi


if [[ $GROUP == docs ]]; then
    # Verify tutorial docs build
    pushd docs
    pip install -r requirements.txt
    make html

    # Remove internal sphinx files and use pytest-check-links on the generated html
    rm build/html/genindex.html
    rm build/html/search.html

    # Changelog has a lot of links and is covered in a separate job.
    changelog=./docs/source/getting_started/changelog.rst
    py.test --check-links --links-ext .html -k .html --deselect $changelog build/html || py.test --check-links --links-ext .html -k .html --deselect $changelog --lf build/html

    popd
fi


if [[ $GROUP == docs2 ]]; then
    # Run the link check on md files - allow for a link to fail once (--lf means only run last failed)
    py.test --check-links --links-ext .md -k .md . || py.test --check-links --links-ext .md -k .md --lf .

    # Build the API docs
    jlpm build:packages
    jlpm docs

    # Run the link check on the changelog - allow for a link to fail once (--lf means only run last failed)
    changelog=./docs/source/getting_started/changelog.rst
    py.test --check-links $changelog || py.test --check-links --lf $changelog
fi


if [[ $GROUP == integrity ]]; then
    # Run the integrity script first
    jlpm run integrity --force

    # Check yarn.lock file
    jlpm check --integrity

    # Lint our files.
    jlpm run lint:check || (echo 'Please run `jlpm run lint` locally and push changes' && exit 1)
fi<commit_msg>correctly skip redundant linkcheck of changelog<commit_after>#!/bin/bash

# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.

set -ex
set -o pipefail

if [[ $GROUP != nonode ]]; then
    python -c "from jupyterlab.commands import build_check; build_check()"
fi


if [[ $GROUP == python ]]; then
    # Run the python tests
    py.test -v --junitxml=junit.xml
fi


if [[ $GROUP == js* ]]; then

    if [[ $GROUP == js-* ]]; then
        # extract the group name
        export PKG="${GROUP#*-}"
        jlpm run build:packages:scope --scope "@jupyterlab/$PKG"
        here=$(pwd)
        if [[ -d "${here}/tests/test-${GROUP}" ]];then
            scope="@jupyterlab/test-${PKG}"
        else
            scope="@jupyterlab/${PKG}"
            jlpm run build:testutils
        fi
        jlpm run build:test:scope --scope ${scope}
        FORCE_COLOR=1 jlpm run test:scope --loglevel success --scope ${scope}
    else
        jlpm build:packages
        jlpm build:test
        FORCE_COLOR=1 jlpm test --loglevel success
    fi

    jlpm run clean
fi

if [[ $GROUP == jsscope ]]; then

    jlpm build:packages
    jlpm build:test
    FORCE_COLOR=1 jlpm test:scope --loglevel success --scope $JSTESTGROUP

    jlpm run clean
fi


if [[ $GROUP == docs ]]; then
    # Verify tutorial docs build
    pushd docs
    pip install -r ./requirements.txt
    make html

    # Remove internal sphinx files and use pytest-check-links on the generated html
    rm build/html/genindex.html
    rm build/html/search.html

    # Changelog has a lot of links and is covered in a separate job.
    changelog_html=./build/html/getting_started/changelog.html
    py.test --check-links --links-ext .html -k .html --ignore $changelog_html build/html || py.test --check-links --links-ext .html -k .html --ignore $changelog_html --lf build/html

    popd
fi


if [[ $GROUP == docs2 ]]; then
    # Run the link check on md files - allow for a link to fail once (--lf means only run last failed)
    py.test --check-links --links-ext .md -k .md . || py.test --check-links --links-ext .md -k .md --lf .

    # Build the API docs
    jlpm build:packages
    jlpm docs

    # Run the link check on the changelog - allow for a link to fail once (--lf means only run last failed)
    changelog=./docs/source/getting_started/changelog.rst
    py.test --check-links $changelog || py.test --check-links --lf $changelog
fi


if [[ $GROUP == integrity ]]; then
    # Run the integrity script first
    jlpm run integrity --force

    # Check yarn.lock file
    jlpm check --integrity

    # Lint our files.
    jlpm run lint:check || (echo 'Please run `jlpm run lint` locally and push changes' && exit 1)
fi
{'fraction_non_alphanumeric': 0.11838931156058638, 'fraction_numerical': 0.0018556318426424198, 'mean_word_length': 2.43312101910828, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '194397', 'n_tokens_mistral': 1813, 'n_tokens_neox': 1701, 'n_words': 604}