<commit_before>version: 2
jobs:
    run_tests_torch_and_tf:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.5
        environment:
            OMP_NUM_THREADS: 1
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - run: sudo pip install .[sklearn,tf,torch,testing]
            - run: sudo pip install codecov pytest-cov
            - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
            - run: codecov
    run_tests_torch:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.7
        environment:
            OMP_NUM_THREADS: 1
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - run: sudo pip install .[sklearn,torch,testing]
            - run: sudo pip install codecov pytest-cov
            - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
            - run: codecov
    run_tests_tf:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.7
        environment:
            OMP_NUM_THREADS: 1
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - run: sudo pip install .[sklearn,tf,testing]
            - run: sudo pip install codecov pytest-cov
            - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
            - run: codecov
    run_tests_custom_tokenizers:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.5
        steps:
            - checkout
            - run: sudo pip install .[mecab,testing]
            - run: RUN_CUSTOM_TOKENIZERS=1 python -m pytest -sv ./tests/test_tokenization_bert_japanese.py
    run_examples_torch:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.5
        environment:
            OMP_NUM_THREADS: 1
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - run: sudo pip install .[sklearn,torch,testing]
            - run: sudo pip install -r examples/requirements.txt
            - run: python -m pytest -n 8 --dist=loadfile -s -v ./examples/
    deploy_doc:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.5
        steps:
            - add_ssh_keys:
                  fingerprints:
                      - "5b:7a:95:18:07:8c:aa:76:4c:60:35:88:ad:60:56:71"
            - checkout
            - run: sudo pip install .[tf,torch,docs]
            - run: ./.circleci/deploy.sh
    check_code_quality:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.6
        resource_class: medium
        parallelism: 1
        steps:
            - checkout
            # we need a version of isort with https://github.com/timothycrosley/isort/pull/1000
            - run: sudo pip install git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort
            - run: sudo pip install .[tf,torch,quality]
            - run: black --check --line-length 119 examples templates tests src utils
            - run: isort --check-only --recursive examples templates tests src utils
            - run: flake8 examples templates tests src utils
    check_repository_consistency:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.5
        resource_class: small
        parallelism: 1
        steps:
            - checkout
            - run: sudo pip install requests
            - run: python ./utils/link_tester.py
workflow_filters: &workflow_filters
    filters:
        branches:
            only:
                - master
workflows:
    version: 2
    build_and_test:
        jobs:
            - check_code_quality
            - check_repository_consistency
            - run_examples_torch
            - run_tests_custom_tokenizers
            - run_tests_torch_and_tf
            - run_tests_torch
            - run_tests_tf
            - deploy_doc: *workflow_filters<commit_msg>Run the slow tests every Monday morning.<commit_after>version: 2
jobs:
    run_tests_torch_and_tf:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.5
        environment:
            OMP_NUM_THREADS: 1
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - run: sudo pip install .[sklearn,tf,torch,testing]
            - run: sudo pip install codecov pytest-cov
            - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
            - run: codecov
    run_all_tests_torch_and_tf:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.5
        environment:
            OMP_NUM_THREADS: 1
            RUN_SLOW: yes
            RUN_CUSTOM_TOKENIZERS: yes
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - run: sudo pip install .[mecab,sklearn,tf,torch,testing]
            - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/
    run_tests_torch:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.7
        environment:
            OMP_NUM_THREADS: 1
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - run: sudo pip install .[sklearn,torch,testing]
            - run: sudo pip install codecov pytest-cov
            - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
            - run: codecov
    run_tests_tf:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.7
        environment:
            OMP_NUM_THREADS: 1
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - run: sudo pip install .[sklearn,tf,testing]
            - run: sudo pip install codecov pytest-cov
            - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
            - run: codecov
    run_tests_custom_tokenizers:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.5
        environment:
            RUN_CUSTOM_TOKENIZERS: yes
        steps:
            - checkout
            - run: sudo pip install .[mecab,testing]
            - run: python -m pytest -sv ./tests/test_tokenization_bert_japanese.py
    run_examples_torch:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.5
        environment:
            OMP_NUM_THREADS: 1
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - run: sudo pip install .[sklearn,torch,testing]
            - run: sudo pip install -r examples/requirements.txt
            - run: python -m pytest -n 8 --dist=loadfile -s -v ./examples/
    deploy_doc:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.5
        steps:
            - add_ssh_keys:
                fingerprints:
                    - "5b:7a:95:18:07:8c:aa:76:4c:60:35:88:ad:60:56:71"
            - checkout
            - run: sudo pip install .[tf,torch,docs]
            - run: ./.circleci/deploy.sh
    check_code_quality:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.6
        resource_class: medium
        parallelism: 1
        steps:
            - checkout
            # we need a version of isort with https://github.com/timothycrosley/isort/pull/1000
            - run: sudo pip install git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort
            - run: sudo pip install .[tf,torch,quality]
            - run: black --check --line-length 119 examples templates tests src utils
            - run: isort --check-only --recursive examples templates tests src utils
            - run: flake8 examples templates tests src utils
    check_repository_consistency:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.5
        resource_class: small
        parallelism: 1
        steps:
            - checkout
            - run: sudo pip install requests
            - run: python ./utils/link_tester.py
workflow_filters: &workflow_filters
    filters:
        branches:
            only:
                - master
workflows:
    version: 2
    build_and_test:
        jobs:
            - check_code_quality
            - check_repository_consistency
            - run_examples_torch
            - run_tests_custom_tokenizers
            - run_tests_torch_and_tf
            - run_tests_torch
            - run_tests_tf
            - deploy_doc: *workflow_filters
    run_slow_tests:
        triggers:
            - schedule:
                cron: "0 4 * * 1"
                filters:
                    branches:
                        only:
                            - master
        jobs:
            - run_all_tests_torch_and_tf
{'fraction_non_alphanumeric': 0.0815749401783772, 'fraction_numerical': 0.020556884924951054, 'mean_word_length': 1.5177984665936473, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1415803', 'n_tokens_mistral': 2810, 'n_tokens_neox': 2674, 'n_words': 634}