<commit_before>name: Self-hosted runner (scheduled)

on:
  push:
    branches:
      - ci_*
  repository_dispatch:
  schedule:
    - cron: "0 0 * * *"

jobs:
  run_all_tests_torch_and_tf_gpu:
    runs-on: [self-hosted, single-gpu]
    steps:
      - uses: actions/checkout@v2

      - name: Loading cache.
        uses: actions/cache@v2
        id: cache
        with:
          path: .env
          key: v0-slow_tests_tf_torch_gpu-${{ hashFiles('setup.py') }}

      - name: Python version
        run: |
          which python
          python --version
          pip --version
          
      - name: Current dir
        run: pwd
      - run: nvidia-smi
        
      - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          python -m venv .env
          source .env/bin/activate
          which python
          python --version
          pip --version
          
      - name: Install dependencies
        run: |
          source .env/bin/activate
          pip install --upgrade pip
          pip install torch!=1.6.0
          pip install .[sklearn,testing,onnxruntime]
          pip install git+https://github.com/huggingface/datasets

      - name: Are GPUs recognized by our DL frameworks
        run: |
          source .env/bin/activate
          python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
          python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"

      - name: Run all tests on GPU
        env:
          TF_FORCE_GPU_ALLOW_GROWTH: "true"
          OMP_NUM_THREADS: 1
          RUN_SLOW: yes
        run: |
          source .env/bin/activate
          python -m pytest -n 1 --dist=loadfile -s --make_reports=tests tests

      - name: Run examples tests on GPU
        env:
          TF_FORCE_GPU_ALLOW_GROWTH: "true"
          OMP_NUM_THREADS: 1
          RUN_SLOW: yes
        run: |
          source .env/bin/activate
          pip install -r examples/requirements.txt
          python -m pytest -n 1 --dist=loadfile -s --make_reports=examples examples

      - name: Run all pipeline tests on GPU
        env:
          TF_FORCE_GPU_ALLOW_GROWTH: "true"
          OMP_NUM_THREADS: 1
          RUN_SLOW: yes
          RUN_PIPELINE_TESTS: yes
        run: |
          source .env/bin/activate
          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make_reports=tests_pipeline tests 

      - name: test suite reports artifacts
        uses: actions/upload-artifact@v2
        with:
          name: test_reports
          path: reports


  run_all_tests_torch_and_tf_multiple_gpu:
    runs-on: [self-hosted, multi-gpu]
    steps:
      - uses: actions/checkout@v2

      - name: Loading cache.
        uses: actions/cache@v2
        id: cache
        with:
          path: .env
          key: v0-slow_tests_tf_torch_multi_gpu-${{ hashFiles('setup.py') }}

      - name: Python version
        run: |
          which python
          python --version
          pip --version
      - name: Current dir
        run: pwd
      - run: nvidia-smi<commit_msg>[gh actions] run artifacts job always (#8110)<commit_after>name: Self-hosted runner (scheduled)

on:
  push:
    branches:
      - ci_*
  repository_dispatch:
  schedule:
    - cron: "0 0 * * *"

jobs:
  run_all_tests_torch_and_tf_gpu:
    runs-on: [self-hosted, single-gpu]
    steps:
      - uses: actions/checkout@v2

      - name: Loading cache.
        uses: actions/cache@v2
        id: cache
        with:
          path: .env
          key: v0-slow_tests_tf_torch_gpu-${{ hashFiles('setup.py') }}

      - name: Python version
        run: |
          which python
          python --version
          pip --version
          
      - name: Current dir
        run: pwd
      - run: nvidia-smi
        
      - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          python -m venv .env
          source .env/bin/activate
          which python
          python --version
          pip --version
          
      - name: Install dependencies
        run: |
          source .env/bin/activate
          pip install --upgrade pip
          pip install torch!=1.6.0
          pip install .[sklearn,testing,onnxruntime]
          pip install git+https://github.com/huggingface/datasets

      - name: Are GPUs recognized by our DL frameworks
        run: |
          source .env/bin/activate
          python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
          python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"

      - name: Run all tests on GPU
        env:
          TF_FORCE_GPU_ALLOW_GROWTH: "true"
          OMP_NUM_THREADS: 1
          RUN_SLOW: yes
        run: |
          source .env/bin/activate
          python -m pytest -n 1 --dist=loadfile -s --make_reports=tests tests

      - name: Run examples tests on GPU
        env:
          TF_FORCE_GPU_ALLOW_GROWTH: "true"
          OMP_NUM_THREADS: 1
          RUN_SLOW: yes
        run: |
          source .env/bin/activate
          pip install -r examples/requirements.txt
          python -m pytest -n 1 --dist=loadfile -s --make_reports=examples examples

      - name: Run all pipeline tests on GPU
        env:
          TF_FORCE_GPU_ALLOW_GROWTH: "true"
          OMP_NUM_THREADS: 1
          RUN_SLOW: yes
          RUN_PIPELINE_TESTS: yes
        run: |
          source .env/bin/activate
          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make_reports=tests_pipeline tests 

      - name: test suite reports artifacts
        if: ${{ always() }}
        uses: actions/upload-artifact@v2
        with:
          name: test_reports
          path: reports


  run_all_tests_torch_and_tf_multiple_gpu:
    runs-on: [self-hosted, multi-gpu]
    steps:
      - uses: actions/checkout@v2

      - name: Loading cache.
        uses: actions/cache@v2
        id: cache
        with:
          path: .env
          key: v0-slow_tests_tf_torch_multi_gpu-${{ hashFiles('setup.py') }}

      - name: Python version
        run: |
          which python
          python --version
          pip --version
      - name: Current dir
        run: pwd
      - run: nvidia-smi
{'fraction_non_alphanumeric': 0.09157566302652106, 'fraction_numerical': 0.0062402496099844, 'mean_word_length': 1.6568586821384168, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6628606', 'n_tokens_mistral': 2021, 'n_tokens_neox': 1949, 'n_words': 527}