<commit_before>name: CI
on: [push, pull_request]

# Run linter with github actions for quick feedbacks.
# Run macos tests with github actions. Linux (CPU & GPU) tests currently runs on CircleCI
jobs:
  linter:
    runs-on: ubuntu-latest
    # run on PRs, or commits to facebookresearch (not internal)
    if: ${{ github.repository_owner == 'facebookresearch' || github.event_name == 'pull_request' }}
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python 3.7
        uses: actions/setup-python@v2
        with:
          python-version: 3.7
      - name: Install dependencies
        # flake8-bugbear flake8-comprehensions are useful but not available internally
        run: |
          python -m pip install --upgrade pip
          python -m pip install flake8==3.8.1 isort==4.3.21
          python -m pip install black==22.3.0
          flake8 --version
      - name: Lint
        run: |
          echo "Running isort"
          isort -c -sp .
          echo "Running black"
          black -l 100 --check .
          echo "Running flake8"
          flake8 .
<commit_msg>use py3.9 for linter CI<commit_after>name: CI
on: [push, pull_request]

# Run linter with github actions for quick feedbacks.
# Run macos tests with github actions. Linux (CPU & GPU) tests currently runs on CircleCI
jobs:
  linter:
    runs-on: ubuntu-latest
    # run on PRs, or commits to facebookresearch (not internal)
    if: ${{ github.repository_owner == 'facebookresearch' || github.event_name == 'pull_request' }}
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python 3.9
        uses: actions/setup-python@v2
        with:
          python-version: 3.9
      - name: Install dependencies
        # flake8-bugbear flake8-comprehensions are useful but not available internally
        run: |
          python -m pip install --upgrade pip
          python -m pip install flake8==3.8.1 isort==4.3.21
          python -m pip install black==22.3.0
          flake8 --version
      - name: Lint
        run: |
          echo "Running isort"
          isort -c -sp .
          echo "Running black"
          black -l 100 --check .
          echo "Running flake8"
          flake8 .

{'fraction_non_alphanumeric': 0.09335154826958106, 'fraction_numerical': 0.02459016393442623, 'mean_word_length': 2.0261707988980717, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7033207', 'n_tokens_mistral': 698, 'n_tokens_neox': 652, 'n_words': 222}