<commit_before><commit_msg>ci: add test github action<commit_after>name: Test

on:
  push:
    branches:
      - main
  pull_request:

jobs:
  test:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [3.6, 3.7, 3.8, 3.9]
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install Poetry
        run: pip install poetry
      - name: Install dependencies
        run: |
          poetry config virtualenvs.create false
          poetry install --no-interaction
      - name: Test
        run: bash scripts/test.sh
{'fraction_non_alphanumeric': 0.10229007633587786, 'fraction_numerical': 0.015267175572519083, 'mean_word_length': 1.9417040358744395, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7118383', 'n_tokens_mistral': 211, 'n_tokens_neox': 205, 'n_words': 53}