<commit_before><commit_msg>Add github action<commit_after>on: [push, pull_request]
name: Test

jobs:
  test:
    strategy:
      matrix:
        go-version: [1.13.x, 1.14.x]
        platform: [ubuntu-latest, macos-latest, windows-latest]
      fail-fast: false
    runs-on: ${{ matrix.platform }}
    env:
      GO111MODULE: on

    steps:
    - name: Install Go
      uses: actions/setup-go@v1
      with:
        go-version: ${{ matrix.go-version }}

    - name: Go environment
      run: |
        go version
        go env
        echo PATH is $PATH
      shell: bash

    - name: Checkout code
      uses: actions/checkout@v1

    - name: Check gofmt
      run: |
        gofmt -l .
        test `gofmt -l . | wc -l` = 0
      shell: bash

    - name: Run tests
      run: |
        go test -race -v ./...
      shell: bash

{'fraction_non_alphanumeric': 0.11821471652593486, 'fraction_numerical': 0.014475271411338963, 'mean_word_length': 1.6182965299684542, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2819577', 'n_tokens_mistral': 297, 'n_tokens_neox': 277, 'n_words': 77}