<commit_before>name: Release-Deployment CI

on:
  release:
    types: [created]

jobs:

  lint:
    runs-on: ${{ matrix.os }}

    strategy:
      fail-fast: false
      matrix:
        python-version: [ 3.9 ]
        os: [ ubuntu-latest ]

    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          pip install -U pip setuptools
          pip install -r dev-requirements.txt
          pip uninstall -y charset-normalizer
      - name: Install the package
        run: |
          python setup.py install
      - name: Type checking (Mypy)
        run: |
          mypy charset_normalizer
      - name: Import sorting check (isort)
        run: |
          isort --check charset_normalizer
      - name: Code format (Black)
        run: |
          black --check --diff --target-version=py35 charset_normalizer
      - name: Style guide enforcement (Flake8)
        run: |
          flake8 charset_normalizer

  tests:
    runs-on: ${{ matrix.os }}
    needs:
      - lint

    strategy:
      fail-fast: false
      matrix:
        python-version: [ 3.6, 3.7, 3.8, 3.9, "3.10" ]
        os: [ ubuntu-latest ]

    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          pip install -U pip setuptools
          pip install -r dev-requirements.txt
          pip uninstall -y charset-normalizer
      - name: Install the package
        run: |
          python setup.py install
      - name: Run tests
        run: |
          pytest

  detection_coverage:
    runs-on: ${{ matrix.os }}
    needs:
      - tests

    strategy:
      fail-fast: false
      matrix:
        python-version: [ 3.9, "3.10" ]
        os: [ ubuntu-latest ]

    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          pip install -U pip setuptools
          pip install -r dev-requirements.txt
          pip uninstall -y charset-normalizer
      - name: Install the package
        run: |
          python setup.py install
      - name: Clone the complete dataset
        run: |
          git clone https://github.com/Ousret/char-dataset.git
      - name: Coverage WITH preemptive
        run: |
          python ./bin/coverage.py --coverage 97 --with-preemptive
      - name: Coverage WITHOUT preemptive
        run: |
          python ./bin/coverage.py --coverage 95
      - name: BC Coverage (With Chardet)
        run: |
          python ./bin/bc.py --coverage 80

  integration:
    runs-on: ${{ matrix.os }}
    needs:
      - detection_coverage
    strategy:
      fail-fast: false
      matrix:
        python-version: [ 3.9, "3.10" ]
        os: [ ubuntu-latest ]

    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          pip install -U pip setuptools
          pip install -r dev-requirements.txt
      - name: Remove Chardet & Charset-Normalizer
        run: |
          pip uninstall -y chardet
          pip uninstall -y charset-normalizer
      - name: Install the package
        run: |
          python setup.py install
      - name: Clone the complete dataset
        run: |
          git clone https://github.com/Ousret/char-dataset.git
      - name: Start the Flask server
        run: |
          python ./bin/serve.py &
      - name: Integration Tests with Requests
        run: |
          python ./bin/integration.py
  universal-wheel:
    runs-on: ubuntu-latest
    needs:
      - integration
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: '3.x'
      - name: Update pip, setuptools, wheel and twine
        run: |
          python -m pip install --upgrade pip
          pip install setuptools wheel twine
      - name: Build Wheel
        env:
          CHARSET_NORMALIZER_USE_MYPYC: '0'
        run: python setup.py sdist bdist_wheel
      - name: Upload artifacts
        uses: actions/upload-artifact@v3
        with:
          name: dist
          path: dist

  build-wheels:
    name: Build wheels on ${{ matrix.os }} ${{ matrix.qemu }}
    runs-on: ${{ matrix.os }}-latest
    needs: universal-wheel
    strategy:
      matrix:
        os: [ ubuntu, windows, macos ]
        qemu: [ '' ]
        python-version: [ 3.6, 3.7, 3.8, 3.9, "3.10" ]
        include:
          # Split ubuntu job for the sake of speed-up
          - os: ubuntu
            qemu: aarch64
          - os: ubuntu
            qemu: ppc64le
          - os: ubuntu
            qemu: s390x
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: true
      - name: Set up QEMU
        if: ${{ matrix.qemu }}
        uses: docker/setup-qemu-action@v2
        with:
          platforms: all
        id: qemu
      - name: Prepare emulation
        run: |
          if [[ -n "${{ matrix.qemu }}" ]]; then
            # Build emulated architectures only if QEMU is set,
            # use default "auto" otherwise
            echo "CIBW_ARCHS_LINUX=${{ matrix.qemu }}" >> $GITHUB_ENV
          fi
        shell: bash
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Update pip, wheel, setuptools, build, twine
        run: |
          python -m pip install -U pip wheel setuptools build twine
      - name: Install dev-dependencies
        run: |
          pip install -r dev-requirements.txt
      - name: Build Wheel
        env:
          CHARSET_NORMALIZER_USE_MYPYC: '1'
        run: python setup.py sdist bdist_wheel
      - name: Upload artifacts
        uses: actions/upload-artifact@v3
        with:
          name: dist
          path: dist
<commit_msg>Use cibuildwheel action<commit_after>name: Release-Deployment CI

on:
  workflow_dispatch: 
  release:
    types: [created]

jobs:

  lint:
    runs-on: ${{ matrix.os }}

    strategy:
      fail-fast: false
      matrix:
        python-version: [ 3.9 ]
        os: [ ubuntu-latest ]

    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          pip install -U pip setuptools
          pip install -r dev-requirements.txt
          pip uninstall -y charset-normalizer
      - name: Install the package
        run: |
          python setup.py install
      - name: Type checking (Mypy)
        run: |
          mypy charset_normalizer
      - name: Import sorting check (isort)
        run: |
          isort --check charset_normalizer
      - name: Code format (Black)
        run: |
          black --check --diff --target-version=py35 charset_normalizer
      - name: Style guide enforcement (Flake8)
        run: |
          flake8 charset_normalizer

  tests:
    runs-on: ${{ matrix.os }}
    needs:
      - lint

    strategy:
      fail-fast: false
      matrix:
        python-version: [ 3.6, 3.7, 3.8, 3.9, "3.10" ]
        os: [ ubuntu-latest ]

    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          pip install -U pip setuptools
          pip install -r dev-requirements.txt
          pip uninstall -y charset-normalizer
      - name: Install the package
        run: |
          python setup.py install
      - name: Run tests
        run: |
          pytest

  detection_coverage:
    runs-on: ${{ matrix.os }}
    needs:
      - tests

    strategy:
      fail-fast: false
      matrix:
        python-version: [ 3.9, "3.10" ]
        os: [ ubuntu-latest ]

    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          pip install -U pip setuptools
          pip install -r dev-requirements.txt
          pip uninstall -y charset-normalizer
      - name: Install the package
        run: |
          python setup.py install
      - name: Clone the complete dataset
        run: |
          git clone https://github.com/Ousret/char-dataset.git
      - name: Coverage WITH preemptive
        run: |
          python ./bin/coverage.py --coverage 97 --with-preemptive
      - name: Coverage WITHOUT preemptive
        run: |
          python ./bin/coverage.py --coverage 95
      - name: BC Coverage (With Chardet)
        run: |
          python ./bin/bc.py --coverage 80

  integration:
    runs-on: ${{ matrix.os }}
    needs:
      - detection_coverage
    strategy:
      fail-fast: false
      matrix:
        python-version: [ 3.9, "3.10" ]
        os: [ ubuntu-latest ]

    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          pip install -U pip setuptools
          pip install -r dev-requirements.txt
      - name: Remove Chardet & Charset-Normalizer
        run: |
          pip uninstall -y chardet
          pip uninstall -y charset-normalizer
      - name: Install the package
        run: |
          python setup.py install
      - name: Clone the complete dataset
        run: |
          git clone https://github.com/Ousret/char-dataset.git
      - name: Start the Flask server
        run: |
          python ./bin/serve.py &
      - name: Integration Tests with Requests
        run: |
          python ./bin/integration.py
  universal-wheel:
    runs-on: ubuntu-latest
    needs:
      - integration
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: '3.x'
      - name: Update pip, setuptools, wheel and twine
        run: |
          python -m pip install --upgrade pip
          pip install setuptools wheel twine
      - name: Build Wheel
        env:
          CHARSET_NORMALIZER_USE_MYPYC: '0'
        run: python setup.py sdist bdist_wheel
      - name: Upload artifacts
        uses: actions/upload-artifact@v3
        with:
          name: dist
          path: dist

  build-wheels:
    name: Build wheels on ${{ matrix.os }} ${{ matrix.qemu }}
    runs-on: ${{ matrix.os }}-latest
    needs: universal-wheel
    strategy:
      matrix:
        os: [ ubuntu, windows, macos ]
        qemu: [ '' ]
        include:
          # Split ubuntu job for the sake of speed-up
          - os: ubuntu
            qemu: aarch64
          - os: ubuntu
            qemu: ppc64le
          - os: ubuntu
            qemu: s390x
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: true
      - name: Set up QEMU
        if: ${{ matrix.qemu }}
        uses: docker/setup-qemu-action@v2
        with:
          platforms: all
        id: qemu
      - name: Prepare emulation
        run: |
          if [[ -n "${{ matrix.qemu }}" ]]; then
            # Build emulated architectures only if QEMU is set,
            # use default "auto" otherwise
            echo "CIBW_ARCHS_LINUX=${{ matrix.qemu }}" >> $GITHUB_ENV
          fi
        shell: bash
      - name: Setup Python
        uses: actions/setup-python@v4
      - name: Update pip, wheel, setuptools, build, twine
        run: |
          python -m pip install -U pip wheel setuptools build twine
      - name: Install dev-dependencies
        run: |
          pip install -r dev-requirements.txt
      - name: Build wheels
        uses: pypa/cibuildwheel@2.7.0
        env:
          CIBW_ARCHS_MACOS: x86_64 arm64 universal2
          CIBW_ENVIRONMENT: CHARSET_NORMALIZER_USE_MYPYC='1'
          CHARSET_NORMALIZER_USE_MYPYC: '1'
      - name: Upload artifacts
        uses: actions/upload-artifact@v3
        with:
          name: dist
          path: dist

{'fraction_non_alphanumeric': 0.09874815333177825, 'fraction_numerical': 0.01073011429904362, 'mean_word_length': 1.5952380952380953, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2347026', 'n_tokens_mistral': 3971, 'n_tokens_neox': 3800, 'n_words': 1063}