.github/workflows/gru_pipeline.yaml

name: GRU Pipeline

on:
  push:
    branches:
      - main
    tags:
      - '*'   
  schedule:
    - cron: "0 7 * * *"
    # 14 - 7 = 7


jobs:
  scraping_extracting:
    name: Scraping and Extracting
    runs-on: ubuntu-latest
    environment: Scraping and Extraction

    steps:
      - name: Set global directory
        run: git config --global --add safe.directory /github/workspace

      - uses: actions/checkout@v3     
        with:
          lfs: true
          persist-credentials: false
          fetch-depth: 1

      - name: Scraping Yahoo Finance
        run: |
          mkdir datasets
          go run scraper.go \
            --symbols-file=./postman/symbols.json

      - name: Zip Datasets
        run: zip -r datasets.zip datasets

      - name: Store Datasets to Google Drive
        uses: adityak74/google-drive-upload-git-action@main
        with:
          credentials: ${{ secrets.GDRIVE_GRU_CRED }}
          filename: datasets.zip
          folderId: ${{ secrets.GDRIVE_GRU_ID }}
          name: datasets.zip
          overwrite: "true"

      - name: Upload Artifact (datasets)
        uses: actions/upload-artifact@v3
        with:
          name: datasets
          path: datasets.zip

      - name: Remove Temporarary Files and Directories
        run: |
          rm datasets.zip
          rm -rf datasets


  preprocessing_training:
    name: Preprocessing and Training 
    runs-on: ubuntu-latest
    needs: scraping_extracting
    environment: Preprocessing and Training 

    steps:
      - name: Set global directory
        run: git config --global --add safe.directory /github/workspace

      - uses: actions/checkout@v3     
        with:
          lfs: true
          persist-credentials: false
          fetch-depth: 1

      - name: Install Libraries
        run: pip install -r dev.requirements.txt

      - name: Download Artifact (datasets)
        uses: actions/download-artifact@v3
        with:
          name: datasets

      - name: Install Cython and Build ML Module
        run: |
          sudo apt-get update && \
          sudo apt-get install -y \
            gcc python3-dev gnupg

          pip install cython
          cd training && \
            python setup.py build_ext --inplace && cd ..

      - name: Modeling and Training
        run: |
          unzip datasets.zip

          mkdir models
          mkdir pickles
          mkdir posttrained

          python trainingcli.py \
            --epochs=200        \
            --batchs=32         \
            --sequences=60      \
            --algorithm=GRU
          
      - name: Zip Posttrained, Models, and Pickles
        run: |
          zip -r models.zip models
          zip -r pickles.zip pickles
          zip -r posttrained.zip posttrained
          
      - name: Store Models to Google Drive
        uses: adityak74/google-drive-upload-git-action@main
        with:
          credentials: ${{ secrets.GDRIVE_GRU_CRED }}
          filename: models.zip
          folderId: ${{ secrets.GDRIVE_GRU_ID }}
          name: models.zip
          overwrite: "true"

      - name: Store Pickles to Google Drive
        uses: adityak74/google-drive-upload-git-action@main
        with:
          credentials: ${{ secrets.GDRIVE_GRU_CRED }}
          filename: pickles.zip
          folderId: ${{ secrets.GDRIVE_GRU_ID }}
          name: pickles.zip
          overwrite: "true"

      - name: Store Posttrained to Google Drive
        uses: adityak74/google-drive-upload-git-action@main
        with:
          credentials: ${{ secrets.GDRIVE_GRU_CRED }}
          filename: posttrained.zip
          folderId: ${{ secrets.GDRIVE_GRU_ID }}
          name: posttrained.zip
          overwrite: "true"

      - name: Remove Temporarary Files and Directories
        run: |
          rm models.zip
          rm pickles.zip
          rm datasets.zip
          rm posttrained.zip

          rm -rf models
          rm -rf pickles
          rm -rf datasets
          rm -rf posttrained

      - name: Flush All Caching Data (Redis)
        run: |
          sudo apt install -y redis-tools
          echo "FLUSHALL" | redis-cli --tls \
            -h ${{ secrets.REDIS_HOST  }}   \
            -p ${{ secrets.REDIS_PORT  }}   \
            -a ${{ secrets.REDIS_TOKEN }}


  tebakaja_crypto_space-0:
    name: crypto-forecast-svc-0
    runs-on: ubuntu-latest
    needs: preprocessing_training
    environment: Deployment

    env:
      HF_TOKEN: ${{ secrets.HF_TOKEN }}
      SPACE_NAME: tebakaja_cryptocurrency_space-0
      HF_USERNAME: tebakaja

    steps:
      - name: Set global directory
        run: git config --global --add safe.directory /github/workspace

      - uses: actions/checkout@v3     
        with:
          persist-credentials: false
          fetch-depth: 1000

      - name: Check git status
        run: git status

      - name: Configure git
        run: |
          git config --local user.email "[email protected]"
          git config --local user.name "qywok"

      - name: Pull changes from remote
        run: |
          git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
          (git merge --strategy-option theirs)

      - name: Add and commit changes
        run: |
          git add -A 
          git diff-index --quiet HEAD || git commit -m "Model Deployment"

      - name: Push to Hugging Face
        run: |
          git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force


  tebakaja_crypto_space-1:
    name: crypto-forecast-svc-1
    runs-on: ubuntu-latest
    needs: preprocessing_training
    environment: Deployment

    env:
      HF_TOKEN: ${{ secrets.HF_TOKEN }}
      SPACE_NAME: tebakaja_cryptocurrency_space-1
      HF_USERNAME: tebakaja

    steps:
      - name: Set global directory
        run: git config --global --add safe.directory /github/workspace

      - uses: actions/checkout@v3     
        with:
          persist-credentials: false
          fetch-depth: 1000

      - name: Check git status
        run: git status

      - name: Configure git
        run: |
          git config --local user.email "[email protected]"
          git config --local user.name "qywok"

      - name: Pull changes from remote
        run: |
          git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
          (git merge --strategy-option theirs)

      - name: Add and commit changes
        run: |
          git add -A 
          git diff-index --quiet HEAD || git commit -m "Model Deployment"

      - name: Push to Hugging Face
        run: |
          git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force


  tebakaja_crypto_space-2:
    name: crypto-forecast-svc-2
    runs-on: ubuntu-latest
    needs: preprocessing_training
    environment: Deployment

    env:
      HF_TOKEN: ${{ secrets.HF_TOKEN }}
      SPACE_NAME: tebakaja_cryptocurrency_space-2
      HF_USERNAME: tebakaja

    steps:
      - name: Set global directory
        run: git config --global --add safe.directory /github/workspace

      - uses: actions/checkout@v3     
        with:
          persist-credentials: false
          fetch-depth: 1000

      - name: Check git status
        run: git status

      - name: Configure git
        run: |
          git config --local user.email "[email protected]"
          git config --local user.name "qywok"

      - name: Pull changes from remote
        run: |
          git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
          (git merge --strategy-option theirs)

      - name: Add and commit changes
        run: |
          git add -A 
          git diff-index --quiet HEAD || git commit -m "Model Deployment"

      - name: Push to Hugging Face
        run: |
          git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force


  tebakaja_crypto_space-3:
    name: crypto-forecast-svc-3
    runs-on: ubuntu-latest
    needs: preprocessing_training
    environment: Deployment

    env:
      HF_TOKEN: ${{ secrets.HF_TOKEN }}
      SPACE_NAME: tebakaja_cryptocurrency_space-3
      HF_USERNAME: tebakaja

    steps:
      - name: Set global directory
        run: git config --global --add safe.directory /github/workspace

      - uses: actions/checkout@v3     
        with:
          persist-credentials: false
          fetch-depth: 1000

      - name: Check git status
        run: git status

      - name: Configure git
        run: |
          git config --local user.email "[email protected]"
          git config --local user.name "qywok"

      - name: Pull changes from remote
        run: |
          git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
          (git merge --strategy-option theirs)

      - name: Add and commit changes
        run: |
          git add -A 
          git diff-index --quiet HEAD || git commit -m "Model Deployment"

      - name: Push to Hugging Face
        run: |
          git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force


  tebakaja_crypto_space-4:
    name: crypto-forecast-svc-4
    runs-on: ubuntu-latest
    needs: preprocessing_training
    environment: Deployment

    env:
      HF_TOKEN: ${{ secrets.HF_TOKEN }}
      SPACE_NAME: tebakaja_cryptocurrency_space-4
      HF_USERNAME: tebakaja

    steps:
      - name: Set global directory
        run: git config --global --add safe.directory /github/workspace

      - uses: actions/checkout@v3     
        with:
          persist-credentials: false
          fetch-depth: 1000

      - name: Check git status
        run: git status

      - name: Configure git
        run: |
          git config --local user.email "[email protected]"
          git config --local user.name "qywok"

      - name: Pull changes from remote
        run: |
          git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
          (git merge --strategy-option theirs)

      - name: Add and commit changes
        run: |
          git add -A 
          git diff-index --quiet HEAD || git commit -m "Model Deployment"

      - name: Push to Hugging Face
        run: |
          git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force


  functional_testing:
    name: Functional Testing
    runs-on: ubuntu-latest

    needs:
      - tebakaja_crypto_space-0
      - tebakaja_crypto_space-1
      - tebakaja_crypto_space-2
      - tebakaja_crypto_space-3
      - tebakaja_crypto_space-4

    steps:
      - name: Set global directory
        run: git config --global --add safe.directory /github/workspace

      - uses: actions/checkout@v3     
        with:
          persist-credentials: false
          fetch-depth: 1

      - name: Testing Proxy Endpoints
        run: |
          sleep 60
          chmod +x endpoints_test.sh && ./endpoints_test.sh