<commit_before>version: 2

jobs:
  build:
    docker:
      - image: circleci/node:10
    working_directory: ~/repo
    steps:
      - checkout
      - restore_cache:
          keys:
          - v1-dependencies-{{ checksum "package-lock.json" }}
          - v1-dependencies- # used if checksum fails
      - run: npm install
      - save_cache:
          paths:
            - node_modules
          key: v1-dependencies-{{ checksum "package-lock.json" }}
      - run: npm test
      - run: npm run build
  deploy:
    docker:
      - image: circleci/node:10
    working_directory: ~/repo
    environment:
      - SOURCE_BRANCH: master
      - TARGET_BRANCH: gh-pages
    steps:
      - checkout
      - restore_cache:
          keys:
          - v1-dependencies-{{ checksum "package.json" }}
          - v1-dependencies-
      - run: npm install
      - save_cache:
          paths:
            - node_modules
          key: v1-dependencies-{{ checksum "package.json" }}
      - run: npm test
      - run: npm run build
      - deploy:
          name: Deploy to Github Pages
          command: |
            if [ $CIRCLE_BRANCH == $SOURCE_BRANCH ]; then
              git config --global user.email $GH_EMAIL
              git config --global user.name $GH_NAME

              git clone https://${GH_TOKEN}@github.com/developersdo/opensource.git out

              cd out
              git checkout $TARGET_BRANCH || git checkout --orphan $TARGET_BRANCH
              git rm -rf .
              cd ..

              cp -a public/. out/.

              mkdir -p out/.circleci && cp -a .circleci/. out/.circleci/.

              cd out
              git add -A
              git commit -m "Deployed: website to Github Pages (automatic): ${CIRCLE_SHA1}" --allow-empty
              git push -q https://${GH_TOKEN}@github.com/developersdo/opensource.git $TARGET_BRANCH
            fi
  scrape:
    docker:
      - image: circleci/node:10
    working_directory: ~/repo
    environment:
      - SOURCE_BRANCH: master
      - TARGET_BRANCH: gh-pages
    steps:
      - checkout
      - restore_cache:
          keys:
          - v1-dependencies-{{ checksum "package.json" }}
          - v1-dependencies-
      - run: npm install
      - save_cache:
          paths:
            - node_modules
          key: v1-dependencies-{{ checksum "package.json" }}
      - run: npm test
      - run: npm run build
      - run:
          name: Scraping Github data
          command: |
            if [ $CIRCLE_BRANCH == $SOURCE_BRANCH ]; then
              git config --global user.email $GH_EMAIL
              git config --global user.name $GH_NAME

              npm run refresh

              git add -A
              git commit -m "Updated: Github data (automatic): ${CIRCLE_SHA1}" --allow-empty
              git push -q https://${GH_TOKEN}@github.com/developersdo/opensource.git $SOURCE_BRANCH
            fi

workflows:
  version: 2
  build_scrape_and_deploy_daily:
    jobs:
      - build
      - scrape:
          requires:
            - build
      - deploy:
          requires:
            - scrape
    triggers:
      - schedule:
          cron: "0 0 * * *"
          filters:
            branches:
                only:
                  - master
  build:
    jobs:
      - build:
          filters:
            branches:
              ignore:
                - gh-pages
                - master
  build_and_deploy:
    jobs:
      - build:
          filters:
            branches:
              only:
                - master
      - deploy:
          requires:
            - build
          filters:
            branches:
              only:
                - master<commit_msg>Updated: circle ci config.<commit_after>version: 2

jobs:
  build:
    docker:
      - image: circleci/node:10
    working_directory: ~/repo
    steps:
      - checkout
      - restore_cache:
          keys:
          - v2-dependencies-{{ checksum "package-lock.json" }}
          - v2-dependencies- # used if checksum fails
      - run: npm install
      - save_cache:
          paths:
            - node_modules
          key: v2-dependencies-{{ checksum "package-lock.json" }}
      - run: npm test
      - run: npm run build
  deploy:
    docker:
      - image: circleci/node:10
    working_directory: ~/repo
    environment:
      - SOURCE_BRANCH: master
      - TARGET_BRANCH: gh-pages
    steps:
      - checkout
      - restore_cache:
          keys:
          - v2-dependencies-{{ checksum "package-lock.json" }}
          - v2-dependencies-
      - run: npm install
      - save_cache:
          paths:
            - node_modules
          key: v2-dependencies-{{ checksum "package-lock.json" }}
      - run: npm test
      - run: npm run build
      - deploy:
          name: Deploy to Github Pages
          command: |
            if [ $CIRCLE_BRANCH == $SOURCE_BRANCH ]; then
              git config --global user.email $GH_EMAIL
              git config --global user.name $GH_NAME

              git clone https://${GH_TOKEN}@github.com/developersdo/opensource.git out

              cd out
              git checkout $TARGET_BRANCH || git checkout --orphan $TARGET_BRANCH
              git rm -rf .
              cd ..

              cp -a public/. out/.

              mkdir -p out/.circleci && cp -a .circleci/. out/.circleci/.

              cd out
              git add -A
              git commit -m "Deployed: website to Github Pages (automatic): ${CIRCLE_SHA1}" --allow-empty
              git push -q https://${GH_TOKEN}@github.com/developersdo/opensource.git $TARGET_BRANCH
            fi
  scrape:
    docker:
      - image: circleci/node:10
    working_directory: ~/repo
    environment:
      - SOURCE_BRANCH: master
      - TARGET_BRANCH: gh-pages
    steps:
      - checkout
      - restore_cache:
          keys:
          - v2-dependencies-{{ checksum "package-lock.json" }}
          - v2-dependencies-
      - run: npm install
      - save_cache:
          paths:
            - node_modules
          key: v2-dependencies-{{ checksum "package-lock.json" }}
      - run:
          name: Scraping Github data
          command: |
            if [ $CIRCLE_BRANCH == $SOURCE_BRANCH ]; then
              git config --global user.email $GH_EMAIL
              git config --global user.name $GH_NAME

              npm run refresh

              git add -A
              git commit -m "Updated: Github data (automatic): ${CIRCLE_SHA1}" --allow-empty
              git push -q https://${GH_TOKEN}@github.com/developersdo/opensource.git $SOURCE_BRANCH
            fi

workflows:
  version: 2
  build_scrape_and_deploy_daily:
    jobs:
      - build
      - scrape:
          requires:
            - build
      - deploy:
          requires:
            - scrape
    triggers:
      - schedule:
          cron: "0 0 * * *"
          filters:
            branches:
                only:
                  - master
  build:
    jobs:
      - build:
          filters:
            branches:
              ignore:
                - gh-pages
                - master
  build_and_deploy:
    jobs:
      - build:
          filters:
            branches:
              only:
                - master
      - deploy:
          requires:
            - build
          filters:
            branches:
              only:
                - master
{'fraction_non_alphanumeric': 0.09460196292257361, 'fraction_numerical': 0.0057251908396946565, 'mean_word_length': 1.27715704531347, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2929502', 'n_tokens_mistral': 2208, 'n_tokens_neox': 2089, 'n_words': 525}