<commit_before>#!/bin/bash

 # Copyright 2022 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #      http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.

# Running terraform scripts with storing required files in GCS bucket 
BUCKET_NAME=$1
gsutil cp -r ../../resource/* gs://$BUCKET_NAME
mkdir -p ../../binaries
# unzip tool
curl -fSL "http://ftp.de.debian.org/debian/pool/main/u/unzip/unzip_6.0-23+deb10u2_amd64.deb" -o ../binaries/unzip_6.0-23+deb10u2_amd64.deb
curl -fSL "http://mirror.centos.org/centos/7/os/x86_64/Packages/unzip-6.0-21.el7.x86_64.rpm" -o ../binaries/unzip-6.0-21.el7.x86_64.rpm

# nifi 
curl -fSL "https://archive.apache.org/dist/nifi/1.15.3/nifi-1.15.3-bin.zip" -o ../binaries/nifi-1.15.3-bin.zip

# nifi toolkit
curl -fSL "https://archive.apache.org/dist/nifi/1.15.3/nifi-toolkit-1.15.3-bin.zip" -o ../binaries/nifi-toolkit-1.15.3-bin.zip

# zookeeper
curl -fSL "https://downloads.apache.org/zookeeper/zookeeper-3.8.0/apache-zookeeper-3.8.0-bin.tar.gz" -o ../binaries/apache-zookeeper-3.8.0-bin.tar.gz

# jdk 11
curl -fSL "https://download.java.net/openjdk/jdk11/ri/openjdk-11+28_linux-x64_bin.tar.gz" -o ../binaries/openjdk-11+28_linux-x64_bin.tar.gz

# Copying resource folder to GCS bucket
gsutil cp -r ../../binaries/* gs://$BUCKET_NAME/binaries

# If you are using an existing network & subnetwork, comment all the lines in network.tf and firewall.tf and use the network id of your own network OR shared VPC
# in network_interface block in nifi.tf, nifi-ca.tf & zookeeper.tf with the below format:
# network_interface {  
#    network            = "projects/$ProjectName//global/networks/$NetworkName
#    subnetwork         = "projects/$ProjectName/regions/$SubnetRegion/subnetworks/$SubnetName"
#}
<commit_msg>Update run.sh<commit_after>#!/bin/bash

 # Copyright 2022 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #      http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.

# Running terraform scripts with storing required files in GCS bucket 
BUCKET_NAME=$1
gsutil cp -r ../../resource/* gs://$BUCKET_NAME
mkdir -p ../../binaries
# unzip tool
curl -fSL "http://ftp.de.debian.org/debian/pool/main/u/unzip/unzip_6.0-23+deb10u2_amd64.deb" -o ../../binaries/unzip_6.0-23+deb10u2_amd64.deb
curl -fSL "http://mirror.centos.org/centos/7/os/x86_64/Packages/unzip-6.0-21.el7.x86_64.rpm" -o ../../binaries/unzip-6.0-21.el7.x86_64.rpm

# nifi 
curl -fSL "https://archive.apache.org/dist/nifi/1.15.3/nifi-1.15.3-bin.zip" -o ../../binaries/nifi-1.15.3-bin.zip

# nifi toolkit
curl -fSL "https://archive.apache.org/dist/nifi/1.15.3/nifi-toolkit-1.15.3-bin.zip" -o ../../binaries/nifi-toolkit-1.15.3-bin.zip

# zookeeper
curl -fSL "https://downloads.apache.org/zookeeper/zookeeper-3.8.0/apache-zookeeper-3.8.0-bin.tar.gz" -o ../../binaries/apache-zookeeper-3.8.0-bin.tar.gz

# jdk 11
curl -fSL "https://download.java.net/openjdk/jdk11/ri/openjdk-11+28_linux-x64_bin.tar.gz" -o ../../binaries/openjdk-11+28_linux-x64_bin.tar.gz

# Copying resource folder to GCS bucket
gsutil cp -r ../../binaries/* gs://$BUCKET_NAME/binaries

# If you are using an existing network & subnetwork, comment all the lines in network.tf and firewall.tf and use the network id of your own network OR shared VPC
# in network_interface block in nifi.tf, nifi-ca.tf & zookeeper.tf with the below format:
# network_interface {  
#    network            = "projects/$ProjectName//global/networks/$NetworkName
#    subnetwork         = "projects/$ProjectName/regions/$SubnetRegion/subnetworks/$SubnetName"
#}

{'fraction_non_alphanumeric': 0.1421147763217352, 'fraction_numerical': 0.0447356529597831, 'mean_word_length': 3.370187561697927, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 8, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3206576', 'n_tokens_mistral': 1702, 'n_tokens_neox': 1581, 'n_words': 391}