<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8

# # Initializing new VM environment with docker, text analysis tools, etc.
# RUN THIS SCRIPT AS SUPER-USER, i.e: `sudo bash init_VM.sh`

# Install latest python 3, pip, and easy-install:
apt-get install python3
pip install --upgrade pip
apt-get install python3-pip

# Docker environment:
#apt-get install docker-machine
#docker-machine create default
#docker-machine env default # Run the lines returned by this command
pip install docker
pip install docker-compose

# Scraping web content and URLs:
pip install bs4
pip install lxml
pip install google
pip install https://github.com/slimkrazy/python-google-places/zipball/master
pip install selenium
ansible-galaxy install cmprescott.chrome

# Processing text:
pip3 install nltk
pip3 install pandas
pip3 install tqdm
pip install sklearn
pip install gensim
pip install scipy
pip3 install timeout_decorator

# Shell utilities:
apt install htop # More readable version of top, for process management
apt install ncdu # Fast, comprehensive disk investigation
byobu-enable # Make sure window management software is turned on

# Setting up git-LFS:
curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
apt-get install git-lfs
git lfs install
git lfs track "*.csv" # Possibly add other file types here
git config --global push.default simple

# Set user permissions with custom playbook:
ansible-playbook jetstream-playbook.yaml<commit_msg>install box SDK as part of initializing VM<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8

# # Initializing new VM environment with docker, text analysis tools, etc.
# RUN THIS SCRIPT AS SUPER-USER, i.e: `sudo bash init_VM.sh`

# Install latest python 3, pip, and easy-install:
apt-get install python3
pip install --upgrade pip
apt-get install python3-pip

# Docker environment:
#apt-get install docker-machine
#docker-machine create default
#docker-machine env default # Run the lines returned by this command
pip install docker
pip install docker-compose

# Scraping web content and URLs:
pip install bs4
pip install lxml
pip install google
pip install https://github.com/slimkrazy/python-google-places/zipball/master
pip install selenium
ansible-galaxy install cmprescott.chrome

# Processing text:
pip3 install nltk
pip3 install pandas
pip3 install tqdm
pip install sklearn
pip install gensim
pip install scipy
pip3 install timeout_decorator

# Shell utilities:
apt install htop # More readable version of top, for process management
apt install ncdu # Fast, comprehensive disk investigation
byobu-enable # Make sure window management software is turned on

# Setting up git-LFS:
curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
apt-get install git-lfs
git lfs install
git lfs track "*.csv" # Possibly add other file types here
git config --global push.default simple

# Install Box SDK for working with files
pip install boxsdk

# Set user permissions with custom playbook:
ansible-playbook jetstream-playbook.yaml
{'fraction_non_alphanumeric': 0.06843483955468238, 'fraction_numerical': 0.005893909626719057, 'mean_word_length': 4.742481203007519, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3555447', 'n_tokens_mistral': 963, 'n_tokens_neox': 867, 'n_words': 368}