<commit_before>#!/usr/bin/env bash

curl https://raw.githubusercontent.com/edinburghlivinglab/dds-notebooks/master/requirements.txt > requirements.txt
curl https://raw.githubusercontent.com/CamDavidsonPilon/Probabilistic-Programming-and-Bayesian-Methods-for-Hackers/master/requirements.txt > ppbmh_requirements.txt

# have to remove certain packages
cp requirements.txt requirements.txt.backup
cat requirements.txt.backup | grep -v igraph > requirements.txt
cat requirements.txt | grep -v jupyterhub > requirements.txt.2
cp requirements.txt requirements.txt.3

# clone in dds lab
git clone https://github.com/edinburghlivinglab/dds-notebooks.git

for PYTHONVER in 2 3 ; do
  PIP="pip$PYTHONVER"
  PYTHON="python$PYTHONVER"
  REQ="requirements.txt.$PYTHONVER"
  $PIP install --upgrade pip
  # Installing living lab requirements
  $PIP install -r $REQ
  # and install extra requirements (have to do this due to wsgiref)
  cat ppbmh_requirements.txt | grep -v wsgiref | xargs $PIP install
  $PIP install scikit-learn
  $PIP install nltk
  # finally install dds_notebooks
  cd dds-notebooks
  $PIP install --no-deps -e .
  cd ..
done

# Reduce the image size
apt-get autoremove -y<commit_msg>include installing extra requirements in the build process<commit_after>#!/usr/bin/env bash

curl https://raw.githubusercontent.com/edinburghlivinglab/dds-notebooks/master/requirements.txt > requirements.txt
curl https://raw.githubusercontent.com/edinburghlivinglab/dds-notebooks/master/extra_requirements.txt > extra_requirements.txt
curl https://raw.githubusercontent.com/CamDavidsonPilon/Probabilistic-Programming-and-Bayesian-Methods-for-Hackers/master/requirements.txt > ppbmh_requirements.txt

# have to remove certain packages
cp requirements.txt requirements.txt.backup
cat requirements.txt.backup | grep -v igraph > requirements.txt
cat requirements.txt | grep -v jupyterhub > requirements.txt.2
cp requirements.txt requirements.txt.3

# clone in dds lab
git clone https://github.com/edinburghlivinglab/dds-notebooks.git

for PYTHONVER in 2 3 ; do
  PIP="pip$PYTHONVER"
  PYTHON="python$PYTHONVER"
  REQ="requirements.txt.$PYTHONVER"
  $PIP install --upgrade pip
  # Installing living lab requirements
  $PIP install -r $REQ
  # Install more requirements, just in case we need them later
  $PIP install -r extra_requirements.txt
  # and install extra requirements (have to do this due to wsgiref)
  cat ppbmh_requirements.txt | grep -v wsgiref | xargs $PIP install
  $PIP install scikit-learn
  $PIP install nltk
  # finally install dds_notebooks
  cd dds-notebooks
  $PIP install --no-deps -e .
  cd ..
done

# Reduce the image size
apt-get autoremove -y
{'fraction_non_alphanumeric': 0.09200603318250378, 'fraction_numerical': 0.0030165912518853697, 'mean_word_length': 4.481404958677686, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 12, 'https://': 7, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2344766', 'n_tokens_mistral': 910, 'n_tokens_neox': 879, 'n_words': 227}