<commit_before>#!/opt/conda/default/bin/python3
import json
import os
import subprocess as sp
import sys
from subprocess import check_output

assert sys.version_info > (3, 0), sys.version_info

if sys.version_info >= (3, 7):
    def safe_call(*args, **kwargs):
        sp.run(args, capture_output=True, check=True, **kwargs)
else:
    def safe_call(*args, **kwargs):
        try:
            sp.check_output(args, stderr=sp.STDOUT, **kwargs)
        except sp.CalledProcessError as e:
            print(e.output).decode()
            raise e


def get_metadata(key):
    return check_output(['/usr/share/google/get_metadata_value', 'attributes/{}'.format(key)]).decode()


def mkdir_if_not_exists(path):
    try:
        os.makedirs(path)
    except OSError as e:
        if e.errno != os.errno.EEXIST:
            raise


# get role of machine (master or worker)
role = get_metadata('dataproc-role')

if role == 'Master':
    # additional packages to install
    pip_pkgs = [
        'setuptools',
        'mkl<2020',
        'lxml<5',
        'google-cloud==0.32.*',
        'jgscm<0.2',
        'ipykernel==4.10.*',
        'ipywidgets==7.4.*',
        'jupyter-console==6.0.*',
        'nbconvert==5.5.*',
        'notebook==5.7.*',
        'qtconsole==4.5.*'
    ]

    # add user-requested packages
    try:
        user_pkgs = get_metadata('PKGS')
    except Exception:
        pass
    else:
        pip_pkgs.extend(user_pkgs.split('|'))

    print('pip packages are {}'.format(pip_pkgs))
    command = ['pip', 'install']
    command.extend(pip_pkgs)
    safe_call(*command)

    print('getting metadata')

    wheel_path = get_metadata('WHEEL')
    wheel_name = wheel_path.split('/')[-1]
<commit_msg>Use Hail version of jgscm (#8079)<commit_after>#!/opt/conda/default/bin/python3
import json
import os
import subprocess as sp
import sys
from subprocess import check_output

assert sys.version_info > (3, 0), sys.version_info

if sys.version_info >= (3, 7):
    def safe_call(*args, **kwargs):
        sp.run(args, capture_output=True, check=True, **kwargs)
else:
    def safe_call(*args, **kwargs):
        try:
            sp.check_output(args, stderr=sp.STDOUT, **kwargs)
        except sp.CalledProcessError as e:
            print(e.output).decode()
            raise e


def get_metadata(key):
    return check_output(['/usr/share/google/get_metadata_value', 'attributes/{}'.format(key)]).decode()


def mkdir_if_not_exists(path):
    try:
        os.makedirs(path)
    except OSError as e:
        if e.errno != os.errno.EEXIST:
            raise


# get role of machine (master or worker)
role = get_metadata('dataproc-role')

if role == 'Master':
    # additional packages to install
    pip_pkgs = [
        'setuptools',
        'mkl<2020',
        'lxml<5',
        'google-cloud-storage==1.25.*',
        'https://github.com/hail-is/jgscm/archive/v0.1.11-hail.zip',
        'ipykernel==4.10.*',
        'ipywidgets==7.4.*',
        'jupyter-console==6.0.*',
        'nbconvert==5.5.*',
        'notebook==5.7.*',
        'qtconsole==4.5.*'
    ]

    # add user-requested packages
    try:
        user_pkgs = get_metadata('PKGS')
    except Exception:
        pass
    else:
        pip_pkgs.extend(user_pkgs.split('|'))

    print('pip packages are {}'.format(pip_pkgs))
    command = ['pip', 'install']
    command.extend(pip_pkgs)
    safe_call(*command)

    print('getting metadata')

    wheel_path = get_metadata('WHEEL')
    wheel_name = wheel_path.split('/')[-1]

{'fraction_non_alphanumeric': 0.15197481396680024, 'fraction_numerical': 0.018317115054378934, 'mean_word_length': 2.633056133056133, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 7, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5011377', 'n_tokens_mistral': 1248, 'n_tokens_neox': 1196, 'n_words': 239}