<commit_before>#!/usr/bin/env python

import subprocess
import os
import glob
import sys
from os.path import dirname, realpath, join

process_mjml = '--no-mjml' not in sys.argv[1:]


def color(num):
    def format(val):
        return '\033[' + str(num) + 'm' + val + '\033[0m'

    return format


green = color(92)
yellow = color(93)


def header(val):
    print('\n', yellow('★'), green(val), '\n')


environ = os.environ.copy()

# do not prompt if dependency source already exists, just wipe and get the required version
environ['PIP_EXISTS_ACTION'] = 'w'

base = dirname(realpath(__file__))

header("Installing python dependencies")
subprocess.run(['pip-sync', 'requirements.txt'], env=environ, check=True)

if process_mjml:

    header("Installing node js dependencies")
    subprocess.run(['yarn'], env=environ, cwd=join(base, 'mjml'), check=True)

    header("Removing old templates")

    entries = glob.glob(join(base, 'karrot/*/templates/*.html.jinja2'))
    for entry in entries:
        os.remove(entry)
    print('Removed {} entries'.format(len(entries)))

    header("Generating new templates")
    subprocess.run(['./mjml/convert'], env=environ, check=True)

header('All done ☺')<commit_msg>Auto-install pip-tools if not installed<commit_after>#!/usr/bin/env python

import subprocess
import os
import glob
import sys
from os.path import dirname, realpath, join
import pkgutil

process_mjml = '--no-mjml' not in sys.argv[1:]


def color(num):
    def format(val):
        return '\033[' + str(num) + 'm' + val + '\033[0m'

    return format


green = color(92)
yellow = color(93)


def header(val):
    print('\n', yellow('★'), green(val), '\n')


environ = os.environ.copy()

# do not prompt if dependency source already exists, just wipe and get the required version
environ['PIP_EXISTS_ACTION'] = 'w'

base = dirname(realpath(__file__))

if 'piptools' not in [m.name for m in pkgutil.iter_modules()]:
  header("Installing pip-tools")
  subprocess.run(['pip', 'install', 'pip-tools'], env=environ, check=True)

header("Installing python dependencies")
subprocess.run(['pip-sync', 'requirements.txt'], env=environ, check=True)

if process_mjml:

    header("Installing node js dependencies")
    subprocess.run(['yarn'], env=environ, cwd=join(base, 'mjml'), check=True)

    header("Removing old templates")

    entries = glob.glob(join(base, 'karrot/*/templates/*.html.jinja2'))
    for entry in entries:
        os.remove(entry)
    print('Removed {} entries'.format(len(entries)))

    header("Generating new templates")
    subprocess.run(['./mjml/convert'], env=environ, check=True)

header('All done ☺')
{'fraction_non_alphanumeric': 0.1575942139322421, 'fraction_numerical': 0.0098972211648268, 'mean_word_length': 4.3963039014373715, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5050709', 'n_tokens_mistral': 905, 'n_tokens_neox': 863, 'n_words': 229}