<commit_before>""" grinder module """

import configparser
from .repository import Repository
from .seasoning import Seasoning
import os
import shutil

# Source configuration
conf = configparser.ConfigParser()
conf.read("conf/grinder.conf")

print("Configuration loaded.")
print("Base directory for environments:  " + str(conf['environment']['environmentdir']))
print("Remote Salt States repository:  " + str(conf['environment']['statesrepo']))
print("Remote Pillar repository:  " + str(conf['environment']['pillarrepo']))

# Set repositories
statesrepo = repository.Repository(conf['environment']['statesrepo'], conf['environment']['environmentdir'], "states")
pillarrepo = repository.Repository(conf['environment']['pillarrepo'], conf['environment']['environmentdir'], "pillar")

# Read repository master branches
statebranches = statesrepo.lsremote()
pillarbranches = pillarrepo.lsremote()

# Clean up old environment
shutil.rmtree(conf['environment']['environmentdir'])

# Make directory structure for the branch
print("Branches of Salt States:  ")
for key in statebranches:
    if key != "HEAD":
        branch = key.split('/')[-1]
        print(branch)
        os.makedirs(conf['environment']['environmentdir'] + "/" + branch, 0o755)

    # Clone branch
    statesrepo.clone(branch)

    # Parse the Seasoning file
    s = seasoning.Seasoning(conf['environment']['environmentdir'], branch)
    s.deploy()

print("Branches of Pillar:  ")
for key in pillarbranches:
    if key != "HEAD":
        branch = key.split('/')[-1]
        print(branch)

        # Clone branch
        pillarrepo.clone(branch)<commit_msg>Added main function<commit_after>""" grinder module """

import configparser
from .repository import Repository
from .seasoning import Seasoning
import os
import shutil

def main():
    # Source configuration
    conf = configparser.ConfigParser()
    conf.read("/etc/grinder/grinder.conf")

    print("Configuration loaded.")
    print("Base directory for environments:  " + str(conf['environment']['environmentdir']))
    print("Remote Salt States repository:  " + str(conf['environment']['statesrepo']))
    print("Remote Pillar repository:  " + str(conf['environment']['pillarrepo']))

    # Set repositories
    statesrepo = repository.Repository(conf['environment']['statesrepo'], conf['environment']['environmentdir'], "states")
    pillarrepo = repository.Repository(conf['environment']['pillarrepo'], conf['environment']['environmentdir'], "pillar")

    # Read repository master branches
    statebranches = statesrepo.lsremote()
    pillarbranches = pillarrepo.lsremote()

    # Clean up old environment
    shutil.rmtree(conf['environment']['environmentdir'])

    # Make directory structure for the branch
    print("Branches of Salt States:  ")
    for key in statebranches:
        if key != "HEAD":
            branch = key.split('/')[-1]
            print(branch)
            os.makedirs(conf['environment']['environmentdir'] + "/" + branch, 0o755)

        # Clone branch
        statesrepo.clone(branch)

        # Parse the Seasoning file
        s = seasoning.Seasoning(conf['environment']['environmentdir'], branch)
        s.deploy()

    print("Branches of Pillar:  ")
    for key in pillarbranches:
        if key != "HEAD":
            branch = key.split('/')[-1]
            print(branch)

            # Clone branch
            pillarrepo.clone(branch)

if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.14023858015711377, 'fraction_numerical': 0.0034914169333721268, 'mean_word_length': 4.048458149779735, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5282371', 'n_tokens_mistral': 946, 'n_tokens_neox': 902, 'n_words': 229}