<commit_before>"""Find spark home, and add pyspark to sys.path.

If SPARK_HOME is defined, it will be used to put pyspark on sys.path.
Otherwise, common locations for spark (currently only Homebrew's default) will be searched.
"""

import glob
import os
import sys

__version__ = '0.0.1'

def find_spark(spark_home=None):
    """Find spark and make pyspark importable.
    
    Ensures SPARK_HOME is set and adds pyspark's location inside SPARK_HOME to sys.path.
    
    Common
    """
    if not spark_home:
        spark_home = os.environ.get('SPARK_HOME', None)
    if not spark_home:
        for path in [
            '/usr/local/opt/apache-spark/libexec', # OS X Homebrew
            # Any other common places to look?
        ]:
            if os.path.exists(path):
                spark_home = path
                break
    
    if not spark_home:
        raise ValueError("Couldn't find spark, please specify spark_home arg or SPARK_HOME env.")
    
    # ensure SPARK_HOME is defined
    os.environ['SPARK_HOME'] = spark_home<commit_msg>fix glob import<commit_after>"""Find spark home, and add pyspark to sys.path.

If SPARK_HOME is defined, it will be used to put pyspark on sys.path.
Otherwise, common locations for spark (currently only Homebrew's default) will be searched.
"""

from glob import glob
import os
import sys

__version__ = '0.0.2'

def find_spark(spark_home=None):
    """Find spark and make pyspark importable.
    
    Ensures SPARK_HOME is set and adds pyspark's location inside SPARK_HOME to sys.path.
    
    Common
    """
    if not spark_home:
        spark_home = os.environ.get('SPARK_HOME', None)
    if not spark_home:
        for path in [
            '/usr/local/opt/apache-spark/libexec', # OS X Homebrew
            # Any other common places to look?
        ]:
            if os.path.exists(path):
                spark_home = path
                break
    
    if not spark_home:
        raise ValueError("Couldn't find spark, please specify spark_home arg or SPARK_HOME env.")
    
    # ensure SPARK_HOME is defined
    os.environ['SPARK_HOME'] = spark_home
{'fraction_non_alphanumeric': 0.08064516129032258, 'fraction_numerical': 0.0028462998102466793, 'mean_word_length': 2.474464579901153, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7095626', 'n_tokens_mistral': 661, 'n_tokens_neox': 634, 'n_words': 221}