<commit_before>import os
import logging
import sys
import stat  # Interpreting the results of os.[stat,fstat,lstat]
import traceback

def filepaths_in_dir(directory):
    return (os.path.join(root, basename) for root, dirs, files in os.walk(directory) for basename in files)

def validate_directory(path, checkmodes):
    """Check if path is directory and it is not a symlink"""
    if not type(path) == str:
        logging.debug('got path which was not a string. Exiting..')
        sys.exit('validate_directory got path which was not a string')
    if not os.path.isdir(path):
        return False
    if os.path.islink(path):
        return False
    if not check_dir_execution_bit(path, checkmodes):
        return False
    return True


def check_dir_execution_bit(path, checkmodes):
    """Check if path has execution bit to check if site is public. Defaults to false."""
    try:
        if checkmodes == None:
            return True
        if not os.path.exists(path):
            return
        if not os.path.isdir(path):
            return
        # http://docs.python.org/library/stat.html#stat.S_IXOTH
        if stat.S_IXOTH & os.stat(path)[stat.ST_MODE]:
            #logging.debug('Execution bit set for directory: %s' % path)
            return True
        else:
            #logging.debug('No execution bit set for directory: %s' % path)
            return False
    except Exception:<commit_msg>Removed useless lines. Added more documentation.<commit_after>import os
import logging
import sys
import stat  # Interpreting the results of os.[stat,fstat,lstat]
import traceback

def filepaths_in_dir(directory):
    return (os.path.join(root, basename) for root, dirs, files in os.walk(directory) for basename in files)

def validate_directory(path, checkmodes):
    """Check if path is directory and it is not a symlink"""
    if not type(path) == str:
        logging.debug('got path which was not a string. Exiting..')
        sys.exit('validate_directory got path which was not a string')
    if not os.path.isdir(path):
        return False
    if os.path.islink(path):
        return False
    if not check_dir_execution_bit(path, checkmodes):
        return False
    return True


def check_dir_execution_bit(path, checkmodes):
    """Check if path has execution bit to check if site is public.
    Defaults to false. False means no execution bit is set."""
    try:
        if checkmodes == None:
            return True
        if not os.path.exists(path):
            return
        if not os.path.isdir(path):
            return
        # http://docs.python.org/library/stat.html#stat.S_IXOTH
        if stat.S_IXOTH & os.stat(path)[stat.ST_MODE]:
            return True
        else:
            return False
    except Exception:
{'fraction_non_alphanumeric': 0.09005424954792043, 'fraction_numerical': 0.0, 'mean_word_length': 2.615686274509804, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3897642', 'n_tokens_mistral': 808, 'n_tokens_neox': 789, 'n_words': 274}