<commit_before>#!/usr/bin/env python

import argparse
import os
import shlex
import stat
import subprocess
import sys

# Gotta have access to the special places.
if os.geteuid() != 0:
    print("Must be root to run this script!")
    sys.exit(1)

# Sets the default hash function.
HASHER = '/usr/bin/openssl sha1'
MAILTO = ''


def main(input=None, output=None):
    # Get the inventory of everything on the filesystem and then sort it
    # alphabetically.
    filesystem_inventory = inventory_filesystem()
    
    # Cross-reference input with inventory, if applicable.
    if input:
        filesystem_inventory = crossref_inventory(input, filesystem_inventory)
    
    # Sort the list for neatness.
    filesystem_inventory = sorted(filesystem_inventory, key=lambda entry: entry[0].lower())
    
    # Print the findings. Tabs are used for separation because they're parsable.
    print("Total dangerous items found: {}").format(len(filesystem_inventory))
    with open(output, "w"):
        pass
    for entry in filesystem_inventory:
        line = "{path}\t{mtime}\t{hash}".format(
            path  = os.path.abspath(entry[0]),
            mtime = entry[1] if entry[1] else "",
            hash  = entry[2] if entry[2] else ""
        )
        if output:
            with open(output, "a") as out_file:
                out_file.write(line + "\n")
        else:
            print(line)


def crossref_inventory(in_file, inventory):
    """
    Takes the contents of a previous run and cross-references them against the
    new findings, eliminating from the list anything that appears in both
    places.
    
    :param in_file: a file containing a list of files generated by this script
    :param inventory: a list of tuples of file information from this script
    :return: the inventory less the duplicated information
    """
    # Check the file path is valid.
    if not os.path.isfile(in_file):
        raise ValueError("Invalid input file given: {}").format(in_file)
    
    # Pull in all of the lines of the file.
    lines = []
    with open(in_file) as f:
        lines = f.read().splitlines()
    lines = [x for x in lines if x] # remove blanks
    
    # Iterate over the lines, and match to the items in inventory. If a match is
    # found, remove it from the inventory. Then return the trimmed-down list of
    # bad files.
    for line in lines:
        path, mtime, hash = line.split('\t')
        tuple = (path, mtime, hash)
        if tuple in inventory:
            inventory.remove(tuple)
    
    return inventory


def inventory_filesystem():
    """
    Composes an inventory of all attaches filesystems and finds files with the
    SUID or GUID bits set.
    
    :return: a list of tuples containing (file, modification time, hash)<commit_msg>Updated comparison checking<commit_after>#!/usr/bin/env python

import argparse
import os
import shlex
import stat
import subprocess
import sys

# Gotta have access to the special places.
if os.geteuid() != 0:
    print("Must be root to run this script!")
    sys.exit(1)

# Sets the default hash function.
HASHER = '/usr/bin/openssl sha1'
MAILTO = ''


def main(in_file=None, output=None):
    # Get the inventory of everything on the filesystem and then sort it
    # alphabetically.
    filesystem_inventory = inventory_filesystem()
    
    # Cross-reference in_file with inventory, if applicable.
    if in_file:
        print("Cross-referencing found items against {}").format(in_file)
        filesystem_inventory = crossref_inventory(in_file, filesystem_inventory)
    
    # Sort the list for neatness.
    filesystem_inventory = sorted(filesystem_inventory, key=lambda entry: entry[0].lower())
    
    # Print the findings. Tabs are used for separation because they're parsable.
    print("Total dangerous items found: {}").format(len(filesystem_inventory))
    with open(output, "w"):
        pass
    for entry in filesystem_inventory:
        line = "{path}\t{mtime}\t{hash}".format(
            path  = os.path.abspath(entry[0]),
            mtime = entry[1],
            hash  = entry[2]
        )
        if output:
            with open(output, "a") as out_file:
                out_file.write(line + "\n")
        else:
            print(line)


def crossref_inventory(in_file, inventory):
    """
    Takes the contents of a previous run and cross-references them against the
    new findings, eliminating from the list anything that appears in both
    places.
    
    :param in_file: a file containing a list of files generated by this script
    :param inventory: a list of tuples of file information from this script
    :return: the inventory less the duplicated information
    """
    # Check the file path is valid.
    if not os.path.isfile(in_file):
        raise ValueError("Invalid input file given: {}").format(in_file)
    
    # Pull in all of the lines of the file.
    lines = []
    with open(in_file) as f:
        lines = f.read().splitlines()
    lines = [x for x in lines if x] # remove blanks
    
    # Iterate over the lines, and match to the items in inventory.
    results = []
    for line in lines:
        path, mtime, hash = line.split('\t')
        tuple = (path, mtime, hash)
        results.append(tuple)
    
    # Remove duplicates.
    results = [x for x in results if all(x[0] == y[0] and x[1] == y[1] and x[2] == y[2] for y in inventory)]
    
    return results


def inventory_filesystem():
    """
    Composes an inventory of all attaches filesystems and finds files with the
    SUID or GUID bits set.
    
    :return: a list of tuples containing (file, modification time, hash)
{'fraction_non_alphanumeric': 0.09011731247778172, 'fraction_numerical': 0.0039104159260575895, 'mean_word_length': 2.701973684210526, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3544461', 'n_tokens_mistral': 1647, 'n_tokens_neox': 1623, 'n_words': 614}