<commit_before>#!/usr/bin/env python2
#Author Phinfinity <<EMAIL>>
'''
This is an attempt by me to write a copy-case detector. It has been
inspired by the diligent efforts by undergraduate students everywhere ;)
'''
import argparse
import os
import re
import sys
from collections import defaultdict
from hashlib import md5
from itertools import combinations

ROOT_DIR = "." # Directory to start crawling at
TRUNCATE_LENGTH=40 # Length to truncate filenames to for display
MIN_PERCENT_MATCH_TO_DSIPLAY = 30.0 # this is only for display not calculation
MIN_LINES_MATCH_TO_DISPLAY = 10 # in addition to percentage
SIZE_LIMIT = 1024*32 # Don't process files bigger than 64K
LINE_TRUNC = 512 # Truncate long lines to first this many characters
REPEAT_TOLERANCE = 10 # Ignore digests which repeat more than 10 times
IGNORE_EXTENSIONS = [".class", ".jar", ".html", ".txt", ".xml", ".png", ".gif", ".css"]
EXTENDED_FILE_PRINT = False
space_remover = re.compile('\s')
digest_list = {} # digest -> line eof text mapping
file_lines = {}

def set_constants_from_argumentes():
    global ROOT_DIR, TRUNCATE_LENGTH, MIN_PERCENT_MATCH_TO_DSIPLAY
    global MIN_LINES_MATCH_TO_DISPLAY, SIZE_LIMIT, LINE_TRUNC, REPEAT_TOLERANCE
    global IGNORE_EXTENSIONS, EXTENDED_FILE_PRINT
    parser = argparse.ArgumentParser()
    parser.add_argument("root_directory",
            help="Root directory to start crawling at. \
                    Every folder inside this is treated as separate submission.\
                    Defaults to current directory.",
            nargs='?',
            default=ROOT_DIR)
    parser.add_argument("-e", "--extended_list", action="store_true",
            help="Enable listing copied files for each submission.")
    parser.add_argument("-r", "--repeat_tolerance", type=int, default=REPEAT_TOLERANCE,
            help="If a section of code occurs more than this many times , it will be treated as boilerplate \
                    and won't be counted towards copied code sections. For performance reasons this needs to \
                    be a small numbers, bigger numbers will quadratically slow down. Defaults to %d" % REPEAT_TOLERANCE)
    parser.add_argument("-s", "--size_limit", type=int, default=SIZE_LIMIT,
            help="Minimum size of a file in bytes , for it to be checked as code for copying. \
                    Defaults to %d" % SIZE_LIMIT)
    parser.add_argument("-t", "--truncate_length", type=int, default=TRUNCATE_LENGTH,
            help="Number of characters to truncate display of files to. \
                     Defaults to %d" % TRUNCATE_LENGTH)
    parser.add_argument("-p", "--min_percent_display", type=float, default=MIN_PERCENT_MATCH_TO_DSIPLAY,
            help="Minimum percentage of file match required to display filename.\
                    Defaults to %.2f" % MIN_PERCENT_MATCH_TO_DSIPLAY)
    parser.add_argument("-l", "--min_lines_display", type=int, default=MIN_LINES_MATCH_TO_DISPLAY,
            help="Minimum number of lines a file match requires to display its name.\
                    Defaults to %d" % MIN_LINES_MATCH_TO_DISPLAY)
    parser.add_argument("--ignore_prefix", type=str, default=",".join(IGNORE_EXTENSIONS),
            help="comma separated list of extensions or file name suffixes to be ignored. Defaults to %s" % ",".join(IGNORE_EXTENSIONS))
    args = parser.parse_args()
    EXTENDED_FILE_PRINT = args.extended_list
    ROOT_DIR = args.root_directory
    TRUNCATE_LENGTH = args.truncate_length
    MIN_PERCENT_MATCH_TO_DSIPLAY = args.min_percent_display
    MIN_LINES_MATCH_TO_DISPLAY = args.min_lines_display
    SIZE_LIMIT = args.size_limit
    REPEAT_TOLERANCE = args.repeat_tolerance
    IGNORE_EXTENSIONS = args.ignore_prefix.split(",")
def trunc_txt(s):
    if len(s) > TRUNCATE_LENGTH:
        s = "..."+s[-32:]<commit_msg>copydiff: Added short argument for ignore list (turns out its reasnobly useful)<commit_after>#!/usr/bin/env python2
#Author Phinfinity <<EMAIL>>
'''
This is an attempt by me to write a copy-case detector. It has been
inspired by the diligent efforts by undergraduate students everywhere ;)
'''
import argparse
import os
import re
import sys
from collections import defaultdict
from hashlib import md5
from itertools import combinations

ROOT_DIR = "." # Directory to start crawling at
TRUNCATE_LENGTH=40 # Length to truncate filenames to for display
MIN_PERCENT_MATCH_TO_DSIPLAY = 30.0 # this is only for display not calculation
MIN_LINES_MATCH_TO_DISPLAY = 10 # in addition to percentage
SIZE_LIMIT = 1024*32 # Don't process files bigger than 64K
LINE_TRUNC = 512 # Truncate long lines to first this many characters
REPEAT_TOLERANCE = 10 # Ignore digests which repeat more than 10 times
IGNORE_EXTENSIONS = [".class", ".jar", ".html", ".txt", ".xml", ".png", ".gif", ".css"]
EXTENDED_FILE_PRINT = False
space_remover = re.compile('\s')
digest_list = {} # digest -> line eof text mapping
file_lines = {}

def set_constants_from_argumentes():
    global ROOT_DIR, TRUNCATE_LENGTH, MIN_PERCENT_MATCH_TO_DSIPLAY
    global MIN_LINES_MATCH_TO_DISPLAY, SIZE_LIMIT, LINE_TRUNC, REPEAT_TOLERANCE
    global IGNORE_EXTENSIONS, EXTENDED_FILE_PRINT
    parser = argparse.ArgumentParser()
    parser.add_argument("root_directory",
            help="Root directory to start crawling at. \
                    Every folder inside this is treated as separate submission.\
                    Defaults to current directory.",
            nargs='?',
            default=ROOT_DIR)
    parser.add_argument("-e", "--extended_list", action="store_true",
            help="Enable listing copied files for each submission.")
    parser.add_argument("-r", "--repeat_tolerance", type=int, default=REPEAT_TOLERANCE,
            help="If a section of code occurs more than this many times , it will be treated as boilerplate \
                    and won't be counted towards copied code sections. For performance reasons this needs to \
                    be a small numbers, bigger numbers will quadratically slow down. Defaults to %d" % REPEAT_TOLERANCE)
    parser.add_argument("-s", "--size_limit", type=int, default=SIZE_LIMIT,
            help="Minimum size of a file in bytes , for it to be checked as code for copying. \
                    Defaults to %d" % SIZE_LIMIT)
    parser.add_argument("-t", "--truncate_length", type=int, default=TRUNCATE_LENGTH,
            help="Number of characters to truncate display of files to. \
                     Defaults to %d" % TRUNCATE_LENGTH)
    parser.add_argument("-p", "--min_percent_display", type=float, default=MIN_PERCENT_MATCH_TO_DSIPLAY,
            help="Minimum percentage of file match required to display filename.\
                    Defaults to %.2f" % MIN_PERCENT_MATCH_TO_DSIPLAY)
    parser.add_argument("-l", "--min_lines_display", type=int, default=MIN_LINES_MATCH_TO_DISPLAY,
            help="Minimum number of lines a file match requires to display its name.\
                    Defaults to %d" % MIN_LINES_MATCH_TO_DISPLAY)
    parser.add_argument("-i", "--ignore_prefix", type=str, default=",".join(IGNORE_EXTENSIONS),
            help="comma separated list of extensions or file name suffixes to be ignored. Defaults to %s" % ",".join(IGNORE_EXTENSIONS))
    args = parser.parse_args()
    EXTENDED_FILE_PRINT = args.extended_list
    ROOT_DIR = args.root_directory
    TRUNCATE_LENGTH = args.truncate_length
    MIN_PERCENT_MATCH_TO_DSIPLAY = args.min_percent_display
    MIN_LINES_MATCH_TO_DISPLAY = args.min_lines_display
    SIZE_LIMIT = args.size_limit
    REPEAT_TOLERANCE = args.repeat_tolerance
    IGNORE_EXTENSIONS = args.ignore_prefix.split(",")
def trunc_txt(s):
    if len(s) > TRUNCATE_LENGTH:
        s = "..."+s[-32:]
{'fraction_non_alphanumeric': 0.09016606123508043, 'fraction_numerical': 0.00700570835495589, 'mean_word_length': 3.466396292004635, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3454405', 'n_tokens_mistral': 2412, 'n_tokens_neox': 2253, 'n_words': 699}