<commit_before>import re
import sys

# reference: https://stackoverflow.com/questions/35761133/python-how-to-check-for-open-and-close-tags
def stack_tag(tag, stack):
    t = tag[1:-1]
    first_space = t.find(' ')
    #print(t)
    if t[-1:] == '/':
        self_closed_tag = True
    elif t[:1] != '/':
        # Add tag to stack
        if first_space == -1:
            stack.append(t)
            # print("TRACE open", stack)
        else:
            stack.append(t[:first_space])
            # print("TRACE open", stack)
    else:
        if first_space != -1:
            t = t[1:first_space]
        else:
            t = t[1:]

        if len(stack) == 0:
            # print("No blocks are open; tried to close", t)
            closed_tag = True
        else:
            if stack[-1] == t:
                # Close the block
                stack.pop()
                # print("TRACE close", t, stack)
            else:
                # print("Tried to close", t, "but most recent open block is", stack[-1])
                if t in stack:
                    stack.remove(t)
                    # print("Prior block closed; continuing")

    # if len(stack):
    #     print("Blocks still open at EOF:", stack)
    return stack

def tag_is_wrapped(pos, content):
    tag_start = pos[0]
    tag_end = pos[1]
    content_previous = content[:tag_start][::-1] # reverse content_previous
    content_later = content[tag_end:]

    left_wraps_findall = re.findall(r'`', content_previous)
    left_single_backtick = len(left_wraps_findall) % 2
    right_wraps_findall = re.findall(r'`', content_later)
    right_single_backtick = len(right_wraps_findall) % 2
    # print(left_single_backtick, right_single_backtick)

    if left_single_backtick != 0 and right_single_backtick != 0:
        # print(content_previous.find('`'), content_later.find('`'))
        # print(content_previous)
        # print(content_later)
        return True
    else:
        # print(content_previous.find('`'), content_later.find('`'))
        # print(content_previous)
        # print(content_later)
        return False

def filter_content(content):
    content_findall = re.findall(r'\n---\n', content)
    if len(content_findall):
        content_finditer = re.finditer(r'\n---\n', content)
        for i in content_finditer:
            meta_pos = i.span()[1]
            # print(content[meta_pos:])
            return content[meta_pos:]
    else:
        return content

status_code = 0

# print(sys.argv[1:])
for filename in sys.argv[1:]:
    #print("Checking " + filename + "......\n")
    with open(filename, 'r') as file:
        content = file.read()

        content = filter_content(content)
        result_findall = re.findall(r'<([^\n`>]*)>', content)
        if len(result_findall) == 0:
            # print("The edited markdown file " + filename + " has no tags!\n")
            continue
        else:
            result_finditer = re.finditer(r'<([^\n`>]*)>', content)
            stack = []
            for i in result_finditer:
                # print(i.group(), i.span())
                tag = i.group()
                pos = i.span()

                if tag[:4] == '<!--' and tag[-3:] == '-->':
                    continue
                elif content[pos[0]-2:pos[0]] == '{{' and content[pos[1]:pos[1]+2] == '}}':
                    # print(tag) # filter copyable shortcodes
                    continue
                elif tag[:5] == '<http': # or tag[:4] == '<ftp'
                    # filter urls
                    continue
                elif tag_is_wrapped(pos, content):
                    # print(content[int(pos[0])-1:int(pos[1]+1)])
                    # print(tag, 'is wrapped by backticks!')
                    continue

                stack = stack_tag(tag, stack)

            if len(stack):
                stack = ['<' + i + '>' for i in stack]
                print("ERROR: " + filename + ' has unclosed tags: ' + ', '.join(stack) + '.\n')
                status_code = 1

if status_code:
    print("HINT: Unclosed tags will cause website build failure. Please fix the reported unclosed tags. You can use backticks `` to wrap them or close them. Thanks.")
    exit(1)
<commit_msg>use os.path.isfile<commit_after>import re
import sys
import os

# reference: https://stackoverflow.com/questions/35761133/python-how-to-check-for-open-and-close-tags
def stack_tag(tag, stack):
    t = tag[1:-1]
    first_space = t.find(' ')
    #print(t)
    if t[-1:] == '/':
        self_closed_tag = True
    elif t[:1] != '/':
        # Add tag to stack
        if first_space == -1:
            stack.append(t)
            # print("TRACE open", stack)
        else:
            stack.append(t[:first_space])
            # print("TRACE open", stack)
    else:
        if first_space != -1:
            t = t[1:first_space]
        else:
            t = t[1:]

        if len(stack) == 0:
            # print("No blocks are open; tried to close", t)
            closed_tag = True
        else:
            if stack[-1] == t:
                # Close the block
                stack.pop()
                # print("TRACE close", t, stack)
            else:
                # print("Tried to close", t, "but most recent open block is", stack[-1])
                if t in stack:
                    stack.remove(t)
                    # print("Prior block closed; continuing")

    # if len(stack):
    #     print("Blocks still open at EOF:", stack)
    return stack

def tag_is_wrapped(pos, content):
    tag_start = pos[0]
    tag_end = pos[1]
    content_previous = content[:tag_start][::-1] # reverse content_previous
    content_later = content[tag_end:]

    left_wraps_findall = re.findall(r'`', content_previous)
    left_single_backtick = len(left_wraps_findall) % 2
    right_wraps_findall = re.findall(r'`', content_later)
    right_single_backtick = len(right_wraps_findall) % 2
    # print(left_single_backtick, right_single_backtick)

    if left_single_backtick != 0 and right_single_backtick != 0:
        # print(content_previous.find('`'), content_later.find('`'))
        # print(content_previous)
        # print(content_later)
        return True
    else:
        # print(content_previous.find('`'), content_later.find('`'))
        # print(content_previous)
        # print(content_later)
        return False

def filter_content(content):
    content_findall = re.findall(r'\n---\n', content)
    if len(content_findall):
        content_finditer = re.finditer(r'\n---\n', content)
        for i in content_finditer:
            meta_pos = i.span()[1]
            # print(content[meta_pos:])
            return content[meta_pos:]
    else:
        return content

status_code = 0

# print(sys.argv[1:])
for filename in sys.argv[1:]:
    #print("Checking " + filename + "......\n")
    if os.path.isfile(filename):
        file = open(filename, "r" )
        content = file.read()
        file.close()

        content = filter_content(content)
        result_findall = re.findall(r'<([^\n`>]*)>', content)
        if len(result_findall) == 0:
            # print("The edited markdown file " + filename + " has no tags!\n")
            continue
        else:
            result_finditer = re.finditer(r'<([^\n`>]*)>', content)
            stack = []
            for i in result_finditer:
                # print(i.group(), i.span())
                tag = i.group()
                pos = i.span()

                if tag[:4] == '<!--' and tag[-3:] == '-->':
                    continue
                elif content[pos[0]-2:pos[0]] == '{{' and content[pos[1]:pos[1]+2] == '}}':
                    # print(tag) # filter copyable shortcodes
                    continue
                elif tag[:5] == '<http': # or tag[:4] == '<ftp'
                    # filter urls
                    continue
                elif tag_is_wrapped(pos, content):
                    # print(content[int(pos[0])-1:int(pos[1]+1)])
                    # print(tag, 'is wrapped by backticks!')
                    continue

                stack = stack_tag(tag, stack)

            if len(stack):
                stack = ['<' + i + '>' for i in stack]
                print("ERROR: " + filename + ' has unclosed tags: ' + ', '.join(stack) + '.\n')
                status_code = 1

if status_code:
    print("HINT: Unclosed tags will cause website build failure. Please fix the reported unclosed tags. You can use backticks `` to wrap them or close them. Thanks.")
    exit(1)

{'fraction_non_alphanumeric': 0.13967730538216935, 'fraction_numerical': 0.01107054528324108, 'mean_word_length': 1.8363393453573815, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4188273', 'n_tokens_mistral': 2679, 'n_tokens_neox': 2564, 'n_words': 602}