<commit_before>#!/usr/bin/env python3

# Copyright 2015 <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import glob
import os

def find_lines_from_file(file):
    lines = list()
    try:
        with open(file, "r") as infile:
            data = [line.strip() for line in infile]
            for line in data:
                if (line[0:2] == "//") and (1 <= line.count(";") <= 2):
                    if (len(line[line.rfind(";") + 1:].strip()) == 10):
                        lines.append(line)
    except UnicodeDecodeError:
        pass
    return lines

def find_lines_from_path(path):
    lines = list()
    for item in glob.glob("{}/*".format(path)):
        if os.path.isdir(item):
            for line in find_lines_from_path(item):
                lines.append(line)
        elif os.path.isfile(item):
            for line in find_lines_from_file(item):
                lines.append(line)
    return lines

def main():
    lines = list()
    print("<table>")
    print("\t<tr>")
    for item in ["Name", "URL", "Date Retrieved"]:
        print("\t\t<td nowrap=\"nowrap\">{}</td>".format(item))
    print("\t</tr>")
    for line in sorted(list(set(find_lines_from_path("{}/app/src".format(os.getcwd()))))):
        line = line[2:].strip().split(";")
        if len(line) == 2:
            line = ["N/A", line[0], line[1]]
        print("\t<tr>")
        for item in line:
            print("\t\t<td nowrap=\"nowrap\">{}</td>".format(item.strip()))
        print("\t</tr>")
    print("</table>")

if __name__ == "__main__":
    main()<commit_msg>Updated citation script to handle xml citations<commit_after>#!/usr/bin/env python3

# Copyright 2015 <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import glob
import os

def find_lines_from_file(file):
    lines = list()
    try:
        with open(file, "r") as infile:
            data = [line.strip() for line in infile]
            for line in data:
                if line[:2] == "//":
                    line = line[2:]
                elif (line[:4] == "<!--") and (line[-3:] == "-->"):
                    line = line[4:-3]
                else:
                    continue
                if (1 <= line.count(";") <= 2) and (len(line[line.rfind(";") + 1:].strip()) == 10):
                        lines.append(line)
    except UnicodeDecodeError:
        pass
    return lines

def find_lines_from_path(path):
    lines = list()
    for item in glob.glob("{}/*".format(path)):
        if os.path.isdir(item):
            for line in find_lines_from_path(item):
                lines.append(line)
        elif os.path.isfile(item):
            for line in find_lines_from_file(item):
                lines.append(line)
    return lines

def main():
    lines = list()
    print("<table>")
    print("\t<tr>")
    for item in ["Name", "URL", "Date Retrieved"]:
        print("\t\t<td nowrap=\"nowrap\">{}</td>".format(item))
    print("\t</tr>")
    for line in sorted(list(set(find_lines_from_path("{}/app/src".format(os.getcwd()))))):
        line = line.strip().split(";")
        if len(line) == 2:
            line = ["N/A", line[0], line[1]]
        print("\t<tr>")
        for item in line:
            print("\t\t<td nowrap=\"nowrap\">{}</td>".format(item.strip()))
        print("\t</tr>")
    print("</table>")

if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.1590394200271862, 'fraction_numerical': 0.009741730856366107, 'mean_word_length': 2.3987682832948423, 'pattern_counts': {'":': 3, '<': 40, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1898280', 'n_tokens_mistral': 1375, 'n_tokens_neox': 1355, 'n_words': 403}