<commit_before>import os
import db
import puzpy
import datetime
import json
import xml.etree.ElementTree as xml_tree

class Manager(object):
    database = None
    base_path = None
    crossword_path = None

    def __init__(self, base_path, db_file):
        self.database = db.DB(db_file)
        self.base_path = base_path
        self.crossword_path = os.path.join(self.base_path, "crosswords")
        self.scan_puzzles()
        return

    def scan_puzzles(self):
        puzzles = self.database.select_puzzles()
        for obj in os.listdir(self.crossword_path):
            full_path = os.path.join(self.crossword_path, obj)
            js = None
            if os.path.isfile(full_path) and obj.endswith(".puz"):
                js = self.read_puz(full_path)
                title = obj.replace(".puz", "")
            if os.path.isfile(full_path) and obj.endswith(".xml"):
                js = self.read_xml(full_path)
                title = obj.replace(".xml", "")

            if js and title not in map(lambda p: p["Title"], puzzles):
                self.database.insert_puzzle(title, db.get_timestamp(datetime.datetime.utcnow()), json.dumps(js))

    def read_puz(self, path):
        return self.puz_to_json(puzpy.read(path))
    def read_xml(self, path):
        xml = None
        with open(path, 'r') as obj:
            xml = obj.read()
            xml = xml.replace( "xmlns=", "blank=")
        return self.xml_to_json(xml_tree.fromstring(xml))

    def xml_to_json(self, puzzle):
        js = dict()

        crossword = puzzle.find(".//crossword")
        grid = crossword.find("./grid")
        js["height"] = int(grid.attrib["height"])
        js["width"] = int(grid.attrib["width"])

        js["rows"] = [[]]
        js["clues"] = [{},{}]

        for x in range(1, js["width"]+1):
            row = []
            for y in range(1, js["height"]+1):
                cell = grid.find("./cell[@x='{0}'][@y='{1}']".format(x,y))
                cell = {
                    "clue" : int(cell.attrib["number"]) if "number" in cell.attrib.keys() else None,
                    "black" : cell.attrib["type"] == "block" if "type" in cell.attrib.keys() else None
                }
                row.append(cell)
            js["rows"].append(row)

        words = crossword.findall("./word")
        for word in words:
            clue = crossword.find(".//clue[@word='{0}']".format(word.attrib["id"]))
            number = clue.attrib["number"]
            clue_index = 1 if "-" in word.attrib["x"] else 0

            js["clues"][clue_index][number] = clue.text<commit_msg>fix xml puzzle orientation<commit_after>import os
import db
import puzpy
import datetime
import json
import xml.etree.ElementTree as xml_tree

class Manager(object):
    database = None
    base_path = None
    crossword_path = None

    def __init__(self, base_path, db_file):
        self.database = db.DB(db_file)
        self.base_path = base_path
        self.crossword_path = os.path.join(self.base_path, "crosswords")
        self.scan_puzzles()
        return

    def scan_puzzles(self):
        puzzles = self.database.select_puzzles()
        for obj in os.listdir(self.crossword_path):
            full_path = os.path.join(self.crossword_path, obj)
            js = None
            if os.path.isfile(full_path) and obj.endswith(".puz"):
                js = self.read_puz(full_path)
                title = obj.replace(".puz", "")
            if os.path.isfile(full_path) and obj.endswith(".xml"):
                js = self.read_xml(full_path)
                title = obj.replace(".xml", "")

            if js and title not in map(lambda p: p["Title"], puzzles):
                self.database.insert_puzzle(title, db.get_timestamp(datetime.datetime.utcnow()), json.dumps(js))

    def read_puz(self, path):
        return self.puz_to_json(puzpy.read(path))
    def read_xml(self, path):
        xml = None
        with open(path, 'r') as obj:
            xml = obj.read()
            xml = xml.replace( "xmlns=", "blank=")
        return self.xml_to_json(xml_tree.fromstring(xml))

    def xml_to_json(self, puzzle):
        js = dict()

        crossword = puzzle.find(".//crossword")
        grid = crossword.find("./grid")
        js["height"] = int(grid.attrib["height"])
        js["width"] = int(grid.attrib["width"])

        js["rows"] = [[]]
        js["clues"] = [{},{}]

        for y in range(1, js["height"]+1):
            row = []
            for x in range(1, js["width"]+1):
                cell = grid.find("./cell[@x='{0}'][@y='{1}']".format(x,y))
                cell = {
                    "clue" : int(cell.attrib["number"]) if "number" in cell.attrib.keys() else None,
                    "black" : cell.attrib["type"] == "block" if "type" in cell.attrib.keys() else None
                }
                row.append(cell)
            js["rows"].append(row)

        words = crossword.findall("./word")
        for word in words:
            clue = crossword.find(".//clue[@word='{0}']".format(word.attrib["id"]))
            number = clue.attrib["number"]
            clue_index = 1 if "-" in word.attrib["x"] else 0

            js["clues"][clue_index][number] = clue.text
{'fraction_non_alphanumeric': 0.15313759355210133, 'fraction_numerical': 0.0034542314335060447, 'mean_word_length': 2.0658823529411765, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 31}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4968509', 'n_tokens_mistral': 1702, 'n_tokens_neox': 1647, 'n_words': 336}