<commit_before>from repodiff.primary import PrimaryMetadata, PrimaryPackage
from repodiff.filelists import FilelistsMetadata, FilelistsPackage
from repodiff.other import OtherMetadata, OtherPackage
from repodiff.repomd import RepomdMetadata, RepomdData
from repodiff.repo import OneRepo, CompleteRepo
from lxml import etree
import os
import gzip
import bz2
import tempfile
import shutil
import os.path
try:
    import sqlite3 as sqlite
except ImportError:
    import sqlite

class XmlRepo(object):
    pass

class Repo(object):
    pass

# namespace
PRI_NS = "{http://linux.duke.edu/metadata/common}"
FIL_NS = "{http://linux.duke.edu/metadata/filelists}"
OTH_NS = "{http://linux.duke.edu/metadata/other}"
MD_NS = "{http://linux.duke.edu/metadata/repo}"

#
# XML
#

def _parse_pco(elem, requires=False):
    req_set = set([])
    for felem in elem:
        if felem.tag.endswith("entry"):
            res = (felem.get("name"),
                   felem.get("flags"),
                   felem.get("epoch"),
                   felem.get("ver"),
                   felem.get("rel"))
            if requires:
                res += (bool(felem.get("pre")),)
        req_set.add(res)
    return req_set


def primarymetadata_from_xml_factory(xmlpath, archpath):
    pri_obj = PrimaryMetadata(xmlpath, archpath)
    for _, elements in etree.iterparse(open(xmlpath), tag="%spackage" % PRI_NS):
        pp = PrimaryPackage()
        for elem in elements:
            if elem.tag.endswith("name"):
                pp.name = elem.text<commit_msg>Fix parsing of pre argument<commit_after>from repodiff.primary import PrimaryMetadata, PrimaryPackage
from repodiff.filelists import FilelistsMetadata, FilelistsPackage
from repodiff.other import OtherMetadata, OtherPackage
from repodiff.repomd import RepomdMetadata, RepomdData
from repodiff.repo import OneRepo, CompleteRepo
from lxml import etree
import os
import gzip
import bz2
import tempfile
import shutil
import os.path
try:
    import sqlite3 as sqlite
except ImportError:
    import sqlite

class XmlRepo(object):
    pass

class Repo(object):
    pass

# namespace
PRI_NS = "{http://linux.duke.edu/metadata/common}"
FIL_NS = "{http://linux.duke.edu/metadata/filelists}"
OTH_NS = "{http://linux.duke.edu/metadata/other}"
MD_NS = "{http://linux.duke.edu/metadata/repo}"

#
# XML
#
import sys
def _parse_pco(elem, requires=False):
    req_set = set([])
    for felem in elem:
        if felem.tag.endswith("entry"):
            res = (felem.get("name"),
                   felem.get("flags"),
                   felem.get("epoch"),
                   felem.get("ver"),
                   felem.get("rel"))
            if requires:
                res += (bool(int(felem.get("pre", 0))),)
        req_set.add(res)
    return req_set


def primarymetadata_from_xml_factory(xmlpath, archpath):
    pri_obj = PrimaryMetadata(xmlpath, archpath)
    for _, elements in etree.iterparse(open(xmlpath), tag="%spackage" % PRI_NS):
        pp = PrimaryPackage()
        for elem in elements:
            if elem.tag.endswith("name"):
                pp.name = elem.text
{'fraction_non_alphanumeric': 0.11387096774193549, 'fraction_numerical': 0.0016129032258064516, 'mean_word_length': 2.940279542566709, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '194267', 'n_tokens_mistral': 983, 'n_tokens_neox': 964, 'n_words': 230}